diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..b5685772804c8af4235a8504dc6752bfc9ae5d1d --- /dev/null +++ b/Makefile @@ -0,0 +1,13 @@ +.PHONY: style format + + +style: + python -m black --line-length 119 . + python -m isort . + ruff check --fix . + + +quality: + python -m black --check --line-length 119 . + python -m isort --check-only . + ruff check . diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..305035d91c584a7976af6437db9f4e10ef9722e9 --- /dev/null +++ b/app.py @@ -0,0 +1,827 @@ +import gradio as gr +from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns +import pandas as pd +from apscheduler.schedulers.background import BackgroundScheduler +from huggingface_hub import snapshot_download +from src.about import CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TITLE +from src.tasks import TASK_DESCRIPTIONS, MEASURE_DESCRIPTION +from src.display.css_html_js import custom_css +from src.display.utils import BENCHMARK_COLS, COLS, EVAL_COLS, EVAL_TYPES, AutoEvalColumn, ModelType, fields, WeightType, Precision +from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN +from src.populate import get_evaluation_queue_df, get_leaderboard_df +from src.submission.submit import add_new_eval +import random +import matplotlib.pyplot as plt +import re +import plotly.express as px +import plotly.graph_objects as go +import numpy as np + + +def mean_of_max_per_field(df): + """ + Calcola il massimo per ciascun campo e poi la media dei massimi. + + Args: + df (pd.DataFrame): DataFrame con colonne TE, SA, HS, AT, WIC, FAQ, LS, SU, NER, REL + + Returns: + float: media dei valori massimi dei campi + """ + #fields = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"] + fields = ["NER", "REL"] + #print(df.columns) + + # Controlla che tutte le colonne esistano nel DataFrame + missing = [f for f in fields if f not in df.columns] + if missing: + raise ValueError(f"Le seguenti colonne mancano nel DataFrame: {missing}") + + # Calcola il massimo per ciascun campo + max_values = df[fields].max() + + # Calcola la media dei massimi + mean_max = max_values.mean() + + return mean_max + + +def barplot_mean_few_minus_zero_shot(dataframe, tasks=None): + if tasks is None: + tasks = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"] + + task_means = {} + + for task in tasks: + if task not in dataframe.columns: + continue + + # Separa few-shot e zero-shot + few_shot = dataframe[dataframe['IS_FS'] == True][["Model", task]] + zero_shot = dataframe[dataframe['IS_FS'] == False][["Model", task]] + + # Allinea i modelli + merged = pd.merge(few_shot, zero_shot, on="Model", suffixes=("_few", "_zero")) + + # Rimuovi righe con valori mancanti + merged = merged.dropna(subset=[f"{task}_few", f"{task}_zero"]) + + if merged.empty: + continue + + # Calcola differenza few - zero + diff = merged[f"{task}_few"] - merged[f"{task}_zero"] + + # Calcola la media + task_means[task] = diff.mean() + + # Crea barplot + fig = go.Figure([go.Bar( + x=list(task_means.keys()), + y=list(task_means.values()), + marker_color="#ff7f0e", + text=[f"{v:.2f}" for v in task_means.values()], + textposition="outside", + hovertemplate="%{x}
Mean Delta Accuracy: %{y:.2f}%" + )]) + + # Linea di riferimento a 0 + ''' + fig.add_shape( + type="line", + x0=-0.5, x1=len(task_means) - 0.5, + y0=0, y1=0, + line=dict(color="black", width=2, dash="dash"), + xref="x", yref="y" + ) + ''' + + fig.update_layout( + title="Mean Accuracy Difference (Few-shot − Zero-shot) per Task", + xaxis_title="", + yaxis_title="Mean Delta Combined Performance", + template="plotly_white", + font=dict(family="Arial", size=13), + #margin=dict(b=100) + ) + + fig.add_annotation( + text="10-shot learning generally outperforms zero-shot.
" + "", + xref="paper", yref="paper", + x=0, y=-0.2, + showarrow=False, + font=dict(size=11, color="gray"), + align="left" + ) + + return fig + + +def boxplot_per_task(dataframe=None, baselines=None, references=None): + + #print(dataframe.columns) + + #tasks = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"] + tasks =["NER", "REL"] + if dataframe is None: + np.random.seed(42) + dataframe = pd.DataFrame({ + task: np.random.uniform(0.4, 0.9, 20) * 100 + for task in tasks + }) + + if baselines is None: + baselines = {task: np.random.randint(50, 70) for task in tasks} + + colors = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd", + "#8c564b", "#e377c2", "#7f7f7f", "#bcbd22", "#17becf"] + + fig = go.Figure() + + for i, task in enumerate(tasks): + if task in dataframe.columns: + y_data = dataframe[task].dropna().tolist() + + # boxplot + fig.add_trace(go.Box( + y=y_data, + name=task, + marker=dict(color=colors[i]), + line=dict(color="black", width=2), + fillcolor=colors[i], + opacity=0.7, + hovertemplate=""+task+"
Accuracy: %{y:.2f}%", + width=0.6, + whiskerwidth=0.2, + quartilemethod="linear" + )) + + # baseline + if task in baselines and baselines[task] is not None: + fig.add_shape( + type="line", + x0=i - 0.3, x1=i + 0.3, + y0=baselines[task], y1=baselines[task], + line=dict(color="black", width=2, dash="dot"), # più visibile + xref="x", yref="y" + ) + ''' + fig.add_annotation( + x=i, y=baselines[task], + text=f"{baselines[task]}%", + showarrow=False, + yshift=10, + font=dict(size=10, color="black") + ) + ''' + + # reference GPT-4o + if task in references and references[task] is not None: + fig.add_shape( + type="line", + x0=i - 0.3, x1=i + 0.3, + y0=references[task], y1=references[task], + line=dict(color="red", width=2, dash="dashdot"), + xref="x", yref="y" + ) + + fig.update_layout( + title="Distribution of Model Accuracy by Task", + xaxis_title="Task", + yaxis_title="Combined Performance", + template="plotly_white", + boxmode="group", + dragmode=False, + font=dict(family="Arial", size=10), + margin=dict(b=80), + ) + + fig.add_annotation( + text=("" + #"In tasks like TE and SA, models approach the accuracy of supervised
" + #"models at EVALITA (dashed black line); in NER and REL they remain lower.
" + # "Dashed red lines show GPT-4o reference results for generative tasks." + ), + xref="paper", yref="paper", + x=0.5, y=-0.30, + showarrow=False, + font=dict(size=11, color="gray"), + align="left" + ) + + fig.update_yaxes(range=[0, 100], fixedrange=True) + + return fig + +# EVALITA results +BASELINES = { + "TE":71.00, "SA": 66.38, "HS": 80.88, "AT": 82.40, "WIC": 85.00, + "LS": 38.82, "SU": 38.91, "NER":88.00, "REL": 62.99 +} + +# GPT-4o +REFERENCES = { + "NER": 79.11, + "REL": 63.32, + "LS": 59.25, + "SU": 33.04 + +} + + +def boxplot_prompts_per_task(dataframe, tasks=None): + if tasks is None: + tasks = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"] + + # Lista delle colonne da aggiornare + cols_to_update = ["REL Best Prompt Id", "NER Best Prompt Id", "SU Best Prompt Id", "LS Best Prompt Id"] + # Applichiamo la trasformazione + for col in cols_to_update: + dataframe[col] = dataframe[col].replace({1: 7, 2: 8}) + + fig = go.Figure() + + # Liste per creare una sola voce in legenda per Average e Best + avg_x, avg_y = [], [] + best_x, best_y, best_text = [], [], [] + + for task in tasks: + avg_col = f"{task} Prompt Average" + best_col = f"{task} Best Prompt" + best_id_col = f"{task} Best Prompt Id" + + if all(col in dataframe.columns for col in [avg_col, best_col, best_id_col]): + avg_value = dataframe[avg_col].mean() + avg_x.append(task) + avg_y.append(avg_value) + + best_value = dataframe[best_col].mean() + best_x.append(task) + best_y.append(best_value) + best_id = dataframe[best_id_col].mode()[0] # Most frequent best prompt id + best_text.append(f"P:{best_id}") + + # Barre Average Accuracy (azzurro) + fig.add_trace(go.Bar( + x=avg_x, + y=avg_y, + name="Avg. Accuracy", + marker_color="#1f77b4", + #hovertemplate="%{y:.2f}%" + #hovertemplate="" + task + "
Accuracy: %{y:.2f}%", + )) + + # Barre Best Prompt (rosso) + fig.add_trace(go.Bar( + x=best_x, + y=best_y, + name="Best Prompt", + marker_color="#d62728", + #hovertemplate="%{y:.2f}%" + #hovertemplate = "" + task + "
Accuracy: %{y:.2f}%", + )) + + # Testo sopra barre Best Prompt con ID + for x, y, text in zip(best_x, best_y, best_text): + fig.add_annotation( + x=x, + y=y + 3, # leggermente sopra la barra + text=text, + showarrow=False, + font=dict(size=12, color="black") + ) + + fig.update_layout( + title= "Prompt Accuracy: Avg vs Best", + xaxis_title="Task", + yaxis_title="Combined Performance", + barmode='group', + template="plotly_white", + font=dict(family="Arial", size=10), + yaxis=dict(range=[0, 100], fixedrange=True) + ) + + # caption come annotazione separata + fig.add_annotation( + text="There is no single prompt that performs best across all tasks.
" + "Different prompts achieve the highest accuracy on different tasks.", + xref="paper", yref="paper", + x=0.5, y=-0.3, + showarrow=False, + font=dict(size=11, color="gray"), + align="center", + xanchor="center" + ) + + return fig + + +def line_chart(dataframe): + + # Normalizza le dimensioni per avere marker non troppo piccoli né enormi + def scale_sizes(values, min_size=8, max_size=30): + vmin, vmax = min(values), max(values) + return [ + min_size + (val - vmin) / (vmax - vmin) * (max_size - min_size) if vmax > vmin else (min_size + max_size) / 2 + for val in values + ] + + # dati in base a IS_FS + df_true = dataframe[dataframe['IS_FS'] == True] + df_false = dataframe[dataframe['IS_FS'] == False] + + # Estrai valori x, y e labels + x_true = df_true['#Params (B)'].tolist() + y_true = df_true['Avg. Comb. Perf. ⬆️'].tolist() + labels_true = [re.search(r'>([^<]+)<', m).group(1) for m in df_true['Model'].tolist()] + + x_false = df_false['#Params (B)'].tolist() + y_false = df_false['Avg. Comb. Perf. ⬆️'].tolist() + labels_false = [re.search(r'>([^<]+)<', m).group(1) for m in df_false['Model'].tolist()] + + fig = go.Figure() + + # Punti IS_FS=True + fig.add_trace(go.Scatter( + x=x_true, + y=y_true, + mode='markers', + name='5-Shot', + marker=dict( + color='blue', + size=scale_sizes(x_true) + ), + hovertemplate='%{customdata}
#Params: %{x}
Performance: %{y}', + customdata=labels_true + )) + + # Punti IS_FS=False + fig.add_trace(go.Scatter( + x=x_false, + y=y_false, + mode='markers', + name='0-Shot', + marker=dict( + color='red', + size=scale_sizes(x_false) + ), + hovertemplate='%{customdata}
#Params: %{x}
Performance: %{y}', + customdata=labels_false + )) + + # Trova il massimo tra tutti i modelli + all_y = y_true + y_false + all_x = x_true + x_false + all_labels = labels_true + labels_false + max_idx = all_y.index(max(all_y)) + max_x = all_x[max_idx] + max_y = all_y[max_idx] + max_label = all_labels[max_idx] + + # Aggiungi annotazione visibile per il modello migliore + fig.add_annotation( + x=max_x, + y=max_y, + #text=f"Top: {max_label} ({max_y:.1f}%)", + text=f"{max_label}", + showarrow=True, + arrowhead=2, + arrowsize=1, + arrowwidth=2, + arrowcolor="black", + font=dict(size=11, color="black"), + xshift=10, + yshift=10, + ax = -30, ay = -20, # sposta la label a sinistra e sopra il punto + xanchor = "right" # allinea la label a destra rispetto al punto + ) + + fig.update_layout( + title="Avg. Combined Performance vs #Params", + xaxis_title="#Params (B)", + yaxis_title="Avg. Combined Performance", + template="plotly_white", + hovermode="closest", + font=dict(family="Arial", size=10), + dragmode=False, + xaxis=dict( + tickvals=[0, 25, 50, 75, 100, 125], + ticktext=["0", "25", "50", "75", "100"] + ), + yaxis=dict( + tickvals=[0, 20, 40, 60, 80, 100], # 👈 tick fissi + range=[0, 100] # 👈 range bloccato + ) + ) + + # Caption + fig.add_annotation( + text="Accuracy generally rises with #Params, but smaller models
" + "with 5-shot can outperform larger zero-shot models.", + xref="paper", yref="paper", + x=0.5, y=-0.3, # 👈 centrata + showarrow=False, + font=dict(size=11, color="gray"), + align="center", + xanchor="center" # 👈 ancora centrata rispetto al testo + ) + + fig.update_xaxes(fixedrange=True, rangeslider_visible=False) + fig.update_yaxes(fixedrange=True) + + return fig + + +# Define task metadata (icons, names, descriptions) +TASK_METADATA_MULTIPLECHOICE = { + #"TE": {"icon": "📊", "name": "Textual Entailment", "tooltip": ""}, + #"SA": {"icon": "😃", "name": "Sentiment Analysis", "tooltip": ""}, + #"HS": {"icon": "⚠️", "name": "Hate Speech", "tooltip": ""}, + #"AT": {"icon": "🏥", "name": "Admission Test", "tooltip": ""}, + #"WIC": {"icon": "🔤", "name": "Word in Context", "tooltip": ""}, + #"FAQ": {"icon": "❓", "name": "Frequently Asked Questions", "tooltip": ""} +} + +# Define task metadata (icons, names, descriptions) +TASK_METADATA_GENERATIVE = { + #"LS": {"icon": "🔄", "name": "Lexical Substitution", "tooltip": ""}, + #"SU": {"icon": "📝", "name": "Summarization", "tooltip": ""}, + "NER": {"icon": "🏷️", "name": "Named Entity Recognition", "tooltip": ""}, + "REL": {"icon": "🔗", "name": "Relation Extraction", "tooltip": ""}, +} + +def restart_space(): + """Restart the Hugging Face space.""" + API.restart_space(repo_id=REPO_ID) + + +def init_leaderboard(dataframe, default_selection=None, hidden_columns=None): + """ + Initialize and return the leaderboard when it is first loaded or when 'benchmark' is selected. + The table is sorted based on the "Avg. Combined Performance" field. + """ + if dataframe is None or dataframe.empty: + raise ValueError("Leaderboard DataFrame is empty or None.") + + #print("????????????????????????????????", mean_of_max_per_field(dataframe)) + + sorted_dataframe = dataframe.sort_values(by="Avg. Comb. Perf. ⬆️", ascending=False) + + sorted_dataframe = sorted_dataframe.reset_index(drop=True) + sorted_dataframe["Rank"] = sorted_dataframe.index + 1 + + # Flag per sapere se la medaglia è già stata assegnata per categoria e tipo + large_medal_fs_assigned = False + medium_medal_fs_assigned = False + small_medal_fs_assigned = False + + large_medal_0shot_assigned = False + medium_medal_0shot_assigned = False + small_medal_0shot_assigned = False + + # Lista temporanea per salvare i nuovi valori della colonna Model + new_model_column = [] + + for _, row in sorted_dataframe.iterrows(): + if row['IS_FS']: # 10-Few-Shot + if row["Size"] == "🔵🔵🔵" and not large_medal_fs_assigned: + new_model_column.append(f"{row['Model']} 🔵🔵🔵🏆") + large_medal_fs_assigned = True + elif row["Size"] == "🔵🔵" and not medium_medal_fs_assigned: + new_model_column.append(f"{row['Model']} 🔵🔵🏆") + medium_medal_fs_assigned = True + elif row["Size"] == "🔵" and not small_medal_fs_assigned: + new_model_column.append(f"{row['Model']} 🔵🏆") + small_medal_fs_assigned = True + else: + new_model_column.append(row["Model"]) + else: # 0-Shot + if row["Size"] == "🔵🔵🔵" and not large_medal_0shot_assigned: + new_model_column.append(f"{row['Model']} 🔵🔵🔵🎖️") + large_medal_0shot_assigned = True + elif row["Size"] == "🔵🔵" and not medium_medal_0shot_assigned: + new_model_column.append(f"{row['Model']} 🔵🔵🎖️") + medium_medal_0shot_assigned = True + elif row["Size"] == "🔵" and not small_medal_0shot_assigned: + new_model_column.append(f"{row['Model']} 🔵🎖️") + small_medal_0shot_assigned = True + else: + new_model_column.append(row["Model"]) + + # Lista delle colonne da aggiornare + #cols_to_update = ["REL Best Prompt Id", "NER Best Prompt Id", "SU Best Prompt Id", "LS Best Prompt Id"] + # Applichiamo la trasformazione + #for col in cols_to_update: + # dataframe[col] = dataframe[col].replace({1: 7, 2: 8}) + + # Aggiorna la colonna Model + sorted_dataframe["Model"] = new_model_column + + field_list = fields(AutoEvalColumn) + + return Leaderboard( + value=sorted_dataframe, + datatype=[c.type for c in field_list], + #select_columns=SelectColumns( + # default_selection=default_selection or [c.name for c in field_list if c.displayed_by_default], + # cant_deselect=[c.name for c in field_list if c.never_hidden], + # label="Select Columns to Display:", + #), + search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name], + hide_columns=hidden_columns or [c.name for c in field_list if c.hidden], + filter_columns=[ + ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Shot Learning (FS)"), + #ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Few-Shot Learning (FS)", + # default=[["0️⃣", "0️⃣"]]), + ColumnFilter(AutoEvalColumn.LANG.name, type="checkboxgroup", label="Languges "), + + ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max = 100, default = [0,100], label="Select the number of parameters (B)"), + ], + #filter_columns=[ + # ColumnFilter("IS_FS", type="checkbox", default=False, label="5-Few-Shot") + # #ColumnFilter("FS", type="dropdown", label="5-Few-Shot") + #], + bool_checkboxgroup_label="Evaluation Mode", + interactive=False, + ) + +def update_task_leaderboard(dataframe, default_selection=None, hidden_columns=None): + """ + Update and return the leaderboard when a specific task is selected. + The table is sorted based on the "Combined Performance" field. + """ + if dataframe is None or dataframe.empty: + raise ValueError("Leaderboard DataFrame is empty or None.") + + sorted_dataframe = dataframe.sort_values(by="Combined Performance", ascending=False) + + # aggiungo la colonna rank in base alla posizione + sorted_dataframe = sorted_dataframe.reset_index(drop=True) + sorted_dataframe["Rank"] = sorted_dataframe.index + 1 + + # Flag per sapere se la medaglia è già stata assegnata per categoria e tipo + large_medal_fs_assigned = False + medium_medal_fs_assigned = False + small_medal_fs_assigned = False + + large_medal_0shot_assigned = False + medium_medal_0shot_assigned = False + small_medal_0shot_assigned = False + + # Lista temporanea per salvare i nuovi valori della colonna Model + new_model_column = [] + + for _, row in sorted_dataframe.iterrows(): + if row['IS_FS']: # 5-Few-Shot + if row["Size"] == "🔵🔵🔵" and not large_medal_fs_assigned: + new_model_column.append(f"{row['Model']} 🔵🔵🔵🏆") + large_medal_fs_assigned = True + elif row["Size"] == "🔵🔵" and not medium_medal_fs_assigned: + new_model_column.append(f"{row['Model']} 🔵🔵🏆") + medium_medal_fs_assigned = True + elif row["Size"] == "🔵" and not small_medal_fs_assigned: + new_model_column.append(f"{row['Model']} 🔵🏆") + small_medal_fs_assigned = True + else: + new_model_column.append(row["Model"]) + else: # 0-Shot + if row["Size"] == "🔵🔵🔵" and not large_medal_0shot_assigned: + new_model_column.append(f"{row['Model']} 🔵🔵🔵🎖️") + large_medal_0shot_assigned = True + elif row["Size"] == "🔵🔵" and not medium_medal_0shot_assigned: + new_model_column.append(f"{row['Model']} 🔵🔵🎖️") + medium_medal_0shot_assigned = True + elif row["Size"] == "🔵" and not small_medal_0shot_assigned: + new_model_column.append(f"{row['Model']} 🔵🎖️") + small_medal_0shot_assigned = True + else: + new_model_column.append(row["Model"]) + + # Aggiorna la colonna Model + sorted_dataframe["Model"] = new_model_column + + pd.set_option('display.max_colwidth', None) + #print("========================", dataframe['Model']) + + #print(sorted_dataframe['Combined Performance']) + + field_list = fields(AutoEvalColumn) + + return Leaderboard( + value=sorted_dataframe, + #datatype=[c.type for c in field_list], + datatype=[c.type for c in field_list] + [int], + #select_columns=SelectColumns( + # default_selection=default_selection or [c.name for c in field_list if c.displayed_by_default], + # cant_deselect=[c.name for c in field_list if c.never_hidden], + # label="Select Columns to Display:", + #), + search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name], + hide_columns=hidden_columns or [c.name for c in field_list if c.hidden], + filter_columns=[ + ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Shot Learning (FS)"), + ColumnFilter(AutoEvalColumn.LANG.name, type="checkboxgroup", label="Languges "), + + ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max=100, default=[0, 100], + label="Select the number of parameters (B)"), + ], + bool_checkboxgroup_label="Evaluation Mode", + interactive=False + ) + +''' +# Helper function for leaderboard initialization +def init_leaderboard(dataframe, default_selection=None, hidden_columns=None): + """Initialize and return a leaderboard.""" + if dataframe is None or dataframe.empty: + raise ValueError("Leaderboard DataFrame is empty or None.") + + return Leaderboard( + value=dataframe, + datatype=[c.type for c in fields(AutoEvalColumn)], + select_columns=SelectColumns( + default_selection=default_selection or [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default], + cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden], + label="Select Columns to Display:", + ), + search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name], + hide_columns=hidden_columns or [c.name for c in fields(AutoEvalColumn) if c.hidden], + filter_columns=[ + ColumnFilter(AutoEvalColumn.fewshot_type.name, type="checkboxgroup", label="N-Few-Shot Learning (FS)"), + ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max=150, label="Select the number of parameters (B)"), + ], + bool_checkboxgroup_label="Hide models", + interactive=False, + ) +''' + +def download_snapshot(repo, local_dir): + """Try to download a snapshot from Hugging Face Hub.""" + try: + print(f"Downloading from {repo} to {local_dir}...") + snapshot_download(repo_id=repo, local_dir=local_dir, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN) + except Exception as e: + print(f"Error downloading {repo}: {e}") + restart_space() + + +# Initialize the app by downloading snapshots +download_snapshot(QUEUE_REPO, EVAL_REQUESTS_PATH) +download_snapshot(RESULTS_REPO, EVAL_RESULTS_PATH) + +# Load leaderboard data +LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS) +finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS) +#print(LEADERBOARD_DF.columns.tolist()) + +theoretical_max_combined_perf = mean_of_max_per_field(LEADERBOARD_DF) + +# Prepare the main interface +demo = gr.Blocks(css=custom_css) +with demo: + #gr.HTML(TITLE) + gr.HTML( + """ +
+

+ EVALITA-LLM Leaderboard +

+ + + + + + + Open Italian LLM Leaderboard + +
+ """ + ) + gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") + + # ⬇️ QUI aggiungiamo i grafici subito sotto la barra del titolo e sopra le tabs + with gr.Row(): + gr.Plot(value=line_chart(LEADERBOARD_DF), elem_id="line-chart") + gr.Plot(value=boxplot_per_task(LEADERBOARD_DF, BASELINES, REFERENCES), elem_id="boxplot-task") + #gr.Plot(value=boxplot_prompts_per_task(LEADERBOARD_DF), elem_id="boxplot-prompt-task") + + with gr.Tabs(elem_classes="tab-buttons") as tabs: + + # Main leaderboard tab + with gr.TabItem("🏅 Benchmark"): + + leaderboard = init_leaderboard( + LEADERBOARD_DF, + default_selection=['Rank', 'Size', 'LANG', 'FS', 'Model', "Avg. Comb. Perf. ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"], + hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['Rank', 'Size', 'LANG', 'FS', 'Model', "Avg. Comb. Perf. ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]] + ) + + # gr.HTML( + # f""" + #
+ # Theoretical performance of a model that scores the highest on every individual task: {theoretical_max_combined_perf:.2f} + #
+ # $ """ + # ) + + ''' + with gr.TabItem("📈 Charts"): + #gr.Plot(value=line_chart(LEADERBOARD_DF), label="Andamento di esempio") + #gr.Plot(value=line_chart_interactive_test(), label="Andamento interattivo") + gr.Plot(value=line_chart(LEADERBOARD_DF)) + gr.Plot(value=boxplot_per_task(LEADERBOARD_DF, BASELINES)) + gr.Plot(value=boxplot_prompts_per_task(LEADERBOARD_DF)) + gr.Plot(value=barplot_mean_few_minus_zero_shot(LEADERBOARD_DF)) + ''' + + # About tab + with gr.TabItem("📝 About"): + gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") + + # About tab + with gr.TabItem("║", interactive=False): + gr.Markdown("", elem_classes="markdown-text") + + + # Task-specific leaderboards + for task, metadata in TASK_METADATA_MULTIPLECHOICE.items(): + + with gr.TabItem(f"{metadata['icon']}{task}"): + + task_description = TASK_DESCRIPTIONS.get(task, "Description not available.") + gr.Markdown(task_description, elem_classes="markdown-text") + + leaderboard = update_task_leaderboard( + LEADERBOARD_DF.rename(columns={f"{task} Prompt Average": "Prompt Average", f"{task} Prompt Std": "Prompt Std", f"{task} Best Prompt": "Best Prompt", f"{task} Best Prompt Id": "Best Prompt Id", task: "Combined Performance"}), + default_selection=['Rank', 'Size','LANG', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id'], + hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['Rank', 'Size','LANG', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id']] + ) + + # About tab + with gr.TabItem("│", interactive=False): + gr.Markdown("", elem_classes="markdown-text") + + # Task-specific leaderboards + for task, metadata in TASK_METADATA_GENERATIVE.items(): + with gr.TabItem(f"{metadata['icon']}{task}"): + task_description = TASK_DESCRIPTIONS.get(task, "Description not available.") + gr.Markdown(task_description, elem_classes="markdown-text") + + leaderboard = update_task_leaderboard( + LEADERBOARD_DF.rename(columns={f"{task} Prompt Average": "Prompt Average", + f"{task} Prompt Std": "Prompt Std", + f"{task} Best Prompt": "Best Prompt", + f"{task} Best Prompt Id": "Best Prompt Id", + task: "Combined Performance"}), + default_selection=['Rank', 'Size', 'LANG', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt', + 'Best Prompt Id'], + hidden_columns=[col for col in LEADERBOARD_DF.columns if + col not in ['Rank', 'Size','LANG', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', + 'Best Prompt', 'Best Prompt Id']] + ) + + # Citation section + with gr.Accordion("📙 Citation", open=False): + gr.Textbox(value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, lines=20, elem_id="citation-button", show_copy_button=True) + + with gr.Accordion("📙 Credits", open=False): + gr.Markdown( + """ + **This project has benefited from the following support:** + + - 🧠 **Codebase**: Based on and extended from the Open Italian LLM Leaderboard, developed by **Alessandro Ercolani** and **Samuele Colombo**. We warmly thank them for their invaluable support and guidance in implementing this leaderboard. + + - 💶 **Funding**: Partially supported by the PNRR project **FAIR - Future AI Research (PE00000013)**, under the NRRP MUR program funded by **NextGenerationEU**. + + - 🖥️ **Computation**: We gratefully acknowledge **CINECA** for granting access to the **LEONARDO** supercomputer. + """ + ) + +# Background job to restart space +scheduler = BackgroundScheduler() +scheduler.add_job(restart_space, "interval", seconds=1800) +scheduler.start() + +# Launch the app with concurrent queueing +demo.queue(default_concurrency_limit=40).launch(debug=True, # Enable Gradio debug mode + show_error=True) \ No newline at end of file diff --git a/e3c_llm_requests/.gitattributes b/e3c_llm_requests/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..1ef325f1b111266a6b26e0196871bd78baa8c2f3 --- /dev/null +++ b/e3c_llm_requests/.gitattributes @@ -0,0 +1,59 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.lz4 filter=lfs diff=lfs merge=lfs -text +*.mds filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +# Audio files - uncompressed +*.pcm filter=lfs diff=lfs merge=lfs -text +*.sam filter=lfs diff=lfs merge=lfs -text +*.raw filter=lfs diff=lfs merge=lfs -text +# Audio files - compressed +*.aac filter=lfs diff=lfs merge=lfs -text +*.flac filter=lfs diff=lfs merge=lfs -text +*.mp3 filter=lfs diff=lfs merge=lfs -text +*.ogg filter=lfs diff=lfs merge=lfs -text +*.wav filter=lfs diff=lfs merge=lfs -text +# Image files - uncompressed +*.bmp filter=lfs diff=lfs merge=lfs -text +*.gif filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text +*.tiff filter=lfs diff=lfs merge=lfs -text +# Image files - compressed +*.jpg filter=lfs diff=lfs merge=lfs -text +*.jpeg filter=lfs diff=lfs merge=lfs -text +*.webp filter=lfs diff=lfs merge=lfs -text +# Video files - compressed +*.mp4 filter=lfs diff=lfs merge=lfs -text +*.webm filter=lfs diff=lfs merge=lfs -text diff --git a/e3c_llm_requests/Henrychur/MMed-Llama-3-8B.json b/e3c_llm_requests/Henrychur/MMed-Llama-3-8B.json new file mode 100644 index 0000000000000000000000000000000000000000..86b0019fb62092325cdc79c7eb8218aed3bad09f --- /dev/null +++ b/e3c_llm_requests/Henrychur/MMed-Llama-3-8B.json @@ -0,0 +1,8 @@ +{ + "model": "Henrychur/MMed-Llama-3-8B", + "base_model": "LlamaForCausalLM", + "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943", + "submitted_time": "2024-05-22 09:17:24+00:00", + "num_params_billion": null, + "language": "en_zh_ja_fr_ru_es" +} \ No newline at end of file diff --git a/e3c_llm_requests/HiTZ/Medical-mT5-large.json b/e3c_llm_requests/HiTZ/Medical-mT5-large.json new file mode 100644 index 0000000000000000000000000000000000000000..42da6bb5dbb7d478648d28988f549c4c3e885a7c --- /dev/null +++ b/e3c_llm_requests/HiTZ/Medical-mT5-large.json @@ -0,0 +1,8 @@ +{ + "model": "HiTZ/Medical-mT5-large", + "base_model": "MT5ForConditionalGeneration", + "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47", + "submitted_time": "2023-10-31 15:15:15+00:00", + "num_params_billion": null, + "language": "en_es_fr_it" +} \ No newline at end of file diff --git a/e3c_llm_requests/Qwen/Qwen2.5-14B-Instruct-1M.json b/e3c_llm_requests/Qwen/Qwen2.5-14B-Instruct-1M.json new file mode 100644 index 0000000000000000000000000000000000000000..e4460926779e971e3317af33665cf9278980c10d --- /dev/null +++ b/e3c_llm_requests/Qwen/Qwen2.5-14B-Instruct-1M.json @@ -0,0 +1,8 @@ +{ + "model": "Qwen/Qwen2.5-14B-Instruct-1M", + "base_model": "Qwen2ForCausalLM", + "revision": "620fad32de7bdd2293b3d99b39eba2fe63e97438", + "submitted_time": "2025-01-23 13:23:24+00:00", + "num_params_billion": 14.770033664, + "language": "en" +} \ No newline at end of file diff --git a/e3c_llm_requests/Qwen/Qwen2.5-32B-Instruct.json b/e3c_llm_requests/Qwen/Qwen2.5-32B-Instruct.json new file mode 100644 index 0000000000000000000000000000000000000000..3bfb3ff4c28797c9aad9070719f798119c2784e3 --- /dev/null +++ b/e3c_llm_requests/Qwen/Qwen2.5-32B-Instruct.json @@ -0,0 +1,8 @@ +{ + "model": "Qwen/Qwen2.5-32B-Instruct", + "base_model": "Qwen2ForCausalLM", + "revision": "5ede1c97bbab6ce5cda5812749b4c0bdf79b18dd", + "submitted_time": "2024-09-17 04:17:55+00:00", + "num_params_billion": 32.763876352, + "language": "en" +} \ No newline at end of file diff --git a/e3c_llm_requests/Qwen/Qwen3-30B-A3B-Instruct-2507.json b/e3c_llm_requests/Qwen/Qwen3-30B-A3B-Instruct-2507.json new file mode 100644 index 0000000000000000000000000000000000000000..8c19206d89e4c8cc448322bcf562cdeea4f686ba --- /dev/null +++ b/e3c_llm_requests/Qwen/Qwen3-30B-A3B-Instruct-2507.json @@ -0,0 +1,8 @@ +{ + "model": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "base_model": "Qwen3MoeForCausalLM", + "revision": "61082d4deaa4785f64943b443cbc2b5de7524fad", + "submitted_time": "2025-07-28 07:31:27+00:00", + "num_params_billion": 30.532122624, + "language": "" +} \ No newline at end of file diff --git a/e3c_llm_requests/deepseek-ai/.ipynb_checkpoints/DeepSeek-R1-Distill-Qwen-32B-checkpoint.json b/e3c_llm_requests/deepseek-ai/.ipynb_checkpoints/DeepSeek-R1-Distill-Qwen-32B-checkpoint.json new file mode 100644 index 0000000000000000000000000000000000000000..3a574a6dcb81e49041fa0f592ee86930e1b43847 --- /dev/null +++ b/e3c_llm_requests/deepseek-ai/.ipynb_checkpoints/DeepSeek-R1-Distill-Qwen-32B-checkpoint.json @@ -0,0 +1,8 @@ +{ + "model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "base_model": "Qwen2ForCausalLM", + "revision": "711ad2ea6aa40cfca18895e8aca02ab92df1a746", + "submitted_time": "2025-01-20 09:19:00+00:00", + "num_params_billion": 32.763876352, + "language": "" +} \ No newline at end of file diff --git a/e3c_llm_requests/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B.json b/e3c_llm_requests/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B.json new file mode 100644 index 0000000000000000000000000000000000000000..3a574a6dcb81e49041fa0f592ee86930e1b43847 --- /dev/null +++ b/e3c_llm_requests/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B.json @@ -0,0 +1,8 @@ +{ + "model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "base_model": "Qwen2ForCausalLM", + "revision": "711ad2ea6aa40cfca18895e8aca02ab92df1a746", + "submitted_time": "2025-01-20 09:19:00+00:00", + "num_params_billion": 32.763876352, + "language": "" +} \ No newline at end of file diff --git a/e3c_llm_requests/epfl-llm/meditron-7b.json b/e3c_llm_requests/epfl-llm/meditron-7b.json new file mode 100644 index 0000000000000000000000000000000000000000..773ee8fbfdfa8623fdc11b71a79ef5122ea682b7 --- /dev/null +++ b/e3c_llm_requests/epfl-llm/meditron-7b.json @@ -0,0 +1,8 @@ +{ + "model": "epfl-llm/meditron-7b", + "base_model": "LlamaForCausalLM", + "revision": "d7d0a5ed929384a6b059ac74198cf1d71f44ba76", + "submitted_time": "2023-11-08 16:03:23+00:00", + "num_params_billion": 6.73855488, + "language": "en" +} \ No newline at end of file diff --git a/e3c_llm_requests/google/gemma-2-9b-it.json b/e3c_llm_requests/google/gemma-2-9b-it.json new file mode 100644 index 0000000000000000000000000000000000000000..bc6b860d3bd567948054e01f7630043dd7220af4 --- /dev/null +++ b/e3c_llm_requests/google/gemma-2-9b-it.json @@ -0,0 +1,8 @@ +{ + "model": "google/gemma-2-9b-it", + "base_model": "Gemma2ForCausalLM", + "revision": "11c9b309abf73637e4b6f9a3fa1e92e615547819", + "submitted_time": "2024-06-24 08:05:41+00:00", + "num_params_billion": 9.241705984, + "language": "" +} \ No newline at end of file diff --git a/e3c_llm_requests/google/gemma-3-27b-it.json b/e3c_llm_requests/google/gemma-3-27b-it.json new file mode 100644 index 0000000000000000000000000000000000000000..0cbbf8fd214db1aa63941c6685829be3c1ef47a6 --- /dev/null +++ b/e3c_llm_requests/google/gemma-3-27b-it.json @@ -0,0 +1,8 @@ +{ + "model": "google/gemma-3-27b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "005ad3404e59d6023443cb575daa05336842228a", + "submitted_time": "2025-03-01 19:10:19+00:00", + "num_params_billion": 27.43240664, + "language": "" +} \ No newline at end of file diff --git a/e3c_llm_requests/google/medgemma-27b-text-it.json b/e3c_llm_requests/google/medgemma-27b-text-it.json new file mode 100644 index 0000000000000000000000000000000000000000..0eadf57aa2ee6563dba193b1a1fd16ebc0362a65 --- /dev/null +++ b/e3c_llm_requests/google/medgemma-27b-text-it.json @@ -0,0 +1,8 @@ +{ + "model": "google/medgemma-27b-text-it", + "base_model": "Gemma3ForCausalLM", + "revision": "6b08c481126ff65a9b8fa5ab4d691b152b8edb5d", + "submitted_time": "2025-05-19 20:53:04+00:00", + "num_params_billion": 27.00900224, + "language": "" +} \ No newline at end of file diff --git a/e3c_llm_requests/google/medgemma-4b-it.json b/e3c_llm_requests/google/medgemma-4b-it.json new file mode 100644 index 0000000000000000000000000000000000000000..7c6a467cac7dee24972df120b27cdd729a1e75fd --- /dev/null +++ b/e3c_llm_requests/google/medgemma-4b-it.json @@ -0,0 +1,8 @@ +{ + "model": "google/medgemma-4b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "efe6cc02361759b6bd501c654ddb7c9d25ec509d", + "submitted_time": "2025-05-19 20:52:44+00:00", + "num_params_billion": 4.300079472, + "language": "" +} \ No newline at end of file diff --git a/e3c_llm_requests/meta-llama/.ipynb_checkpoints/Llama-3.2-1B-Instruct-checkpoint.json b/e3c_llm_requests/meta-llama/.ipynb_checkpoints/Llama-3.2-1B-Instruct-checkpoint.json new file mode 100644 index 0000000000000000000000000000000000000000..dd0dab56bad582a995b770ac81c21b4ad4954553 --- /dev/null +++ b/e3c_llm_requests/meta-llama/.ipynb_checkpoints/Llama-3.2-1B-Instruct-checkpoint.json @@ -0,0 +1,8 @@ +{ + "model": "meta-llama/Llama-3.2-1B-Instruct", + "base_model": "LlamaForCausalLM", + "revision": "9213176726f574b556790deb65791e0c5aa438b6", + "submitted_time": "2024-09-18 15:12:47+00:00", + "num_params_billion": 1.2358144, + "language": "en_de_fr_it_pt_hi_es_th" +} \ No newline at end of file diff --git a/e3c_llm_requests/meta-llama/Llama-3.2-1B-Instruct.json b/e3c_llm_requests/meta-llama/Llama-3.2-1B-Instruct.json new file mode 100644 index 0000000000000000000000000000000000000000..dd0dab56bad582a995b770ac81c21b4ad4954553 --- /dev/null +++ b/e3c_llm_requests/meta-llama/Llama-3.2-1B-Instruct.json @@ -0,0 +1,8 @@ +{ + "model": "meta-llama/Llama-3.2-1B-Instruct", + "base_model": "LlamaForCausalLM", + "revision": "9213176726f574b556790deb65791e0c5aa438b6", + "submitted_time": "2024-09-18 15:12:47+00:00", + "num_params_billion": 1.2358144, + "language": "en_de_fr_it_pt_hi_es_th" +} \ No newline at end of file diff --git a/e3c_llm_requests/microsoft/MediPhi-Clinical.json b/e3c_llm_requests/microsoft/MediPhi-Clinical.json new file mode 100644 index 0000000000000000000000000000000000000000..24031b2427e47fc919f0fda9c2570e2c55afafa6 --- /dev/null +++ b/e3c_llm_requests/microsoft/MediPhi-Clinical.json @@ -0,0 +1,8 @@ +{ + "model": "microsoft/MediPhi-Clinical", + "base_model": "Phi3ForCausalLM", + "revision": "0906e64d321a9c4b058137b34fb3ed6e257e05a0", + "submitted_time": "2025-05-29 20:40:05+00:00", + "num_params_billion": 3.821079552, + "language": "en" +} \ No newline at end of file diff --git a/e3c_llm_requests/microsoft/MediPhi-Instruct.json b/e3c_llm_requests/microsoft/MediPhi-Instruct.json new file mode 100644 index 0000000000000000000000000000000000000000..fd36894ce7cc6bbb938d3639a927ebb3c277254d --- /dev/null +++ b/e3c_llm_requests/microsoft/MediPhi-Instruct.json @@ -0,0 +1,8 @@ +{ + "model": "microsoft/MediPhi-Instruct", + "base_model": "Phi3ForCausalLM", + "revision": "a94ac478e7c246103d55665a0804684042f3b973", + "submitted_time": "2025-07-11 19:28:15+00:00", + "num_params_billion": 3.821079552, + "language": "en" +} \ No newline at end of file diff --git a/e3c_llm_requests/mistralai/Mistral-7B-Instruct-v0.2.json b/e3c_llm_requests/mistralai/Mistral-7B-Instruct-v0.2.json new file mode 100644 index 0000000000000000000000000000000000000000..b36579fb429f3b744a46c6a84fed781411b85cc7 --- /dev/null +++ b/e3c_llm_requests/mistralai/Mistral-7B-Instruct-v0.2.json @@ -0,0 +1,8 @@ +{ + "model": "mistralai/Mistral-7B-Instruct-v0.2", + "base_model": "MistralForCausalLM", + "revision": "63a8b081895390a26e140280378bc85ec8bce07a", + "submitted_time": "2023-12-11 13:18:44+00:00", + "num_params_billion": 7.241732096, + "language": "" +} \ No newline at end of file diff --git a/e3c_llm_requests/mistralai/Mistral-Nemo-Instruct-2407.json b/e3c_llm_requests/mistralai/Mistral-Nemo-Instruct-2407.json new file mode 100644 index 0000000000000000000000000000000000000000..c46a7da06e59d841593280fb14969e4182c18d95 --- /dev/null +++ b/e3c_llm_requests/mistralai/Mistral-Nemo-Instruct-2407.json @@ -0,0 +1,8 @@ +{ + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "base_model": "MistralForCausalLM", + "revision": "04d8a90549d23fc6bd7f642064003592df51e9b3", + "submitted_time": "2024-07-17 17:26:49+00:00", + "num_params_billion": 12.2477824, + "language": "en_fr_de_es_it_pt_ru_zh_ja" +} \ No newline at end of file diff --git a/e3c_llm_requests/tiiuae/Falcon3-10B-Instruct.json b/e3c_llm_requests/tiiuae/Falcon3-10B-Instruct.json new file mode 100644 index 0000000000000000000000000000000000000000..6fb21c1511b3e96748b3e779985cbcbbfca44186 --- /dev/null +++ b/e3c_llm_requests/tiiuae/Falcon3-10B-Instruct.json @@ -0,0 +1,8 @@ +{ + "model": "tiiuae/Falcon3-10B-Instruct", + "base_model": "LlamaForCausalLM", + "revision": "8799bc6aec0152757221dc6b272d824642db6202", + "submitted_time": "2024-12-14 05:17:25+00:00", + "num_params_billion": 10.30565376, + "language": "" +} \ No newline at end of file diff --git a/e3c_llm_requests/unsloth/phi-4.json b/e3c_llm_requests/unsloth/phi-4.json new file mode 100644 index 0000000000000000000000000000000000000000..1aacd86f19c5a378deb37004884fd52f579b6daf --- /dev/null +++ b/e3c_llm_requests/unsloth/phi-4.json @@ -0,0 +1,8 @@ +{ + "model": "unsloth/phi-4", + "base_model": "LlamaForCausalLM", + "revision": "c6220bde10fff762dbd72c3331894aa4cade249d", + "submitted_time": "2025-01-08 21:56:16+00:00", + "num_params_billion": 14.6595072, + "language": "en" +} \ No newline at end of file diff --git a/e3c_llm_results/.gitattributes b/e3c_llm_results/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..1ef325f1b111266a6b26e0196871bd78baa8c2f3 --- /dev/null +++ b/e3c_llm_results/.gitattributes @@ -0,0 +1,59 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.lz4 filter=lfs diff=lfs merge=lfs -text +*.mds filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +# Audio files - uncompressed +*.pcm filter=lfs diff=lfs merge=lfs -text +*.sam filter=lfs diff=lfs merge=lfs -text +*.raw filter=lfs diff=lfs merge=lfs -text +# Audio files - compressed +*.aac filter=lfs diff=lfs merge=lfs -text +*.flac filter=lfs diff=lfs merge=lfs -text +*.mp3 filter=lfs diff=lfs merge=lfs -text +*.ogg filter=lfs diff=lfs merge=lfs -text +*.wav filter=lfs diff=lfs merge=lfs -text +# Image files - uncompressed +*.bmp filter=lfs diff=lfs merge=lfs -text +*.gif filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text +*.tiff filter=lfs diff=lfs merge=lfs -text +# Image files - compressed +*.jpg filter=lfs diff=lfs merge=lfs -text +*.jpeg filter=lfs diff=lfs merge=lfs -text +*.webp filter=lfs diff=lfs merge=lfs -text +# Video files - compressed +*.mp4 filter=lfs diff=lfs merge=lfs -text +*.webm filter=lfs diff=lfs merge=lfs -text diff --git a/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_EN.json b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..68490feaca9d8e9cad19ecf26057ce3e9a1fe436 --- /dev/null +++ b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 20.954842, + "config": { + "model_name": "Henrychur/MMed-Llama-3-8B", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "EN", + "model": "Henrychur/MMed-Llama-3-8B", + "base_model": "LlamaForCausalLM", + "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943", + "submitted_time": "2024-05-22 09:17:24+00:00", + "num_params_billion": null, + "language": "en_zh_ja_fr_ru_es" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 6.29, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 10.41, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 10.83, + "stderr": 0.0 + } + ], + "average_accuracy": 9.176666666666668, + "best_prompt": 10.83, + "prompt_id": "p3", + "CPS": 10.650944 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 12.870000000000001, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 33.94, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 31.31, + "stderr": 0.0 + } + ], + "average_accuracy": 26.040000000000003, + "best_prompt": 33.94, + "prompt_id": "p2", + "CPS": 31.25874 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_GR.json b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..46934473a835ffb1b03ee0650e818e3bbf5a3592 --- /dev/null +++ b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 8.314364166666667, + "config": { + "model_name": "Henrychur/MMed-Llama-3-8B", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "GR", + "model": "Henrychur/MMed-Llama-3-8B", + "base_model": "LlamaForCausalLM", + "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943", + "submitted_time": "2024-05-22 09:17:24+00:00", + "num_params_billion": null, + "language": "en_zh_ja_fr_ru_es" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 6.2, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 5.92, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 6.2, + "stderr": 0.0 + } + ], + "average_accuracy": 6.1066666666666665, + "best_prompt": 6.2, + "prompt_id": "p1", + "CPS": 6.194213333333334 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 10.17, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 5.06, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 10.65, + "stderr": 0.0 + } + ], + "average_accuracy": 8.626666666666667, + "best_prompt": 10.65, + "prompt_id": "p3", + "CPS": 10.434515000000001 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_IT.json b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..2c45c8327f37738538c9a2d22c561e6cce5122ba --- /dev/null +++ b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 12.534040833333332, + "config": { + "model_name": "Henrychur/MMed-Llama-3-8B", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "IT", + "model": "Henrychur/MMed-Llama-3-8B", + "base_model": "LlamaForCausalLM", + "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943", + "submitted_time": "2024-05-22 09:17:24+00:00", + "num_params_billion": null, + "language": "en_zh_ja_fr_ru_es" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 4.35, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 4.29, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 3.84, + "stderr": 0.0 + } + ], + "average_accuracy": 4.16, + "best_prompt": 4.35, + "prompt_id": "p1", + "CPS": 4.341735 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 6.72, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 22.66, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 13.0, + "stderr": 0.0 + } + ], + "average_accuracy": 14.126666666666667, + "best_prompt": 22.66, + "prompt_id": "p2", + "CPS": 20.726346666666664 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_PL.json b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..37405e6ce398ae13f9a5ef49756ab32596976e92 --- /dev/null +++ b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 8.100043833333334, + "config": { + "model_name": "Henrychur/MMed-Llama-3-8B", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "PL", + "model": "Henrychur/MMed-Llama-3-8B", + "base_model": "LlamaForCausalLM", + "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943", + "submitted_time": "2024-05-22 09:17:24+00:00", + "num_params_billion": null, + "language": "en_zh_ja_fr_ru_es" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 3.7900000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 3.7800000000000002, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 3.7900000000000005, + "stderr": 0.0 + } + ], + "average_accuracy": 3.786666666666667, + "best_prompt": 3.7900000000000005, + "prompt_id": "p1", + "CPS": 3.7898736666666673 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 6.02, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 12.93, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 7.779999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 8.909999999999998, + "best_prompt": 12.93, + "prompt_id": "p2", + "CPS": 12.410214 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_SK.json b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..f0540dea613db509477cd214e73035df107270b2 --- /dev/null +++ b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 3.3197085, + "config": { + "model_name": "Henrychur/MMed-Llama-3-8B", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SK", + "model": "Henrychur/MMed-Llama-3-8B", + "base_model": "LlamaForCausalLM", + "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943", + "submitted_time": "2024-05-22 09:17:24+00:00", + "num_params_billion": null, + "language": "en_zh_ja_fr_ru_es" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 3.8699999999999997, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 3.8, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 3.8699999999999997, + "stderr": 0.0 + } + ], + "average_accuracy": 3.8466666666666662, + "best_prompt": 3.8699999999999997, + "prompt_id": "p1", + "CPS": 3.8690969999999996 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 1.21, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 2.8000000000000003, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 1.21, + "stderr": 0.0 + } + ], + "average_accuracy": 1.7400000000000002, + "best_prompt": 2.8000000000000003, + "prompt_id": "p2", + "CPS": 2.77032 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_SL.json b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..d4368cb1b572d1a4b5fbaa80fb0de48b42a9b647 --- /dev/null +++ b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_0_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 11.184996000000002, + "config": { + "model_name": "Henrychur/MMed-Llama-3-8B", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SL", + "model": "Henrychur/MMed-Llama-3-8B", + "base_model": "LlamaForCausalLM", + "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943", + "submitted_time": "2024-05-22 09:17:24+00:00", + "num_params_billion": null, + "language": "en_zh_ja_fr_ru_es" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 4.29, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 4.5600000000000005, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 4.29, + "stderr": 0.0 + } + ], + "average_accuracy": 4.38, + "best_prompt": 4.5600000000000005, + "prompt_id": "p2", + "CPS": 4.551792000000001 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 9.67, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 19.0, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 9.67, + "stderr": 0.0 + } + ], + "average_accuracy": 12.780000000000001, + "best_prompt": 19.0, + "prompt_id": "p2", + "CPS": 17.8182 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_EN.json b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..6a5dcb9b2dbb21591b581cce2196c81f871307b8 --- /dev/null +++ b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 21.475744333333335, + "config": { + "model_name": "Henrychur/MMed-Llama-3-8B", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "EN", + "model": "Henrychur/MMed-Llama-3-8B", + "base_model": "LlamaForCausalLM", + "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943", + "submitted_time": "2024-05-22 09:17:24+00:00", + "num_params_billion": null, + "language": "en_zh_ja_fr_ru_es" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 21.89, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 22.43, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 19.939999999999998, + "stderr": 0.0 + } + ], + "average_accuracy": 21.419999999999998, + "best_prompt": 22.43, + "prompt_id": "p2", + "CPS": 22.203457 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 11.89, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 16.68, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 21.85, + "stderr": 0.0 + } + ], + "average_accuracy": 16.80666666666667, + "best_prompt": 21.85, + "prompt_id": "p3", + "CPS": 20.74803166666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_GR.json b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..1f8bd8998ccd6876a0227a2fa7e9ddebcfdae73c --- /dev/null +++ b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 13.395712833333334, + "config": { + "model_name": "Henrychur/MMed-Llama-3-8B", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "GR", + "model": "Henrychur/MMed-Llama-3-8B", + "base_model": "LlamaForCausalLM", + "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943", + "submitted_time": "2024-05-22 09:17:24+00:00", + "num_params_billion": null, + "language": "en_zh_ja_fr_ru_es" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 16.669999999999998, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 10.89, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 16.669999999999998, + "stderr": 0.0 + } + ], + "average_accuracy": 14.743333333333332, + "best_prompt": 16.669999999999998, + "prompt_id": "p1", + "CPS": 16.348824666666665 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 8.21, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 10.530000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 10.36, + "stderr": 0.0 + } + ], + "average_accuracy": 9.700000000000001, + "best_prompt": 10.530000000000001, + "prompt_id": "p2", + "CPS": 10.442601000000002 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_IT.json b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..da2c2fd177cb5d790551318e999fa1074ce03aad --- /dev/null +++ b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 28.10758633333333, + "config": { + "model_name": "Henrychur/MMed-Llama-3-8B", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "IT", + "model": "Henrychur/MMed-Llama-3-8B", + "base_model": "LlamaForCausalLM", + "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943", + "submitted_time": "2024-05-22 09:17:24+00:00", + "num_params_billion": null, + "language": "en_zh_ja_fr_ru_es" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 32.99, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 40.23, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 39.379999999999995, + "stderr": 0.0 + } + ], + "average_accuracy": 37.53333333333333, + "best_prompt": 40.23, + "prompt_id": "p2", + "CPS": 39.145131 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 9.77, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 12.26, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 17.89, + "stderr": 0.0 + } + ], + "average_accuracy": 13.306666666666667, + "best_prompt": 17.89, + "prompt_id": "p3", + "CPS": 17.070041666666665 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_PL.json b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..40f10d5de954c6c63527e4da5a86c6b5bc87fb0b --- /dev/null +++ b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 25.157004666666666, + "config": { + "model_name": "Henrychur/MMed-Llama-3-8B", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "PL", + "model": "Henrychur/MMed-Llama-3-8B", + "base_model": "LlamaForCausalLM", + "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943", + "submitted_time": "2024-05-22 09:17:24+00:00", + "num_params_billion": null, + "language": "en_zh_ja_fr_ru_es" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 39.92, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 39.160000000000004, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 39.92, + "stderr": 0.0 + } + ], + "average_accuracy": 39.666666666666664, + "best_prompt": 39.92, + "prompt_id": "p1", + "CPS": 39.81886933333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 9.98, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 10.549999999999999, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 9.56, + "stderr": 0.0 + } + ], + "average_accuracy": 10.03, + "best_prompt": 10.549999999999999, + "prompt_id": "p2", + "CPS": 10.49514 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_SK.json b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..67253c8db4db743459db1f7a577231fcf3efbea7 --- /dev/null +++ b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 23.0736205, + "config": { + "model_name": "Henrychur/MMed-Llama-3-8B", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SK", + "model": "Henrychur/MMed-Llama-3-8B", + "base_model": "LlamaForCausalLM", + "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943", + "submitted_time": "2024-05-22 09:17:24+00:00", + "num_params_billion": null, + "language": "en_zh_ja_fr_ru_es" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 34.44, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 36.32, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 34.44, + "stderr": 0.0 + } + ], + "average_accuracy": 35.06666666666666, + "best_prompt": 36.32, + "prompt_id": "p2", + "CPS": 35.864789333333334 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 7.340000000000001, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 10.45, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 8.75, + "stderr": 0.0 + } + ], + "average_accuracy": 8.846666666666666, + "best_prompt": 10.45, + "prompt_id": "p2", + "CPS": 10.282451666666665 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_SL.json b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..dea3d7d1144e157500dfe519f09d449c46cf8c3e --- /dev/null +++ b/e3c_llm_results/Henrychur/MMed-Llama-3-8B_10_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 23.493655333333336, + "config": { + "model_name": "Henrychur/MMed-Llama-3-8B", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SL", + "model": "Henrychur/MMed-Llama-3-8B", + "base_model": "LlamaForCausalLM", + "revision": "6c3057bb49ac499970eb2891daaef9b5c14f6943", + "submitted_time": "2024-05-22 09:17:24+00:00", + "num_params_billion": null, + "language": "en_zh_ja_fr_ru_es" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 35.58, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 40.45, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 35.58, + "stderr": 0.0 + } + ], + "average_accuracy": 37.20333333333333, + "best_prompt": 40.45, + "prompt_id": "p2", + "CPS": 39.136723333333336 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 7.870000000000001, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 7.8100000000000005, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 7.19, + "stderr": 0.0 + } + ], + "average_accuracy": 7.623333333333334, + "best_prompt": 7.870000000000001, + "prompt_id": "p1", + "CPS": 7.850587333333335 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/HiTZ/Medical-mT5-large_0_EN.json b/e3c_llm_results/HiTZ/Medical-mT5-large_0_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..df2d5cef5def0f6a81024f93970902e131347a5c --- /dev/null +++ b/e3c_llm_results/HiTZ/Medical-mT5-large_0_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 4.530016666666667, + "config": { + "model_name": "HiTZ/Medical-mT5-large", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "EN", + "model": "HiTZ/Medical-mT5-large", + "base_model": "MT5ForConditionalGeneration", + "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47", + "submitted_time": "2023-10-31 15:15:15+00:00", + "num_params_billion": null, + "language": "en_es_fr_it" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 9.4, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 3.3099999999999996, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 4.64, + "stderr": 0.0 + } + ], + "average_accuracy": 5.783333333333334, + "best_prompt": 9.4, + "prompt_id": "p1", + "CPS": 9.060033333333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + } + ], + "average_accuracy": 0.0, + "best_prompt": 0.0, + "prompt_id": "p1", + "CPS": 0.0 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/HiTZ/Medical-mT5-large_0_GR.json b/e3c_llm_results/HiTZ/Medical-mT5-large_0_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..d17115f37138071094310f0a6de0ce1a5ccac8af --- /dev/null +++ b/e3c_llm_results/HiTZ/Medical-mT5-large_0_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 4.256631333333333, + "config": { + "model_name": "HiTZ/Medical-mT5-large", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "GR", + "model": "HiTZ/Medical-mT5-large", + "base_model": "MT5ForConditionalGeneration", + "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47", + "submitted_time": "2023-10-31 15:15:15+00:00", + "num_params_billion": null, + "language": "en_es_fr_it" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 8.59, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 5.91, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 8.59, + "stderr": 0.0 + } + ], + "average_accuracy": 7.696666666666666, + "best_prompt": 8.59, + "prompt_id": "p1", + "CPS": 8.513262666666666 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + } + ], + "average_accuracy": 0.0, + "best_prompt": 0.0, + "prompt_id": "p1", + "CPS": 0.0 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/HiTZ/Medical-mT5-large_0_IT.json b/e3c_llm_results/HiTZ/Medical-mT5-large_0_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..f821f49f7870f19cb1f59af29a727562295cc28f --- /dev/null +++ b/e3c_llm_results/HiTZ/Medical-mT5-large_0_IT.json @@ -0,0 +1,63 @@ +{ + "average_CPS": 4.550473333333333, + "config": { + "model_name": "HiTZ/Medical-mT5-large", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "IT", + "model": "HiTZ/Medical-mT5-large", + "base_model": "MT5ForConditionalGeneration", + "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47", + "submitted_time": "2023-10-31 15:15:15+00:00", + "num_params_billion": null, + "language": "en_es_fr_it" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 7.7, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 9.2, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 7.470000000000001, + "stderr": 0.0 + } + ], + "average_accuracy": 8.123333333333333, + "best_prompt": 9.2, + "prompt_id": "p2", + "CPS": 9.100946666666665 + }, + "RE": { + "prompts": [ + { + "prompt": "p2", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + } + ], + "average_accuracy": 0.0, + "best_prompt": 0.0, + "prompt_id": "p2", + "CPS": 0.0 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/HiTZ/Medical-mT5-large_0_PL.json b/e3c_llm_results/HiTZ/Medical-mT5-large_0_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..895222ddaac99f8a4615bb18ed46fa7547aa8903 --- /dev/null +++ b/e3c_llm_results/HiTZ/Medical-mT5-large_0_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 2.1520960000000002, + "config": { + "model_name": "HiTZ/Medical-mT5-large", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "PL", + "model": "HiTZ/Medical-mT5-large", + "base_model": "MT5ForConditionalGeneration", + "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47", + "submitted_time": "2023-10-31 15:15:15+00:00", + "num_params_billion": null, + "language": "en_es_fr_it" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 2.44, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 4.36, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 2.44, + "stderr": 0.0 + } + ], + "average_accuracy": 3.08, + "best_prompt": 4.36, + "prompt_id": "p2", + "CPS": 4.3041920000000005 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + } + ], + "average_accuracy": 0.0, + "best_prompt": 0.0, + "prompt_id": "p1", + "CPS": 0.0 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/HiTZ/Medical-mT5-large_0_SK.json b/e3c_llm_results/HiTZ/Medical-mT5-large_0_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..218eaa52af3ebdbbcbd29265cab583d722ffdc9f --- /dev/null +++ b/e3c_llm_results/HiTZ/Medical-mT5-large_0_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 4.3259333333333325, + "config": { + "model_name": "HiTZ/Medical-mT5-large", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SK", + "model": "HiTZ/Medical-mT5-large", + "base_model": "MT5ForConditionalGeneration", + "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47", + "submitted_time": "2023-10-31 15:15:15+00:00", + "num_params_billion": null, + "language": "en_es_fr_it" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 8.799999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 3.75, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 8.799999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 7.116666666666666, + "best_prompt": 8.799999999999999, + "prompt_id": "p1", + "CPS": 8.651866666666665 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + } + ], + "average_accuracy": 0.0, + "best_prompt": 0.0, + "prompt_id": "p1", + "CPS": 0.0 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/HiTZ/Medical-mT5-large_0_SL.json b/e3c_llm_results/HiTZ/Medical-mT5-large_0_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..ad806afedb3905cb3e67496fc63ffb51e7520387 --- /dev/null +++ b/e3c_llm_results/HiTZ/Medical-mT5-large_0_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 3.859359, + "config": { + "model_name": "HiTZ/Medical-mT5-large", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SL", + "model": "HiTZ/Medical-mT5-large", + "base_model": "MT5ForConditionalGeneration", + "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47", + "submitted_time": "2023-10-31 15:15:15+00:00", + "num_params_billion": null, + "language": "en_es_fr_it" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 7.7700000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 5.79, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 7.7700000000000005, + "stderr": 0.0 + } + ], + "average_accuracy": 7.11, + "best_prompt": 7.7700000000000005, + "prompt_id": "p1", + "CPS": 7.718718 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + } + ], + "average_accuracy": 0.0, + "best_prompt": 0.0, + "prompt_id": "p1", + "CPS": 0.0 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/HiTZ/Medical-mT5-large_10_EN.json b/e3c_llm_results/HiTZ/Medical-mT5-large_10_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..83d44663ba140bd7986281e05f24ae517e5fec0f --- /dev/null +++ b/e3c_llm_results/HiTZ/Medical-mT5-large_10_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 7.250459833333332, + "config": { + "model_name": "HiTZ/Medical-mT5-large", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "EN", + "model": "HiTZ/Medical-mT5-large", + "base_model": "MT5ForConditionalGeneration", + "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47", + "submitted_time": "2023-10-31 15:15:15+00:00", + "num_params_billion": null, + "language": "en_es_fr_it" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 12.15, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 14.149999999999999, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 13.22, + "stderr": 0.0 + } + ], + "average_accuracy": 13.173333333333332, + "best_prompt": 14.149999999999999, + "prompt_id": "p2", + "CPS": 14.011801666666665 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.27999999999999997, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.16, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.49, + "stderr": 0.0 + } + ], + "average_accuracy": 0.31, + "best_prompt": 0.49, + "prompt_id": "p3", + "CPS": 0.489118 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/HiTZ/Medical-mT5-large_10_GR.json b/e3c_llm_results/HiTZ/Medical-mT5-large_10_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..d7cbc234177483f246aa7222fe098ffd9e9050c6 --- /dev/null +++ b/e3c_llm_results/HiTZ/Medical-mT5-large_10_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 7.3897435, + "config": { + "model_name": "HiTZ/Medical-mT5-large", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "GR", + "model": "HiTZ/Medical-mT5-large", + "base_model": "MT5ForConditionalGeneration", + "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47", + "submitted_time": "2023-10-31 15:15:15+00:00", + "num_params_billion": null, + "language": "en_es_fr_it" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 14.549999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 14.34, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 14.549999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 14.479999999999999, + "best_prompt": 14.549999999999999, + "prompt_id": "p1", + "CPS": 14.539814999999999 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.24, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.06999999999999999, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + } + ], + "average_accuracy": 0.10333333333333333, + "best_prompt": 0.24, + "prompt_id": "p1", + "CPS": 0.239672 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/HiTZ/Medical-mT5-large_10_IT.json b/e3c_llm_results/HiTZ/Medical-mT5-large_10_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..027c6bcc93f99fe26efb19434f44ce249e42b32e --- /dev/null +++ b/e3c_llm_results/HiTZ/Medical-mT5-large_10_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 9.117947333333333, + "config": { + "model_name": "HiTZ/Medical-mT5-large", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "IT", + "model": "HiTZ/Medical-mT5-large", + "base_model": "MT5ForConditionalGeneration", + "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47", + "submitted_time": "2023-10-31 15:15:15+00:00", + "num_params_billion": null, + "language": "en_es_fr_it" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 16.16, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 17.740000000000002, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 16.900000000000002, + "stderr": 0.0 + } + ], + "average_accuracy": 16.933333333333334, + "best_prompt": 17.740000000000002, + "prompt_id": "p2", + "CPS": 17.596897333333335 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.35000000000000003, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.64, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.45999999999999996, + "stderr": 0.0 + } + ], + "average_accuracy": 0.48333333333333334, + "best_prompt": 0.64, + "prompt_id": "p2", + "CPS": 0.6389973333333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/HiTZ/Medical-mT5-large_10_PL.json b/e3c_llm_results/HiTZ/Medical-mT5-large_10_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..87c60c0b6167aed67d83e7f074584fa26062c5ce --- /dev/null +++ b/e3c_llm_results/HiTZ/Medical-mT5-large_10_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 7.915078666666666, + "config": { + "model_name": "HiTZ/Medical-mT5-large", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "PL", + "model": "HiTZ/Medical-mT5-large", + "base_model": "MT5ForConditionalGeneration", + "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47", + "submitted_time": "2023-10-31 15:15:15+00:00", + "num_params_billion": null, + "language": "en_es_fr_it" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 15.0, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 15.479999999999999, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 15.0, + "stderr": 0.0 + } + ], + "average_accuracy": 15.159999999999998, + "best_prompt": 15.479999999999999, + "prompt_id": "p2", + "CPS": 15.430463999999999 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.4, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.22999999999999998, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.33999999999999997, + "stderr": 0.0 + } + ], + "average_accuracy": 0.3233333333333333, + "best_prompt": 0.4, + "prompt_id": "p1", + "CPS": 0.39969333333333334 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/HiTZ/Medical-mT5-large_10_SK.json b/e3c_llm_results/HiTZ/Medical-mT5-large_10_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..6bd5f3d46bc83aaa152ed3a3044713e317ef066b --- /dev/null +++ b/e3c_llm_results/HiTZ/Medical-mT5-large_10_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 7.5838598333333325, + "config": { + "model_name": "HiTZ/Medical-mT5-large", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SK", + "model": "HiTZ/Medical-mT5-large", + "base_model": "MT5ForConditionalGeneration", + "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47", + "submitted_time": "2023-10-31 15:15:15+00:00", + "num_params_billion": null, + "language": "en_es_fr_it" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 14.85, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 13.600000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 14.85, + "stderr": 0.0 + } + ], + "average_accuracy": 14.433333333333332, + "best_prompt": 14.85, + "prompt_id": "p1", + "CPS": 14.788124999999999 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.38, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.24, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.2, + "stderr": 0.0 + } + ], + "average_accuracy": 0.2733333333333334, + "best_prompt": 0.38, + "prompt_id": "p1", + "CPS": 0.3795946666666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/HiTZ/Medical-mT5-large_10_SL.json b/e3c_llm_results/HiTZ/Medical-mT5-large_10_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..c0ba959636d77a87783090bc8e67b69922d3e818 --- /dev/null +++ b/e3c_llm_results/HiTZ/Medical-mT5-large_10_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 7.7788705, + "config": { + "model_name": "HiTZ/Medical-mT5-large", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SL", + "model": "HiTZ/Medical-mT5-large", + "base_model": "MT5ForConditionalGeneration", + "revision": "e8ae7101f0ab1ed5b8add8846e44a2d39f6e2c47", + "submitted_time": "2023-10-31 15:15:15+00:00", + "num_params_billion": null, + "language": "en_es_fr_it" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 14.7, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 13.25, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 14.7, + "stderr": 0.0 + } + ], + "average_accuracy": 14.216666666666667, + "best_prompt": 14.7, + "prompt_id": "p1", + "CPS": 14.62895 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.73, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.74, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.9299999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 0.7999999999999999, + "best_prompt": 0.9299999999999999, + "prompt_id": "p3", + "CPS": 0.9287909999999999 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_EN.json b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..1cdc54a8c2cd05b3fcfd7ce10eacdf7bb88adf2b --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 36.19732933333333, + "config": { + "model_name": "Qwen/Qwen2.5-14B-Instruct-1M", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "EN", + "model": "Qwen/Qwen2.5-14B-Instruct-1M", + "base_model": "Qwen2ForCausalLM", + "revision": "620fad32de7bdd2293b3d99b39eba2fe63e97438", + "submitted_time": "2025-01-23 13:23:24+00:00", + "num_params_billion": 14.770033664, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 34.25, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 11.81, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 28.93, + "stderr": 0.0 + } + ], + "average_accuracy": 24.996666666666666, + "best_prompt": 34.25, + "prompt_id": "p1", + "CPS": 31.08073333333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 41.349999999999994, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 39.17, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 41.72, + "stderr": 0.0 + } + ], + "average_accuracy": 40.74666666666666, + "best_prompt": 41.72, + "prompt_id": "p3", + "CPS": 41.31392533333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_GR.json b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..28522394ede758380a3abe1e1db46b14a5c98299 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 27.333585333333332, + "config": { + "model_name": "Qwen/Qwen2.5-14B-Instruct-1M", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "GR", + "model": "Qwen/Qwen2.5-14B-Instruct-1M", + "base_model": "Qwen2ForCausalLM", + "revision": "620fad32de7bdd2293b3d99b39eba2fe63e97438", + "submitted_time": "2025-01-23 13:23:24+00:00", + "num_params_billion": 14.770033664, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 13.389999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 11.91, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 13.389999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 12.896666666666667, + "best_prompt": 13.389999999999999, + "prompt_id": "p1", + "CPS": 13.323942666666666 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 37.96, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 42.66, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 38.1, + "stderr": 0.0 + } + ], + "average_accuracy": 39.57333333333333, + "best_prompt": 42.66, + "prompt_id": "p2", + "CPS": 41.343227999999996 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_IT.json b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..14e002b9a884094b8634b359b62d666612a6bc65 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 32.46564883333333, + "config": { + "model_name": "Qwen/Qwen2.5-14B-Instruct-1M", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "IT", + "model": "Qwen/Qwen2.5-14B-Instruct-1M", + "base_model": "Qwen2ForCausalLM", + "revision": "620fad32de7bdd2293b3d99b39eba2fe63e97438", + "submitted_time": "2025-01-23 13:23:24+00:00", + "num_params_billion": 14.770033664, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 24.67, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 17.09, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 22.34, + "stderr": 0.0 + } + ], + "average_accuracy": 21.366666666666664, + "best_prompt": 24.67, + "prompt_id": "p1", + "CPS": 23.855067666666667 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 41.730000000000004, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 37.7, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 41.06, + "stderr": 0.0 + } + ], + "average_accuracy": 40.163333333333334, + "best_prompt": 41.730000000000004, + "prompt_id": "p1", + "CPS": 41.07623 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_PL.json b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..b343ae5f06a17dc984b8450552fb03dd35b90914 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 24.780516499999997, + "config": { + "model_name": "Qwen/Qwen2.5-14B-Instruct-1M", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "PL", + "model": "Qwen/Qwen2.5-14B-Instruct-1M", + "base_model": "Qwen2ForCausalLM", + "revision": "620fad32de7bdd2293b3d99b39eba2fe63e97438", + "submitted_time": "2025-01-23 13:23:24+00:00", + "num_params_billion": 14.770033664, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 6.97, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 3.64, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 6.97, + "stderr": 0.0 + } + ], + "average_accuracy": 5.859999999999999, + "best_prompt": 6.97, + "prompt_id": "p1", + "CPS": 6.892633 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 38.03, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 44.64, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 38.0, + "stderr": 0.0 + } + ], + "average_accuracy": 40.223333333333336, + "best_prompt": 44.64, + "prompt_id": "p2", + "CPS": 42.6684 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_SK.json b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..f5dfab48e9de53b704965af94b51b83e11f122ab --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 27.026387333333332, + "config": { + "model_name": "Qwen/Qwen2.5-14B-Instruct-1M", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SK", + "model": "Qwen/Qwen2.5-14B-Instruct-1M", + "base_model": "Qwen2ForCausalLM", + "revision": "620fad32de7bdd2293b3d99b39eba2fe63e97438", + "submitted_time": "2025-01-23 13:23:24+00:00", + "num_params_billion": 14.770033664, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 12.2, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 4.26, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 12.2, + "stderr": 0.0 + } + ], + "average_accuracy": 9.553333333333333, + "best_prompt": 12.2, + "prompt_id": "p1", + "CPS": 11.877106666666666 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 40.27, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 42.94, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 40.27, + "stderr": 0.0 + } + ], + "average_accuracy": 41.160000000000004, + "best_prompt": 42.94, + "prompt_id": "p2", + "CPS": 42.175668 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_SL.json b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..923a3e368758bba9fb1739d661fb63872fc04b4b --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_0_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 37.45460366666667, + "config": { + "model_name": "Qwen/Qwen2.5-14B-Instruct-1M", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SL", + "model": "Qwen/Qwen2.5-14B-Instruct-1M", + "base_model": "Qwen2ForCausalLM", + "revision": "620fad32de7bdd2293b3d99b39eba2fe63e97438", + "submitted_time": "2025-01-23 13:23:24+00:00", + "num_params_billion": 14.770033664, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 39.1, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 23.75, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 39.1, + "stderr": 0.0 + } + ], + "average_accuracy": 33.983333333333334, + "best_prompt": 39.1, + "prompt_id": "p1", + "CPS": 37.099383333333336 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 37.75, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 37.830000000000005, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 37.75, + "stderr": 0.0 + } + ], + "average_accuracy": 37.77666666666667, + "best_prompt": 37.830000000000005, + "prompt_id": "p2", + "CPS": 37.809824000000006 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_EN.json b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..ad2e64f692310e59d4979f2769f0728866ed8d23 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 61.561666833333334, + "config": { + "model_name": "Qwen/Qwen2.5-14B-Instruct-1M", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "EN", + "model": "Qwen/Qwen2.5-14B-Instruct-1M", + "base_model": "Qwen2ForCausalLM", + "revision": "620fad32de7bdd2293b3d99b39eba2fe63e97438", + "submitted_time": "2025-01-23 13:23:24+00:00", + "num_params_billion": 14.770033664, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 60.91, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 56.46, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 62.43, + "stderr": 0.0 + } + ], + "average_accuracy": 59.93333333333334, + "best_prompt": 62.43, + "prompt_id": "p3", + "CPS": 60.871331000000005 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 63.32, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 60.25, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 61.33, + "stderr": 0.0 + } + ], + "average_accuracy": 61.63333333333333, + "best_prompt": 63.32, + "prompt_id": "p1", + "CPS": 62.25200266666666 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_GR.json b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..e09277e8cb3a41fed6acfe7d67ee311564df77c1 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 60.84101533333333, + "config": { + "model_name": "Qwen/Qwen2.5-14B-Instruct-1M", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "GR", + "model": "Qwen/Qwen2.5-14B-Instruct-1M", + "base_model": "Qwen2ForCausalLM", + "revision": "620fad32de7bdd2293b3d99b39eba2fe63e97438", + "submitted_time": "2025-01-23 13:23:24+00:00", + "num_params_billion": 14.770033664, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 61.19, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 58.47, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 61.19, + "stderr": 0.0 + } + ], + "average_accuracy": 60.28333333333333, + "best_prompt": 61.19, + "prompt_id": "p1", + "CPS": 60.635210666666666 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 59.62, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 60.24, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 61.83, + "stderr": 0.0 + } + ], + "average_accuracy": 60.56333333333333, + "best_prompt": 61.83, + "prompt_id": "p3", + "CPS": 61.046820000000004 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_IT.json b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..612f3d823ed5c8e68b855df3a6a4d42fcfb3273b --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 63.1285495, + "config": { + "model_name": "Qwen/Qwen2.5-14B-Instruct-1M", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "IT", + "model": "Qwen/Qwen2.5-14B-Instruct-1M", + "base_model": "Qwen2ForCausalLM", + "revision": "620fad32de7bdd2293b3d99b39eba2fe63e97438", + "submitted_time": "2025-01-23 13:23:24+00:00", + "num_params_billion": 14.770033664, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 67.19000000000001, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 63.27, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 66.61, + "stderr": 0.0 + } + ], + "average_accuracy": 65.69000000000001, + "best_prompt": 67.19000000000001, + "prompt_id": "p1", + "CPS": 66.18215000000001 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 57.67, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 59.98, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 60.92999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 59.526666666666664, + "best_prompt": 60.92999999999999, + "prompt_id": "p3", + "CPS": 60.07494899999999 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_PL.json b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..302d2bd2ae5f0e22e7b2e4592d0e6fbc37b4880f --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 61.214045500000005, + "config": { + "model_name": "Qwen/Qwen2.5-14B-Instruct-1M", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "PL", + "model": "Qwen/Qwen2.5-14B-Instruct-1M", + "base_model": "Qwen2ForCausalLM", + "revision": "620fad32de7bdd2293b3d99b39eba2fe63e97438", + "submitted_time": "2025-01-23 13:23:24+00:00", + "num_params_billion": 14.770033664, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 62.260000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 58.24, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 62.260000000000005, + "stderr": 0.0 + } + ], + "average_accuracy": 60.92000000000001, + "best_prompt": 62.260000000000005, + "prompt_id": "p1", + "CPS": 61.42571600000001 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 59.91, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 54.66, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 63.74999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 59.44, + "best_prompt": 63.74999999999999, + "prompt_id": "p3", + "CPS": 61.002375 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_SK.json b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..6babcf1d0705f4e5a54e486e17e2fbcdba611556 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 61.83102316666667, + "config": { + "model_name": "Qwen/Qwen2.5-14B-Instruct-1M", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SK", + "model": "Qwen/Qwen2.5-14B-Instruct-1M", + "base_model": "Qwen2ForCausalLM", + "revision": "620fad32de7bdd2293b3d99b39eba2fe63e97438", + "submitted_time": "2025-01-23 13:23:24+00:00", + "num_params_billion": 14.770033664, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 63.85999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 64.86, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 63.85999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 64.19333333333333, + "best_prompt": 64.86, + "prompt_id": "p2", + "CPS": 64.4276 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 58.940000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 58.45, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 59.589999999999996, + "stderr": 0.0 + } + ], + "average_accuracy": 58.99333333333334, + "best_prompt": 59.589999999999996, + "prompt_id": "p3", + "CPS": 59.23444633333334 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_SL.json b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..cc6a741c3b3abf073543a4dc67faf5ebfd74131b --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-14B-Instruct-1M_10_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 61.436353666666676, + "config": { + "model_name": "Qwen/Qwen2.5-14B-Instruct-1M", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SL", + "model": "Qwen/Qwen2.5-14B-Instruct-1M", + "base_model": "Qwen2ForCausalLM", + "revision": "620fad32de7bdd2293b3d99b39eba2fe63e97438", + "submitted_time": "2025-01-23 13:23:24+00:00", + "num_params_billion": 14.770033664, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 64.67, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 61.78, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 64.67, + "stderr": 0.0 + } + ], + "average_accuracy": 63.70666666666667, + "best_prompt": 64.67, + "prompt_id": "p1", + "CPS": 64.04701233333334 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 59.489999999999995, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 57.82000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 57.809999999999995, + "stderr": 0.0 + } + ], + "average_accuracy": 58.373333333333335, + "best_prompt": 59.489999999999995, + "prompt_id": "p1", + "CPS": 58.825695 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_EN.json b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..abc67fe9fd83ddc5761d1b3dd386127aeaed70f9 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 41.510924, + "config": { + "model_name": "Qwen/Qwen2.5-32B-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "EN", + "model": "Qwen/Qwen2.5-32B-Instruct", + "base_model": "Qwen2ForCausalLM", + "revision": "5ede1c97bbab6ce5cda5812749b4c0bdf79b18dd", + "submitted_time": "2024-09-17 04:17:55+00:00", + "num_params_billion": 32.763876352, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 38.04, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 30.680000000000003, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 29.64, + "stderr": 0.0 + } + ], + "average_accuracy": 32.78666666666667, + "best_prompt": 38.04, + "prompt_id": "p1", + "CPS": 36.041632 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 47.339999999999996, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 46.489999999999995, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 45.910000000000004, + "stderr": 0.0 + } + ], + "average_accuracy": 46.580000000000005, + "best_prompt": 47.339999999999996, + "prompt_id": "p1", + "CPS": 46.980216 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_GR.json b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..98d395fed4d9c8581bbc3ad5ac0f11ddc2d0599d --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 46.818379166666666, + "config": { + "model_name": "Qwen/Qwen2.5-32B-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "GR", + "model": "Qwen/Qwen2.5-32B-Instruct", + "base_model": "Qwen2ForCausalLM", + "revision": "5ede1c97bbab6ce5cda5812749b4c0bdf79b18dd", + "submitted_time": "2024-09-17 04:17:55+00:00", + "num_params_billion": 32.763876352, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 59.760000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 15.68, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 59.760000000000005, + "stderr": 0.0 + } + ], + "average_accuracy": 45.06666666666667, + "best_prompt": 59.760000000000005, + "prompt_id": "p1", + "CPS": 50.979264 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 43.93, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 40.83, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 38.34, + "stderr": 0.0 + } + ], + "average_accuracy": 41.03333333333333, + "best_prompt": 43.93, + "prompt_id": "p1", + "CPS": 42.65749433333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_IT.json b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..4c32c65ecc079690b39748655fa9db2fd8be2f00 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 39.086941, + "config": { + "model_name": "Qwen/Qwen2.5-32B-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "IT", + "model": "Qwen/Qwen2.5-32B-Instruct", + "base_model": "Qwen2ForCausalLM", + "revision": "5ede1c97bbab6ce5cda5812749b4c0bdf79b18dd", + "submitted_time": "2024-09-17 04:17:55+00:00", + "num_params_billion": 32.763876352, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 37.580000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 16.470000000000002, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 27.96, + "stderr": 0.0 + } + ], + "average_accuracy": 27.33666666666667, + "best_prompt": 37.580000000000005, + "prompt_id": "p1", + "CPS": 33.730555333333335 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 45.050000000000004, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 41.589999999999996, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 44.47, + "stderr": 0.0 + } + ], + "average_accuracy": 43.70333333333334, + "best_prompt": 45.050000000000004, + "prompt_id": "p1", + "CPS": 44.44332666666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_PL.json b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..18cb5ec800a3939b7d971eb671c5e385f18131b6 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 34.071664166666665, + "config": { + "model_name": "Qwen/Qwen2.5-32B-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "PL", + "model": "Qwen/Qwen2.5-32B-Instruct", + "base_model": "Qwen2ForCausalLM", + "revision": "5ede1c97bbab6ce5cda5812749b4c0bdf79b18dd", + "submitted_time": "2024-09-17 04:17:55+00:00", + "num_params_billion": 32.763876352, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 24.86, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 23.11, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 24.86, + "stderr": 0.0 + } + ], + "average_accuracy": 24.276666666666667, + "best_prompt": 24.86, + "prompt_id": "p1", + "CPS": 24.714983333333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 38.65, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 45.69, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 37.88, + "stderr": 0.0 + } + ], + "average_accuracy": 40.74, + "best_prompt": 45.69, + "prompt_id": "p2", + "CPS": 43.428345 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_SK.json b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..a6f033aa4fa172cbd0557a2bcedc45049cfc7106 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 38.03573266666667, + "config": { + "model_name": "Qwen/Qwen2.5-32B-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SK", + "model": "Qwen/Qwen2.5-32B-Instruct", + "base_model": "Qwen2ForCausalLM", + "revision": "5ede1c97bbab6ce5cda5812749b4c0bdf79b18dd", + "submitted_time": "2024-09-17 04:17:55+00:00", + "num_params_billion": 32.763876352, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 35.78, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 29.68, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 35.78, + "stderr": 0.0 + } + ], + "average_accuracy": 33.74666666666667, + "best_prompt": 35.78, + "prompt_id": "p1", + "CPS": 35.05247333333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 39.71, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 41.52, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 39.71, + "stderr": 0.0 + } + ], + "average_accuracy": 40.31333333333333, + "best_prompt": 41.52, + "prompt_id": "p2", + "CPS": 41.018992000000004 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_SL.json b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..ef28592ea5aca3a11de2bd06ae18a57e9f4bf594 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_0_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 37.09308866666666, + "config": { + "model_name": "Qwen/Qwen2.5-32B-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SL", + "model": "Qwen/Qwen2.5-32B-Instruct", + "base_model": "Qwen2ForCausalLM", + "revision": "5ede1c97bbab6ce5cda5812749b4c0bdf79b18dd", + "submitted_time": "2024-09-17 04:17:55+00:00", + "num_params_billion": 32.763876352, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 33.44, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 28.63, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 33.44, + "stderr": 0.0 + } + ], + "average_accuracy": 31.836666666666662, + "best_prompt": 33.44, + "prompt_id": "p1", + "CPS": 32.90384533333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 39.79, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 41.86, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 39.79, + "stderr": 0.0 + } + ], + "average_accuracy": 40.48, + "best_prompt": 41.86, + "prompt_id": "p2", + "CPS": 41.282332 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_EN.json b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..548c90450e3b193ae6ba7a5a99aadb0e0d8bcbfb --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 62.17364133333332, + "config": { + "model_name": "Qwen/Qwen2.5-32B-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "EN", + "model": "Qwen/Qwen2.5-32B-Instruct", + "base_model": "Qwen2ForCausalLM", + "revision": "5ede1c97bbab6ce5cda5812749b4c0bdf79b18dd", + "submitted_time": "2024-09-17 04:17:55+00:00", + "num_params_billion": 32.763876352, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 59.699999999999996, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 56.02, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 61.129999999999995, + "stderr": 0.0 + } + ], + "average_accuracy": 58.949999999999996, + "best_prompt": 61.129999999999995, + "prompt_id": "p3", + "CPS": 59.79736599999999 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 64.82, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 64.69, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 63.7, + "stderr": 0.0 + } + ], + "average_accuracy": 64.40333333333332, + "best_prompt": 64.82, + "prompt_id": "p1", + "CPS": 64.54991666666665 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_GR.json b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..f864a1f533f405bf28a1bef9dab915b09c39db9b --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 60.26106666666667, + "config": { + "model_name": "Qwen/Qwen2.5-32B-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "GR", + "model": "Qwen/Qwen2.5-32B-Instruct", + "base_model": "Qwen2ForCausalLM", + "revision": "5ede1c97bbab6ce5cda5812749b4c0bdf79b18dd", + "submitted_time": "2024-09-17 04:17:55+00:00", + "num_params_billion": 32.763876352, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 61.96, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 61.309999999999995, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 61.96, + "stderr": 0.0 + } + ], + "average_accuracy": 61.74333333333333, + "best_prompt": 61.96, + "prompt_id": "p1", + "CPS": 61.82575333333334 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 59.13, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 58.96, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 57.099999999999994, + "stderr": 0.0 + } + ], + "average_accuracy": 58.39666666666667, + "best_prompt": 59.13, + "prompt_id": "p1", + "CPS": 58.696380000000005 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_IT.json b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..93839da32602f4fb71dad6858a4a71f9b91948d7 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 63.77547316666666, + "config": { + "model_name": "Qwen/Qwen2.5-32B-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "IT", + "model": "Qwen/Qwen2.5-32B-Instruct", + "base_model": "Qwen2ForCausalLM", + "revision": "5ede1c97bbab6ce5cda5812749b4c0bdf79b18dd", + "submitted_time": "2024-09-17 04:17:55+00:00", + "num_params_billion": 32.763876352, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 69.34, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 71.52, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 69.3, + "stderr": 0.0 + } + ], + "average_accuracy": 70.05333333333333, + "best_prompt": 71.52, + "prompt_id": "p2", + "CPS": 70.47103999999999 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 58.01, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 55.95, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 55.26, + "stderr": 0.0 + } + ], + "average_accuracy": 56.406666666666666, + "best_prompt": 58.01, + "prompt_id": "p1", + "CPS": 57.079906333333334 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_PL.json b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..fcf4994e83b99b9cdcf2823f96a46f364eacf4ad --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 59.57754466666667, + "config": { + "model_name": "Qwen/Qwen2.5-32B-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "PL", + "model": "Qwen/Qwen2.5-32B-Instruct", + "base_model": "Qwen2ForCausalLM", + "revision": "5ede1c97bbab6ce5cda5812749b4c0bdf79b18dd", + "submitted_time": "2024-09-17 04:17:55+00:00", + "num_params_billion": 32.763876352, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 60.08, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 60.040000000000006, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 60.08, + "stderr": 0.0 + } + ], + "average_accuracy": 60.06666666666666, + "best_prompt": 60.08, + "prompt_id": "p1", + "CPS": 60.071989333333335 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 58.58, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 58.68, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 59.38, + "stderr": 0.0 + } + ], + "average_accuracy": 58.879999999999995, + "best_prompt": 59.38, + "prompt_id": "p3", + "CPS": 59.083099999999995 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_SK.json b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..bff727b6119c1b6cc67c5cbcd8fb8780d195f344 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 62.042391166666675, + "config": { + "model_name": "Qwen/Qwen2.5-32B-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SK", + "model": "Qwen/Qwen2.5-32B-Instruct", + "base_model": "Qwen2ForCausalLM", + "revision": "5ede1c97bbab6ce5cda5812749b4c0bdf79b18dd", + "submitted_time": "2024-09-17 04:17:55+00:00", + "num_params_billion": 32.763876352, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 67.43, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 66.73, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 67.43, + "stderr": 0.0 + } + ], + "average_accuracy": 67.19666666666667, + "best_prompt": 67.43, + "prompt_id": "p1", + "CPS": 67.27266333333334 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 57.330000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 55.86, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 56.089999999999996, + "stderr": 0.0 + } + ], + "average_accuracy": 56.42666666666667, + "best_prompt": 57.330000000000005, + "prompt_id": "p1", + "CPS": 56.812119 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_SL.json b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..02036f3a397a3c55c02e5f35f718065b9b393a2e --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen2.5-32B-Instruct_10_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 61.441851333333325, + "config": { + "model_name": "Qwen/Qwen2.5-32B-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SL", + "model": "Qwen/Qwen2.5-32B-Instruct", + "base_model": "Qwen2ForCausalLM", + "revision": "5ede1c97bbab6ce5cda5812749b4c0bdf79b18dd", + "submitted_time": "2024-09-17 04:17:55+00:00", + "num_params_billion": 32.763876352, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 62.529999999999994, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 66.14999999999999, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 62.529999999999994, + "stderr": 0.0 + } + ], + "average_accuracy": 63.73666666666666, + "best_prompt": 66.14999999999999, + "prompt_id": "p2", + "CPS": 64.55358 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 59.919999999999995, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 58.489999999999995, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 53.39, + "stderr": 0.0 + } + ], + "average_accuracy": 57.26666666666666, + "best_prompt": 59.919999999999995, + "prompt_id": "p1", + "CPS": 58.330122666666654 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_EN.json b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..3a6a7f8fca9436871a9e9bfd4b23c97795c64cf9 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_EN.json @@ -0,0 +1,63 @@ +{ + "average_CPS": 44.38809091666667, + "config": { + "model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "EN", + "model": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "base_model": "Qwen3MoeForCausalLM", + "revision": "61082d4deaa4785f64943b443cbc2b5de7524fad", + "submitted_time": "2025-07-28 07:31:27+00:00", + "num_params_billion": 30.532122624, + "language": "" + }, + "tasks": { + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 43.94, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 40.31, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 39.97, + "stderr": 0.0 + } + ], + "average_accuracy": 41.406666666666666, + "best_prompt": 43.94, + "prompt_id": "p1", + "CPS": 42.82685333333333 + }, + "NER": { + "prompts": [ + { + "prompt": "p2", + "metric": "f1", + "value": 41.620000000000005, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 47.29, + "stderr": 0.0 + } + ], + "average_accuracy": 44.455, + "best_prompt": 47.29, + "prompt_id": "p3", + "CPS": 45.9493285 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_GR.json b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..5d5235692cb2b2a26acac34e0ddead6885b70bdf --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 41.21096783333334, + "config": { + "model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "GR", + "model": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "base_model": "Qwen3MoeForCausalLM", + "revision": "61082d4deaa4785f64943b443cbc2b5de7524fad", + "submitted_time": "2025-07-28 07:31:27+00:00", + "num_params_billion": 30.532122624, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 42.91, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 45.21, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 42.91, + "stderr": 0.0 + } + ], + "average_accuracy": 43.67666666666667, + "best_prompt": 45.21, + "prompt_id": "p2", + "CPS": 44.516780000000004 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 37.330000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 37.99, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 37.980000000000004, + "stderr": 0.0 + } + ], + "average_accuracy": 37.76666666666667, + "best_prompt": 37.99, + "prompt_id": "p2", + "CPS": 37.90515566666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_IT.json b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..621ba9b6c5c8499b480ca0f2ff13ae3c7f8707cd --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 42.221850333333336, + "config": { + "model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "IT", + "model": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "base_model": "Qwen3MoeForCausalLM", + "revision": "61082d4deaa4785f64943b443cbc2b5de7524fad", + "submitted_time": "2025-07-28 07:31:27+00:00", + "num_params_billion": 30.532122624, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 8.85, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 53.16, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 45.14, + "stderr": 0.0 + } + ], + "average_accuracy": 35.71666666666666, + "best_prompt": 53.16, + "prompt_id": "p2", + "CPS": 43.887124 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 37.84, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 41.23, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 39.72, + "stderr": 0.0 + } + ], + "average_accuracy": 39.596666666666664, + "best_prompt": 41.23, + "prompt_id": "p2", + "CPS": 40.556576666666665 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_PL.json b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..8ecf7f428cecbf2bd14164c023305251e7c9c940 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 42.480305333333334, + "config": { + "model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "PL", + "model": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "base_model": "Qwen3MoeForCausalLM", + "revision": "61082d4deaa4785f64943b443cbc2b5de7524fad", + "submitted_time": "2025-07-28 07:31:27+00:00", + "num_params_billion": 30.532122624, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 43.32, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 40.43, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 43.32, + "stderr": 0.0 + } + ], + "average_accuracy": 42.35666666666666, + "best_prompt": 43.32, + "prompt_id": "p1", + "CPS": 42.902684 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 41.52, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 42.199999999999996, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 41.870000000000005, + "stderr": 0.0 + } + ], + "average_accuracy": 41.86333333333334, + "best_prompt": 42.199999999999996, + "prompt_id": "p2", + "CPS": 42.05792666666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_SK.json b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..8e99f3289159910ad4036974ff8522a1b6e720e7 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 36.625888, + "config": { + "model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SK", + "model": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "base_model": "Qwen3MoeForCausalLM", + "revision": "61082d4deaa4785f64943b443cbc2b5de7524fad", + "submitted_time": "2025-07-28 07:31:27+00:00", + "num_params_billion": 30.532122624, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 32.31, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 33.98, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 32.31, + "stderr": 0.0 + } + ], + "average_accuracy": 32.86666666666667, + "best_prompt": 33.98, + "prompt_id": "p2", + "CPS": 33.60168933333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 39.800000000000004, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 38.67, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 39.800000000000004, + "stderr": 0.0 + } + ], + "average_accuracy": 39.42333333333334, + "best_prompt": 39.800000000000004, + "prompt_id": "p1", + "CPS": 39.65008666666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_SL.json b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..7aafffedea28794cf208c08656cf0da911bb313f --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_0_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 43.20190633333334, + "config": { + "model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SL", + "model": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "base_model": "Qwen3MoeForCausalLM", + "revision": "61082d4deaa4785f64943b443cbc2b5de7524fad", + "submitted_time": "2025-07-28 07:31:27+00:00", + "num_params_billion": 30.532122624, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 44.86, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 45.31, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 44.86, + "stderr": 0.0 + } + ], + "average_accuracy": 45.01, + "best_prompt": 45.31, + "prompt_id": "p2", + "CPS": 45.17407 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 41.15, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 41.260000000000005, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 41.15, + "stderr": 0.0 + } + ], + "average_accuracy": 41.18666666666667, + "best_prompt": 41.260000000000005, + "prompt_id": "p2", + "CPS": 41.229742666666674 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_EN.json b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..66ab4cf4698db844a85ef026fc1a8a70a24f8376 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 56.52775183333333, + "config": { + "model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "EN", + "model": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "base_model": "Qwen3MoeForCausalLM", + "revision": "61082d4deaa4785f64943b443cbc2b5de7524fad", + "submitted_time": "2025-07-28 07:31:27+00:00", + "num_params_billion": 30.532122624, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 59.86, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 55.93, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 61.42999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 59.07333333333333, + "best_prompt": 61.42999999999999, + "prompt_id": "p3", + "CPS": 59.98229966666666 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.5, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 52.61, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 53.64, + "stderr": 0.0 + } + ], + "average_accuracy": 52.583333333333336, + "best_prompt": 53.64, + "prompt_id": "p3", + "CPS": 53.073204000000004 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_GR.json b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..c9cf96c34cbea8c56f5cfa4a717dead09459b61a --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 56.23321899999999, + "config": { + "model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "GR", + "model": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "base_model": "Qwen3MoeForCausalLM", + "revision": "61082d4deaa4785f64943b443cbc2b5de7524fad", + "submitted_time": "2025-07-28 07:31:27+00:00", + "num_params_billion": 30.532122624, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 61.63999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 56.69, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 61.63999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 59.98999999999999, + "best_prompt": 61.63999999999999, + "prompt_id": "p1", + "CPS": 60.62293999999999 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 50.14999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 52.09, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 52.23, + "stderr": 0.0 + } + ], + "average_accuracy": 51.49, + "best_prompt": 52.23, + "prompt_id": "p3", + "CPS": 51.843498 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_IT.json b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..69395b5a37635269966540d3fa71a8d2235d41fa --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 63.62756866666667, + "config": { + "model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "IT", + "model": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "base_model": "Qwen3MoeForCausalLM", + "revision": "61082d4deaa4785f64943b443cbc2b5de7524fad", + "submitted_time": "2025-07-28 07:31:27+00:00", + "num_params_billion": 30.532122624, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 67.93, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 64.47, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 67.78, + "stderr": 0.0 + } + ], + "average_accuracy": 66.72666666666667, + "best_prompt": 67.93, + "prompt_id": "p1", + "CPS": 67.11257566666667 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 60.41, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 58.379999999999995, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 60.650000000000006, + "stderr": 0.0 + } + ], + "average_accuracy": 59.81333333333333, + "best_prompt": 60.650000000000006, + "prompt_id": "p3", + "CPS": 60.142561666666666 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_PL.json b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..da5844ca39a76b5c2025b5fb76ba16c2ac95517c --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 56.796842, + "config": { + "model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "PL", + "model": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "base_model": "Qwen3MoeForCausalLM", + "revision": "61082d4deaa4785f64943b443cbc2b5de7524fad", + "submitted_time": "2025-07-28 07:31:27+00:00", + "num_params_billion": 30.532122624, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 62.760000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 58.03, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 62.760000000000005, + "stderr": 0.0 + } + ], + "average_accuracy": 61.18333333333334, + "best_prompt": 62.760000000000005, + "prompt_id": "p1", + "CPS": 61.770484 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.03, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 52.0, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 51.949999999999996, + "stderr": 0.0 + } + ], + "average_accuracy": 51.66, + "best_prompt": 52.0, + "prompt_id": "p2", + "CPS": 51.8232 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_SK.json b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..5c4ac821413c34ab2455dd7a927cbca44faf8e39 --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 56.404352833333334, + "config": { + "model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SK", + "model": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "base_model": "Qwen3MoeForCausalLM", + "revision": "61082d4deaa4785f64943b443cbc2b5de7524fad", + "submitted_time": "2025-07-28 07:31:27+00:00", + "num_params_billion": 30.532122624, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 60.85, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 59.19, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 60.85, + "stderr": 0.0 + } + ], + "average_accuracy": 60.29666666666666, + "best_prompt": 60.85, + "prompt_id": "p1", + "CPS": 60.51329666666666 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 49.2, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 50.24999999999999, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 53.73, + "stderr": 0.0 + } + ], + "average_accuracy": 51.06, + "best_prompt": 53.73, + "prompt_id": "p3", + "CPS": 52.295409 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_SL.json b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..f6e80a3a2fbd0ad95707bba1d29834ed15eda75e --- /dev/null +++ b/e3c_llm_results/Qwen/Qwen3-30B-A3B-Instruct-2507_10_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 59.600933166666664, + "config": { + "model_name": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SL", + "model": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "base_model": "Qwen3MoeForCausalLM", + "revision": "61082d4deaa4785f64943b443cbc2b5de7524fad", + "submitted_time": "2025-07-28 07:31:27+00:00", + "num_params_billion": 30.532122624, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 66.14999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 59.440000000000005, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 66.14999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 63.91333333333333, + "best_prompt": 66.14999999999999, + "prompt_id": "p1", + "CPS": 64.67044499999999 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 50.62, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 55.76, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 54.290000000000006, + "stderr": 0.0 + } + ], + "average_accuracy": 53.55666666666667, + "best_prompt": 55.76, + "prompt_id": "p2", + "CPS": 54.531421333333334 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_EN.json b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..9443fb54d23a6e830e20452f33cc1ccc3dbede21 --- /dev/null +++ b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 38.6086025, + "config": { + "model_name": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "EN", + "model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "base_model": "Qwen2ForCausalLM", + "revision": "711ad2ea6aa40cfca18895e8aca02ab92df1a746", + "submitted_time": "2025-01-20 09:19:00+00:00", + "num_params_billion": 32.763876352, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 19.63, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 34.589999999999996, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 32.08, + "stderr": 0.0 + } + ], + "average_accuracy": 28.766666666666666, + "best_prompt": 34.589999999999996, + "prompt_id": "p2", + "CPS": 32.575708999999996 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 44.87, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 44.92, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 43.11, + "stderr": 0.0 + } + ], + "average_accuracy": 44.300000000000004, + "best_prompt": 44.92, + "prompt_id": "p2", + "CPS": 44.641496000000004 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_GR.json b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..ebaed835667bfa55463e8bd63065dc42e82a37e1 --- /dev/null +++ b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 36.46137383333333, + "config": { + "model_name": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "GR", + "model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "base_model": "Qwen2ForCausalLM", + "revision": "711ad2ea6aa40cfca18895e8aca02ab92df1a746", + "submitted_time": "2025-01-20 09:19:00+00:00", + "num_params_billion": 32.763876352, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 34.55, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 33.54, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 34.55, + "stderr": 0.0 + } + ], + "average_accuracy": 34.21333333333333, + "best_prompt": 34.55, + "prompt_id": "p1", + "CPS": 34.433681666666665 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 24.060000000000002, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 39.47, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 41.02, + "stderr": 0.0 + } + ], + "average_accuracy": 34.85, + "best_prompt": 41.02, + "prompt_id": "p3", + "CPS": 38.489066 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_IT.json b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..c623fc8b460f424bb49439700c29cbae22bfb0b4 --- /dev/null +++ b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 39.908362, + "config": { + "model_name": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "IT", + "model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "base_model": "Qwen2ForCausalLM", + "revision": "711ad2ea6aa40cfca18895e8aca02ab92df1a746", + "submitted_time": "2025-01-20 09:19:00+00:00", + "num_params_billion": 32.763876352, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 26.779999999999998, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 35.68, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 34.14, + "stderr": 0.0 + } + ], + "average_accuracy": 32.199999999999996, + "best_prompt": 35.68, + "prompt_id": "p2", + "CPS": 34.438336 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 45.190000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 46.11, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 42.27, + "stderr": 0.0 + } + ], + "average_accuracy": 44.52333333333333, + "best_prompt": 46.11, + "prompt_id": "p2", + "CPS": 45.378388 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_PL.json b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..2676754bc09347f8f4118a2aa9b8e4464ebc04e4 --- /dev/null +++ b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 39.199796666666664, + "config": { + "model_name": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "PL", + "model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "base_model": "Qwen2ForCausalLM", + "revision": "711ad2ea6aa40cfca18895e8aca02ab92df1a746", + "submitted_time": "2025-01-20 09:19:00+00:00", + "num_params_billion": 32.763876352, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 32.04, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 37.28, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 32.04, + "stderr": 0.0 + } + ], + "average_accuracy": 33.78666666666667, + "best_prompt": 37.28, + "prompt_id": "p2", + "CPS": 35.97768533333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 39.83, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 43.269999999999996, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 40.83, + "stderr": 0.0 + } + ], + "average_accuracy": 41.309999999999995, + "best_prompt": 43.269999999999996, + "prompt_id": "p2", + "CPS": 42.421907999999995 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_SK.json b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..0f437e9c81034e31bffd74755a079d792d8cd66c --- /dev/null +++ b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 33.894328, + "config": { + "model_name": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SK", + "model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "base_model": "Qwen2ForCausalLM", + "revision": "711ad2ea6aa40cfca18895e8aca02ab92df1a746", + "submitted_time": "2025-01-20 09:19:00+00:00", + "num_params_billion": 32.763876352, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 28.29, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 19.05, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 28.29, + "stderr": 0.0 + } + ], + "average_accuracy": 25.209999999999997, + "best_prompt": 28.29, + "prompt_id": "p1", + "CPS": 27.418667999999997 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 38.93, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 40.910000000000004, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 38.93, + "stderr": 0.0 + } + ], + "average_accuracy": 39.59, + "best_prompt": 40.910000000000004, + "prompt_id": "p2", + "CPS": 40.369988000000006 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_SL.json b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..c2730c60754fcf3b445979e83232805eb9996ee7 --- /dev/null +++ b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_0_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 34.339884000000005, + "config": { + "model_name": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SL", + "model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "base_model": "Qwen2ForCausalLM", + "revision": "711ad2ea6aa40cfca18895e8aca02ab92df1a746", + "submitted_time": "2025-01-20 09:19:00+00:00", + "num_params_billion": 32.763876352, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 28.1, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 21.92, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 28.1, + "stderr": 0.0 + } + ], + "average_accuracy": 26.040000000000003, + "best_prompt": 28.1, + "prompt_id": "p1", + "CPS": 27.521140000000003 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 41.160000000000004, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 41.15, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 41.160000000000004, + "stderr": 0.0 + } + ], + "average_accuracy": 41.156666666666666, + "best_prompt": 41.160000000000004, + "prompt_id": "p1", + "CPS": 41.15862800000001 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_EN.json b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..696e89129e5dd6e0370d3d2d9bc4b04394b43ac8 --- /dev/null +++ b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 56.163166000000004, + "config": { + "model_name": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "EN", + "model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "base_model": "Qwen2ForCausalLM", + "revision": "711ad2ea6aa40cfca18895e8aca02ab92df1a746", + "submitted_time": "2025-01-20 09:19:00+00:00", + "num_params_billion": 32.763876352, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 60.24, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 59.29, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 59.35, + "stderr": 0.0 + } + ], + "average_accuracy": 59.626666666666665, + "best_prompt": 60.24, + "prompt_id": "p1", + "CPS": 59.870528 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.910000000000004, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 51.99, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 52.73, + "stderr": 0.0 + } + ], + "average_accuracy": 52.21, + "best_prompt": 52.73, + "prompt_id": "p3", + "CPS": 52.455804 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_GR.json b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..dc934142fa8f6ae05bf7b9037604dfca268b6b8e --- /dev/null +++ b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 52.0035325, + "config": { + "model_name": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "GR", + "model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "base_model": "Qwen2ForCausalLM", + "revision": "711ad2ea6aa40cfca18895e8aca02ab92df1a746", + "submitted_time": "2025-01-20 09:19:00+00:00", + "num_params_billion": 32.763876352, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 59.28, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 57.96, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 59.28, + "stderr": 0.0 + } + ], + "average_accuracy": 58.84, + "best_prompt": 59.28, + "prompt_id": "p1", + "CPS": 59.019168 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 44.67, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 42.1, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 45.69, + "stderr": 0.0 + } + ], + "average_accuracy": 44.153333333333336, + "best_prompt": 45.69, + "prompt_id": "p3", + "CPS": 44.987897 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_IT.json b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..2da72f0769a9db34459312397de684c3b6a0326c --- /dev/null +++ b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 62.18253033333333, + "config": { + "model_name": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "IT", + "model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "base_model": "Qwen2ForCausalLM", + "revision": "711ad2ea6aa40cfca18895e8aca02ab92df1a746", + "submitted_time": "2025-01-20 09:19:00+00:00", + "num_params_billion": 32.763876352, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 69.82000000000001, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 66.79, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 69.3, + "stderr": 0.0 + } + ], + "average_accuracy": 68.63666666666667, + "best_prompt": 69.82000000000001, + "prompt_id": "p1", + "CPS": 68.99379666666667 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 55.46, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 55.26, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 55.17999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 55.29999999999999, + "best_prompt": 55.46, + "prompt_id": "p1", + "CPS": 55.37126399999999 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_PL.json b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..005df5057a6c562af58001c2d2df443d7ad25fde --- /dev/null +++ b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 56.36566883333333, + "config": { + "model_name": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "PL", + "model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "base_model": "Qwen2ForCausalLM", + "revision": "711ad2ea6aa40cfca18895e8aca02ab92df1a746", + "submitted_time": "2025-01-20 09:19:00+00:00", + "num_params_billion": 32.763876352, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 62.13999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 61.4, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 62.13999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 61.893333333333324, + "best_prompt": 62.13999999999999, + "prompt_id": "p1", + "CPS": 61.98672133333332 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 48.63, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 51.29, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 50.760000000000005, + "stderr": 0.0 + } + ], + "average_accuracy": 50.22666666666667, + "best_prompt": 51.29, + "prompt_id": "p2", + "CPS": 50.74461633333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_SK.json b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..89f66951b3703ddcd6b456073bcb139515aea658 --- /dev/null +++ b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 55.21981100000001, + "config": { + "model_name": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SK", + "model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "base_model": "Qwen2ForCausalLM", + "revision": "711ad2ea6aa40cfca18895e8aca02ab92df1a746", + "submitted_time": "2025-01-20 09:19:00+00:00", + "num_params_billion": 32.763876352, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 63.470000000000006, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 62.11, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 63.470000000000006, + "stderr": 0.0 + } + ], + "average_accuracy": 63.01666666666667, + "best_prompt": 63.470000000000006, + "prompt_id": "p1", + "CPS": 63.182269333333345 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 47.99, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 44.51, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 46.89, + "stderr": 0.0 + } + ], + "average_accuracy": 46.46333333333333, + "best_prompt": 47.99, + "prompt_id": "p1", + "CPS": 47.25735266666666 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_SL.json b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..cdfe1ae096d22d5a9a8912f4dbf8f7094c659b74 --- /dev/null +++ b/e3c_llm_results/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B_10_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 55.28181983333334, + "config": { + "model_name": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SL", + "model": "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "base_model": "Qwen2ForCausalLM", + "revision": "711ad2ea6aa40cfca18895e8aca02ab92df1a746", + "submitted_time": "2025-01-20 09:19:00+00:00", + "num_params_billion": 32.763876352, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 60.150000000000006, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 60.49, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 60.150000000000006, + "stderr": 0.0 + } + ], + "average_accuracy": 60.26333333333334, + "best_prompt": 60.49, + "prompt_id": "p2", + "CPS": 60.35288933333334 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.370000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 46.739999999999995, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 49.230000000000004, + "stderr": 0.0 + } + ], + "average_accuracy": 49.11333333333334, + "best_prompt": 51.370000000000005, + "prompt_id": "p1", + "CPS": 50.21075033333334 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/epfl-llm/meditron-7b_0_EN.json b/e3c_llm_results/epfl-llm/meditron-7b_0_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..e82541630f5756da51e91eae2580a9551d335a90 --- /dev/null +++ b/e3c_llm_results/epfl-llm/meditron-7b_0_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 6.579212, + "config": { + "model_name": "epfl-llm/meditron-7b", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "EN", + "model": "epfl-llm/meditron-7b", + "base_model": "LlamaForCausalLM", + "revision": "d7d0a5ed929384a6b059ac74198cf1d71f44ba76", + "submitted_time": "2023-11-08 16:03:23+00:00", + "num_params_billion": 6.73855488, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 5.779999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 4.1000000000000005, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 8.48, + "stderr": 0.0 + } + ], + "average_accuracy": 6.12, + "best_prompt": 8.48, + "prompt_id": "p3", + "CPS": 8.279872000000001 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 4.42, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 4.97, + "stderr": 0.0 + } + ], + "average_accuracy": 3.1300000000000003, + "best_prompt": 4.97, + "prompt_id": "p3", + "CPS": 4.878552 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/epfl-llm/meditron-7b_0_GR.json b/e3c_llm_results/epfl-llm/meditron-7b_0_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..e46b2fc6098efde35dd9c59ea5a7a12c4b985e48 --- /dev/null +++ b/e3c_llm_results/epfl-llm/meditron-7b_0_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 19.223575999999998, + "config": { + "model_name": "epfl-llm/meditron-7b", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "GR", + "model": "epfl-llm/meditron-7b", + "base_model": "LlamaForCausalLM", + "revision": "d7d0a5ed929384a6b059ac74198cf1d71f44ba76", + "submitted_time": "2023-11-08 16:03:23+00:00", + "num_params_billion": 6.73855488, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 24.169999999999998, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 24.43, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 24.169999999999998, + "stderr": 0.0 + } + ], + "average_accuracy": 24.256666666666664, + "best_prompt": 24.43, + "prompt_id": "p2", + "CPS": 24.387654666666666 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 15.559999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 1.6099999999999999, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.58, + "stderr": 0.0 + } + ], + "average_accuracy": 5.916666666666667, + "best_prompt": 15.559999999999999, + "prompt_id": "p1", + "CPS": 14.059497333333331 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/epfl-llm/meditron-7b_0_IT.json b/e3c_llm_results/epfl-llm/meditron-7b_0_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..6b1199416d61cf7eb8da6d6bfaeae14a514bbb9a --- /dev/null +++ b/e3c_llm_results/epfl-llm/meditron-7b_0_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 12.631825166666665, + "config": { + "model_name": "epfl-llm/meditron-7b", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "IT", + "model": "epfl-llm/meditron-7b", + "base_model": "LlamaForCausalLM", + "revision": "d7d0a5ed929384a6b059ac74198cf1d71f44ba76", + "submitted_time": "2023-11-08 16:03:23+00:00", + "num_params_billion": 6.73855488, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 7.7299999999999995, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 6.12, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 5.3100000000000005, + "stderr": 0.0 + } + ], + "average_accuracy": 6.386666666666667, + "best_prompt": 7.7299999999999995, + "prompt_id": "p1", + "CPS": 7.626160333333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.2, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 19.29, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 12.68, + "stderr": 0.0 + } + ], + "average_accuracy": 10.723333333333334, + "best_prompt": 19.29, + "prompt_id": "p2", + "CPS": 17.63749 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/epfl-llm/meditron-7b_0_PL.json b/e3c_llm_results/epfl-llm/meditron-7b_0_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..433136d11aa7e75af354ec87bf5564002610ef38 --- /dev/null +++ b/e3c_llm_results/epfl-llm/meditron-7b_0_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 6.367811666666667, + "config": { + "model_name": "epfl-llm/meditron-7b", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "PL", + "model": "epfl-llm/meditron-7b", + "base_model": "LlamaForCausalLM", + "revision": "d7d0a5ed929384a6b059ac74198cf1d71f44ba76", + "submitted_time": "2023-11-08 16:03:23+00:00", + "num_params_billion": 6.73855488, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 11.4, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 12.030000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 11.4, + "stderr": 0.0 + } + ], + "average_accuracy": 11.61, + "best_prompt": 12.030000000000001, + "prompt_id": "p2", + "CPS": 11.979474000000002 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.76, + "stderr": 0.0 + } + ], + "average_accuracy": 0.25333333333333335, + "best_prompt": 0.76, + "prompt_id": "p3", + "CPS": 0.7561493333333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/epfl-llm/meditron-7b_0_SK.json b/e3c_llm_results/epfl-llm/meditron-7b_0_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..c1088a80244410d5643344fd84cb21aef85cbccd --- /dev/null +++ b/e3c_llm_results/epfl-llm/meditron-7b_0_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 4.508018, + "config": { + "model_name": "epfl-llm/meditron-7b", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SK", + "model": "epfl-llm/meditron-7b", + "base_model": "LlamaForCausalLM", + "revision": "d7d0a5ed929384a6b059ac74198cf1d71f44ba76", + "submitted_time": "2023-11-08 16:03:23+00:00", + "num_params_billion": 6.73855488, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 8.74, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 5.86, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 8.74, + "stderr": 0.0 + } + ], + "average_accuracy": 7.78, + "best_prompt": 8.74, + "prompt_id": "p1", + "CPS": 8.656096 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.36, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.31, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.36, + "stderr": 0.0 + } + ], + "average_accuracy": 0.3433333333333333, + "best_prompt": 0.36, + "prompt_id": "p1", + "CPS": 0.35994 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/epfl-llm/meditron-7b_0_SL.json b/e3c_llm_results/epfl-llm/meditron-7b_0_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..e14c8cd313fd6f7b95ea2e1c72a4de13c6fa791a --- /dev/null +++ b/e3c_llm_results/epfl-llm/meditron-7b_0_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 8.782022166666668, + "config": { + "model_name": "epfl-llm/meditron-7b", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SL", + "model": "epfl-llm/meditron-7b", + "base_model": "LlamaForCausalLM", + "revision": "d7d0a5ed929384a6b059ac74198cf1d71f44ba76", + "submitted_time": "2023-11-08 16:03:23+00:00", + "num_params_billion": 6.73855488, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 11.97, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 4.6, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 11.97, + "stderr": 0.0 + } + ], + "average_accuracy": 9.513333333333334, + "best_prompt": 11.97, + "prompt_id": "p1", + "CPS": 11.675937000000001 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 5.9799999999999995, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 1.37, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 5.9799999999999995, + "stderr": 0.0 + } + ], + "average_accuracy": 4.4433333333333325, + "best_prompt": 5.9799999999999995, + "prompt_id": "p1", + "CPS": 5.888107333333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/epfl-llm/meditron-7b_10_EN.json b/e3c_llm_results/epfl-llm/meditron-7b_10_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..34e9f8006e74510a3b49e80b95c96b5d8c8c1896 --- /dev/null +++ b/e3c_llm_results/epfl-llm/meditron-7b_10_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 10.821493833333335, + "config": { + "model_name": "epfl-llm/meditron-7b", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "EN", + "model": "epfl-llm/meditron-7b", + "base_model": "LlamaForCausalLM", + "revision": "d7d0a5ed929384a6b059ac74198cf1d71f44ba76", + "submitted_time": "2023-11-08 16:03:23+00:00", + "num_params_billion": 6.73855488, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 8.03, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 14.790000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 14.540000000000001, + "stderr": 0.0 + } + ], + "average_accuracy": 12.453333333333333, + "best_prompt": 14.790000000000001, + "prompt_id": "p2", + "CPS": 14.444407000000002 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 7.22, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 6.92, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 6.63, + "stderr": 0.0 + } + ], + "average_accuracy": 6.923333333333333, + "best_prompt": 7.22, + "prompt_id": "p1", + "CPS": 7.1985806666666665 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/epfl-llm/meditron-7b_10_GR.json b/e3c_llm_results/epfl-llm/meditron-7b_10_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..f4e964940f8370c4443037e6fc5a553f27de8810 --- /dev/null +++ b/e3c_llm_results/epfl-llm/meditron-7b_10_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 0.0, + "config": { + "model_name": "epfl-llm/meditron-7b", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "GR", + "model": "epfl-llm/meditron-7b", + "base_model": "LlamaForCausalLM", + "revision": "d7d0a5ed929384a6b059ac74198cf1d71f44ba76", + "submitted_time": "2023-11-08 16:03:23+00:00", + "num_params_billion": 6.73855488, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + } + ], + "average_accuracy": 0.0, + "best_prompt": 0.0, + "prompt_id": "p1", + "CPS": 0.0 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + } + ], + "average_accuracy": 0.0, + "best_prompt": 0.0, + "prompt_id": "p1", + "CPS": 0.0 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/epfl-llm/meditron-7b_10_IT.json b/e3c_llm_results/epfl-llm/meditron-7b_10_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..244cb48a61f03eb4ab7bb9983b4eff7e62f2398f --- /dev/null +++ b/e3c_llm_results/epfl-llm/meditron-7b_10_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 22.130671999999997, + "config": { + "model_name": "epfl-llm/meditron-7b", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "IT", + "model": "epfl-llm/meditron-7b", + "base_model": "LlamaForCausalLM", + "revision": "d7d0a5ed929384a6b059ac74198cf1d71f44ba76", + "submitted_time": "2023-11-08 16:03:23+00:00", + "num_params_billion": 6.73855488, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 29.909999999999997, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 35.63, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 33.11, + "stderr": 0.0 + } + ], + "average_accuracy": 32.88333333333333, + "best_prompt": 35.63, + "prompt_id": "p2", + "CPS": 34.651362666666664 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 8.32, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 8.870000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 9.68, + "stderr": 0.0 + } + ], + "average_accuracy": 8.956666666666667, + "best_prompt": 9.68, + "prompt_id": "p3", + "CPS": 9.609981333333334 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/epfl-llm/meditron-7b_10_PL.json b/e3c_llm_results/epfl-llm/meditron-7b_10_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..09d598b30fd86852c02b549323a6e7a0aea76051 --- /dev/null +++ b/e3c_llm_results/epfl-llm/meditron-7b_10_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 19.029036333333334, + "config": { + "model_name": "epfl-llm/meditron-7b", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "PL", + "model": "epfl-llm/meditron-7b", + "base_model": "LlamaForCausalLM", + "revision": "d7d0a5ed929384a6b059ac74198cf1d71f44ba76", + "submitted_time": "2023-11-08 16:03:23+00:00", + "num_params_billion": 6.73855488, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 31.840000000000003, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 32.97, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 31.840000000000003, + "stderr": 0.0 + } + ], + "average_accuracy": 32.21666666666667, + "best_prompt": 32.97, + "prompt_id": "p2", + "CPS": 32.721626 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 5.33, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 4.61, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 5.35, + "stderr": 0.0 + } + ], + "average_accuracy": 5.096666666666667, + "best_prompt": 5.35, + "prompt_id": "p3", + "CPS": 5.336446666666666 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/epfl-llm/meditron-7b_10_SK.json b/e3c_llm_results/epfl-llm/meditron-7b_10_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..cd967d4b173404407d135310fdf5de55bc864653 --- /dev/null +++ b/e3c_llm_results/epfl-llm/meditron-7b_10_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 17.218929, + "config": { + "model_name": "epfl-llm/meditron-7b", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SK", + "model": "epfl-llm/meditron-7b", + "base_model": "LlamaForCausalLM", + "revision": "d7d0a5ed929384a6b059ac74198cf1d71f44ba76", + "submitted_time": "2023-11-08 16:03:23+00:00", + "num_params_billion": 6.73855488, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 30.04, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 29.7, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 30.04, + "stderr": 0.0 + } + ], + "average_accuracy": 29.926666666666666, + "best_prompt": 30.04, + "prompt_id": "p1", + "CPS": 30.005954666666668 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 4.45, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 3.93, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 3.75, + "stderr": 0.0 + } + ], + "average_accuracy": 4.043333333333334, + "best_prompt": 4.45, + "prompt_id": "p1", + "CPS": 4.4319033333333335 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/epfl-llm/meditron-7b_10_SL.json b/e3c_llm_results/epfl-llm/meditron-7b_10_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..ec1bf7f96297f1f25a32b81c46c2521199f022f2 --- /dev/null +++ b/e3c_llm_results/epfl-llm/meditron-7b_10_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 18.122609833333332, + "config": { + "model_name": "epfl-llm/meditron-7b", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SL", + "model": "epfl-llm/meditron-7b", + "base_model": "LlamaForCausalLM", + "revision": "d7d0a5ed929384a6b059ac74198cf1d71f44ba76", + "submitted_time": "2023-11-08 16:03:23+00:00", + "num_params_billion": 6.73855488, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 31.19, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 29.160000000000004, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 31.19, + "stderr": 0.0 + } + ], + "average_accuracy": 30.513333333333335, + "best_prompt": 31.19, + "prompt_id": "p1", + "CPS": 30.978947666666667 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 4.77, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 5.01, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 5.28, + "stderr": 0.0 + } + ], + "average_accuracy": 5.02, + "best_prompt": 5.28, + "prompt_id": "p3", + "CPS": 5.266272 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-2-9b-it_0_EN.json b/e3c_llm_results/google/gemma-2-9b-it_0_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..7b5ea61ddf5254922257c58258b8a74bd284a13c --- /dev/null +++ b/e3c_llm_results/google/gemma-2-9b-it_0_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 46.266848333333336, + "config": { + "model_name": "google/gemma-2-9b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "EN", + "model": "google/gemma-2-9b-it", + "base_model": "Gemma2ForCausalLM", + "revision": "11c9b309abf73637e4b6f9a3fa1e92e615547819", + "submitted_time": "2024-06-24 08:05:41+00:00", + "num_params_billion": 9.241705984, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 32.67, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 51.739999999999995, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 53.7, + "stderr": 0.0 + } + ], + "average_accuracy": 46.03666666666667, + "best_prompt": 53.7, + "prompt_id": "p3", + "CPS": 49.584790000000005 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 43.6, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 42.05, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 40.67, + "stderr": 0.0 + } + ], + "average_accuracy": 42.10666666666666, + "best_prompt": 43.6, + "prompt_id": "p1", + "CPS": 42.948906666666666 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-2-9b-it_0_GR.json b/e3c_llm_results/google/gemma-2-9b-it_0_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..a3a49bcb2c37c253e9218a06801bd3fc58c37542 --- /dev/null +++ b/e3c_llm_results/google/gemma-2-9b-it_0_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 47.411836666666666, + "config": { + "model_name": "google/gemma-2-9b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "GR", + "model": "google/gemma-2-9b-it", + "base_model": "Gemma2ForCausalLM", + "revision": "11c9b309abf73637e4b6f9a3fa1e92e615547819", + "submitted_time": "2024-06-24 08:05:41+00:00", + "num_params_billion": 9.241705984, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 55.489999999999995, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 47.77, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 55.489999999999995, + "stderr": 0.0 + } + ], + "average_accuracy": 52.916666666666664, + "best_prompt": 55.489999999999995, + "prompt_id": "p1", + "CPS": 54.06205733333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 41.24, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 39.57, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 39.43, + "stderr": 0.0 + } + ], + "average_accuracy": 40.080000000000005, + "best_prompt": 41.24, + "prompt_id": "p1", + "CPS": 40.761616000000004 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-2-9b-it_0_IT.json b/e3c_llm_results/google/gemma-2-9b-it_0_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..64c3f144a8ebc59f5bc551b43992496a059f44b6 --- /dev/null +++ b/e3c_llm_results/google/gemma-2-9b-it_0_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 53.69207316666666, + "config": { + "model_name": "google/gemma-2-9b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "IT", + "model": "google/gemma-2-9b-it", + "base_model": "Gemma2ForCausalLM", + "revision": "11c9b309abf73637e4b6f9a3fa1e92e615547819", + "submitted_time": "2024-06-24 08:05:41+00:00", + "num_params_billion": 9.241705984, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 57.38999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 65.24, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 62.1, + "stderr": 0.0 + } + ], + "average_accuracy": 61.57666666666666, + "best_prompt": 65.24, + "prompt_id": "p2", + "CPS": 62.85004133333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 45.85, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 41.13, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 41.959999999999994, + "stderr": 0.0 + } + ], + "average_accuracy": 42.98, + "best_prompt": 45.85, + "prompt_id": "p1", + "CPS": 44.534105 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-2-9b-it_0_PL.json b/e3c_llm_results/google/gemma-2-9b-it_0_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..8c574103fa989c0e877cfe8ae56e3e9f973279ac --- /dev/null +++ b/e3c_llm_results/google/gemma-2-9b-it_0_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 41.18764683333333, + "config": { + "model_name": "google/gemma-2-9b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "PL", + "model": "google/gemma-2-9b-it", + "base_model": "Gemma2ForCausalLM", + "revision": "11c9b309abf73637e4b6f9a3fa1e92e615547819", + "submitted_time": "2024-06-24 08:05:41+00:00", + "num_params_billion": 9.241705984, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 40.6, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 41.55, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 40.6, + "stderr": 0.0 + } + ], + "average_accuracy": 40.916666666666664, + "best_prompt": 41.55, + "prompt_id": "p2", + "CPS": 41.28685 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 36.74, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 42.71, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 37.29, + "stderr": 0.0 + } + ], + "average_accuracy": 38.913333333333334, + "best_prompt": 42.71, + "prompt_id": "p2", + "CPS": 41.08844366666666 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-2-9b-it_0_SK.json b/e3c_llm_results/google/gemma-2-9b-it_0_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..68c81ec7fbe90d9da66b5c7b8e8e4e5d670c9b68 --- /dev/null +++ b/e3c_llm_results/google/gemma-2-9b-it_0_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 45.32347, + "config": { + "model_name": "google/gemma-2-9b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SK", + "model": "google/gemma-2-9b-it", + "base_model": "Gemma2ForCausalLM", + "revision": "11c9b309abf73637e4b6f9a3fa1e92e615547819", + "submitted_time": "2024-06-24 08:05:41+00:00", + "num_params_billion": 9.241705984, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 48.75, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 45.75, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 48.75, + "stderr": 0.0 + } + ], + "average_accuracy": 47.75, + "best_prompt": 48.75, + "prompt_id": "p1", + "CPS": 48.2625 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 39.89, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 43.4, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 39.89, + "stderr": 0.0 + } + ], + "average_accuracy": 41.06, + "best_prompt": 43.4, + "prompt_id": "p2", + "CPS": 42.38444 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-2-9b-it_0_SL.json b/e3c_llm_results/google/gemma-2-9b-it_0_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..293853a75701a2b24ce483ca57b4b063fe2a7b39 --- /dev/null +++ b/e3c_llm_results/google/gemma-2-9b-it_0_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 43.368616, + "config": { + "model_name": "google/gemma-2-9b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SL", + "model": "google/gemma-2-9b-it", + "base_model": "Gemma2ForCausalLM", + "revision": "11c9b309abf73637e4b6f9a3fa1e92e615547819", + "submitted_time": "2024-06-24 08:05:41+00:00", + "num_params_billion": 9.241705984, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 47.07, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 40.46, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 47.07, + "stderr": 0.0 + } + ], + "average_accuracy": 44.86666666666667, + "best_prompt": 47.07, + "prompt_id": "p1", + "CPS": 46.032891 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 40.79, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 40.160000000000004, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 40.79, + "stderr": 0.0 + } + ], + "average_accuracy": 40.580000000000005, + "best_prompt": 40.79, + "prompt_id": "p1", + "CPS": 40.704341 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-2-9b-it_10_EN.json b/e3c_llm_results/google/gemma-2-9b-it_10_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..82bee9cdb274254c7284f05a5ae03969262a3588 --- /dev/null +++ b/e3c_llm_results/google/gemma-2-9b-it_10_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 56.887223, + "config": { + "model_name": "google/gemma-2-9b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "EN", + "model": "google/gemma-2-9b-it", + "base_model": "Gemma2ForCausalLM", + "revision": "11c9b309abf73637e4b6f9a3fa1e92e615547819", + "submitted_time": "2024-06-24 08:05:41+00:00", + "num_params_billion": 9.241705984, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 62.0, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 56.38999999999999, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 59.18, + "stderr": 0.0 + } + ], + "average_accuracy": 59.19, + "best_prompt": 62.0, + "prompt_id": "p1", + "CPS": 60.257799999999996 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.629999999999995, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 53.37, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 54.09, + "stderr": 0.0 + } + ], + "average_accuracy": 53.03, + "best_prompt": 54.09, + "prompt_id": "p3", + "CPS": 53.516646 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-2-9b-it_10_GR.json b/e3c_llm_results/google/gemma-2-9b-it_10_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..0e2098f70ae078b548e5f599d56bc7e7e39b083d --- /dev/null +++ b/e3c_llm_results/google/gemma-2-9b-it_10_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 56.44067866666666, + "config": { + "model_name": "google/gemma-2-9b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "GR", + "model": "google/gemma-2-9b-it", + "base_model": "Gemma2ForCausalLM", + "revision": "11c9b309abf73637e4b6f9a3fa1e92e615547819", + "submitted_time": "2024-06-24 08:05:41+00:00", + "num_params_billion": 9.241705984, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 60.83, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 56.63, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 60.83, + "stderr": 0.0 + } + ], + "average_accuracy": 59.43, + "best_prompt": 60.83, + "prompt_id": "p1", + "CPS": 59.978379999999994 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 50.7, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 49.71, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 54.44, + "stderr": 0.0 + } + ], + "average_accuracy": 51.61666666666667, + "best_prompt": 54.44, + "prompt_id": "p3", + "CPS": 52.90297733333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-2-9b-it_10_IT.json b/e3c_llm_results/google/gemma-2-9b-it_10_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..64051e3c760740b47f74e2fc169fed984b21a83c --- /dev/null +++ b/e3c_llm_results/google/gemma-2-9b-it_10_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 60.25721083333333, + "config": { + "model_name": "google/gemma-2-9b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "IT", + "model": "google/gemma-2-9b-it", + "base_model": "Gemma2ForCausalLM", + "revision": "11c9b309abf73637e4b6f9a3fa1e92e615547819", + "submitted_time": "2024-06-24 08:05:41+00:00", + "num_params_billion": 9.241705984, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 69.1, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 66.43, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 65.69, + "stderr": 0.0 + } + ], + "average_accuracy": 67.07333333333334, + "best_prompt": 69.1, + "prompt_id": "p1", + "CPS": 67.69957333333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 49.58, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 53.65, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 53.05, + "stderr": 0.0 + } + ], + "average_accuracy": 52.093333333333334, + "best_prompt": 53.65, + "prompt_id": "p2", + "CPS": 52.81484833333334 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-2-9b-it_10_PL.json b/e3c_llm_results/google/gemma-2-9b-it_10_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..f90e29dbb87b8f82e390fa684d0c6094d0f9a559 --- /dev/null +++ b/e3c_llm_results/google/gemma-2-9b-it_10_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 54.98672666666667, + "config": { + "model_name": "google/gemma-2-9b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "PL", + "model": "google/gemma-2-9b-it", + "base_model": "Gemma2ForCausalLM", + "revision": "11c9b309abf73637e4b6f9a3fa1e92e615547819", + "submitted_time": "2024-06-24 08:05:41+00:00", + "num_params_billion": 9.241705984, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 59.08, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 58.620000000000005, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 59.08, + "stderr": 0.0 + } + ], + "average_accuracy": 58.92666666666667, + "best_prompt": 59.08, + "prompt_id": "p1", + "CPS": 58.98941066666667 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.68000000000001, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 48.08, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 51.239999999999995, + "stderr": 0.0 + } + ], + "average_accuracy": 50.333333333333336, + "best_prompt": 51.68000000000001, + "prompt_id": "p1", + "CPS": 50.98404266666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-2-9b-it_10_SK.json b/e3c_llm_results/google/gemma-2-9b-it_10_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..40670e1d69af82c8078e42c74eba38fd62ee83b9 --- /dev/null +++ b/e3c_llm_results/google/gemma-2-9b-it_10_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 56.074384499999994, + "config": { + "model_name": "google/gemma-2-9b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SK", + "model": "google/gemma-2-9b-it", + "base_model": "Gemma2ForCausalLM", + "revision": "11c9b309abf73637e4b6f9a3fa1e92e615547819", + "submitted_time": "2024-06-24 08:05:41+00:00", + "num_params_billion": 9.241705984, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 61.41, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 61.22, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 61.41, + "stderr": 0.0 + } + ], + "average_accuracy": 61.346666666666664, + "best_prompt": 61.41, + "prompt_id": "p1", + "CPS": 61.371106999999995 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.53, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 47.54, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 51.13999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 50.06999999999999, + "best_prompt": 51.53, + "prompt_id": "p1", + "CPS": 50.777662 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-2-9b-it_10_SL.json b/e3c_llm_results/google/gemma-2-9b-it_10_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..1d426183767a1511dcbef244831d32a4b3a2e57d --- /dev/null +++ b/e3c_llm_results/google/gemma-2-9b-it_10_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 55.7992, + "config": { + "model_name": "google/gemma-2-9b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SL", + "model": "google/gemma-2-9b-it", + "base_model": "Gemma2ForCausalLM", + "revision": "11c9b309abf73637e4b6f9a3fa1e92e615547819", + "submitted_time": "2024-06-24 08:05:41+00:00", + "num_params_billion": 9.241705984, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 63.65, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 57.37, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 63.65, + "stderr": 0.0 + } + ], + "average_accuracy": 61.556666666666665, + "best_prompt": 63.65, + "prompt_id": "p1", + "CPS": 62.31759333333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 48.010000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 48.78, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 49.72, + "stderr": 0.0 + } + ], + "average_accuracy": 48.836666666666666, + "best_prompt": 49.72, + "prompt_id": "p3", + "CPS": 49.28080666666666 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-3-27b-it_0_EN.json b/e3c_llm_results/google/gemma-3-27b-it_0_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..af7286d3941ae9d09182310991fcf45796a7a2ab --- /dev/null +++ b/e3c_llm_results/google/gemma-3-27b-it_0_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 51.5885295, + "config": { + "model_name": "google/gemma-3-27b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "EN", + "model": "google/gemma-3-27b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "005ad3404e59d6023443cb575daa05336842228a", + "submitted_time": "2025-03-01 19:10:19+00:00", + "num_params_billion": 27.43240664, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 54.459999999999994, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 58.3, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 51.94, + "stderr": 0.0 + } + ], + "average_accuracy": 54.9, + "best_prompt": 58.3, + "prompt_id": "p2", + "CPS": 56.3178 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 45.43, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 45.82, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 47.43, + "stderr": 0.0 + } + ], + "average_accuracy": 46.22666666666667, + "best_prompt": 47.43, + "prompt_id": "p3", + "CPS": 46.859259 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-3-27b-it_0_GR.json b/e3c_llm_results/google/gemma-3-27b-it_0_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..57f5fce3e48473d1150218b62964e4d9a27b9c69 --- /dev/null +++ b/e3c_llm_results/google/gemma-3-27b-it_0_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 50.113703, + "config": { + "model_name": "google/gemma-3-27b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "GR", + "model": "google/gemma-3-27b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "005ad3404e59d6023443cb575daa05336842228a", + "submitted_time": "2025-03-01 19:10:19+00:00", + "num_params_billion": 27.43240664, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 48.66, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 57.21000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 48.66, + "stderr": 0.0 + } + ], + "average_accuracy": 51.51, + "best_prompt": 57.21000000000001, + "prompt_id": "p2", + "CPS": 53.94903000000001 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 39.550000000000004, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 46.949999999999996, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 47.69, + "stderr": 0.0 + } + ], + "average_accuracy": 44.73, + "best_prompt": 47.69, + "prompt_id": "p3", + "CPS": 46.278376 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-3-27b-it_0_IT.json b/e3c_llm_results/google/gemma-3-27b-it_0_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..dd110f42cc6d62942b3a0db4684a57a2bb96e159 --- /dev/null +++ b/e3c_llm_results/google/gemma-3-27b-it_0_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 55.535388, + "config": { + "model_name": "google/gemma-3-27b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "IT", + "model": "google/gemma-3-27b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "005ad3404e59d6023443cb575daa05336842228a", + "submitted_time": "2025-03-01 19:10:19+00:00", + "num_params_billion": 27.43240664, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 55.43, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 66.97, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 59.540000000000006, + "stderr": 0.0 + } + ], + "average_accuracy": 60.64666666666667, + "best_prompt": 66.97, + "prompt_id": "p2", + "CPS": 62.73526366666667 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 43.9, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 48.949999999999996, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 49.27, + "stderr": 0.0 + } + ], + "average_accuracy": 47.373333333333335, + "best_prompt": 49.27, + "prompt_id": "p3", + "CPS": 48.335512333333334 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-3-27b-it_0_PL.json b/e3c_llm_results/google/gemma-3-27b-it_0_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..7b0569f8e12ef06b8b2a03c6a3320f0624580425 --- /dev/null +++ b/e3c_llm_results/google/gemma-3-27b-it_0_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 44.29942833333334, + "config": { + "model_name": "google/gemma-3-27b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "PL", + "model": "google/gemma-3-27b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "005ad3404e59d6023443cb575daa05336842228a", + "submitted_time": "2025-03-01 19:10:19+00:00", + "num_params_billion": 27.43240664, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 45.06, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 45.11, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 45.06, + "stderr": 0.0 + } + ], + "average_accuracy": 45.076666666666675, + "best_prompt": 45.11, + "prompt_id": "p2", + "CPS": 45.09496333333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 43.84, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 42.67, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 42.71, + "stderr": 0.0 + } + ], + "average_accuracy": 43.07333333333333, + "best_prompt": 43.84, + "prompt_id": "p1", + "CPS": 43.50389333333334 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-3-27b-it_0_SK.json b/e3c_llm_results/google/gemma-3-27b-it_0_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..e06cfef6bc90db6e038c33f31af47a8a8b26853b --- /dev/null +++ b/e3c_llm_results/google/gemma-3-27b-it_0_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 37.22623216666667, + "config": { + "model_name": "google/gemma-3-27b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SK", + "model": "google/gemma-3-27b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "005ad3404e59d6023443cb575daa05336842228a", + "submitted_time": "2025-03-01 19:10:19+00:00", + "num_params_billion": 27.43240664, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 31.830000000000002, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 21.57, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 31.830000000000002, + "stderr": 0.0 + } + ], + "average_accuracy": 28.41, + "best_prompt": 31.830000000000002, + "prompt_id": "p1", + "CPS": 30.741414000000002 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 43.730000000000004, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 43.6, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 43.730000000000004, + "stderr": 0.0 + } + ], + "average_accuracy": 43.68666666666667, + "best_prompt": 43.730000000000004, + "prompt_id": "p1", + "CPS": 43.71105033333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-3-27b-it_0_SL.json b/e3c_llm_results/google/gemma-3-27b-it_0_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..9d3ffdae1b80b261b72c163b99fa145ed7818d28 --- /dev/null +++ b/e3c_llm_results/google/gemma-3-27b-it_0_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 45.01248166666667, + "config": { + "model_name": "google/gemma-3-27b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SL", + "model": "google/gemma-3-27b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "005ad3404e59d6023443cb575daa05336842228a", + "submitted_time": "2025-03-01 19:10:19+00:00", + "num_params_billion": 27.43240664, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 43.7, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 47.83, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 43.7, + "stderr": 0.0 + } + ], + "average_accuracy": 45.076666666666675, + "best_prompt": 47.83, + "prompt_id": "p2", + "CPS": 46.513080666666674 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 42.55, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 43.91, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 42.55, + "stderr": 0.0 + } + ], + "average_accuracy": 43.00333333333333, + "best_prompt": 43.91, + "prompt_id": "p2", + "CPS": 43.511882666666665 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-3-27b-it_10_EN.json b/e3c_llm_results/google/gemma-3-27b-it_10_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..903c6cb300673438b7555124cb15fe2b7e0e795f --- /dev/null +++ b/e3c_llm_results/google/gemma-3-27b-it_10_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 59.271406000000006, + "config": { + "model_name": "google/gemma-3-27b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "EN", + "model": "google/gemma-3-27b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "005ad3404e59d6023443cb575daa05336842228a", + "submitted_time": "2025-03-01 19:10:19+00:00", + "num_params_billion": 27.43240664, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 61.6, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 63.080000000000005, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 60.940000000000005, + "stderr": 0.0 + } + ], + "average_accuracy": 61.873333333333335, + "best_prompt": 63.080000000000005, + "prompt_id": "p2", + "CPS": 62.318834666666675 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.910000000000004, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 56.00000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 57.64, + "stderr": 0.0 + } + ], + "average_accuracy": 55.18333333333334, + "best_prompt": 57.64, + "prompt_id": "p3", + "CPS": 56.22397733333334 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-3-27b-it_10_GR.json b/e3c_llm_results/google/gemma-3-27b-it_10_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..fe1b22dab57d0df99b5b5323e5b5fda6b4bcdbbc --- /dev/null +++ b/e3c_llm_results/google/gemma-3-27b-it_10_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 60.327389833333335, + "config": { + "model_name": "google/gemma-3-27b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "GR", + "model": "google/gemma-3-27b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "005ad3404e59d6023443cb575daa05336842228a", + "submitted_time": "2025-03-01 19:10:19+00:00", + "num_params_billion": 27.43240664, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 65.51, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 66.08000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 65.51, + "stderr": 0.0 + } + ], + "average_accuracy": 65.7, + "best_prompt": 66.08000000000001, + "prompt_id": "p2", + "CPS": 65.828896 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 50.83, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 55.50000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 55.81, + "stderr": 0.0 + } + ], + "average_accuracy": 54.046666666666674, + "best_prompt": 55.81, + "prompt_id": "p3", + "CPS": 54.82588366666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-3-27b-it_10_IT.json b/e3c_llm_results/google/gemma-3-27b-it_10_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..1bc2570adab2f3f2d327d8254b5bd03aa0a0f0de --- /dev/null +++ b/e3c_llm_results/google/gemma-3-27b-it_10_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 64.24948583333332, + "config": { + "model_name": "google/gemma-3-27b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "IT", + "model": "google/gemma-3-27b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "005ad3404e59d6023443cb575daa05336842228a", + "submitted_time": "2025-03-01 19:10:19+00:00", + "num_params_billion": 27.43240664, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 71.41999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 69.92, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 72.11999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 71.15333333333332, + "best_prompt": 72.11999999999999, + "prompt_id": "p3", + "CPS": 71.42284 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 52.23, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 58.37, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 57.86, + "stderr": 0.0 + } + ], + "average_accuracy": 56.15333333333333, + "best_prompt": 58.37, + "prompt_id": "p2", + "CPS": 57.07613166666666 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-3-27b-it_10_PL.json b/e3c_llm_results/google/gemma-3-27b-it_10_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..8de0837499b10ebd8ec72ea7bc81beaa038d1983 --- /dev/null +++ b/e3c_llm_results/google/gemma-3-27b-it_10_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 61.5666635, + "config": { + "model_name": "google/gemma-3-27b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "PL", + "model": "google/gemma-3-27b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "005ad3404e59d6023443cb575daa05336842228a", + "submitted_time": "2025-03-01 19:10:19+00:00", + "num_params_billion": 27.43240664, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 65.91, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 66.72, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 65.91, + "stderr": 0.0 + } + ], + "average_accuracy": 66.17999999999999, + "best_prompt": 66.72, + "prompt_id": "p2", + "CPS": 66.35971199999999 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 57.95, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 56.010000000000005, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 53.800000000000004, + "stderr": 0.0 + } + ], + "average_accuracy": 55.92000000000001, + "best_prompt": 57.95, + "prompt_id": "p1", + "CPS": 56.77361500000001 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-3-27b-it_10_SK.json b/e3c_llm_results/google/gemma-3-27b-it_10_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..cc28cf5961280cd567f4368f68a587617b7be08c --- /dev/null +++ b/e3c_llm_results/google/gemma-3-27b-it_10_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 59.623766999999994, + "config": { + "model_name": "google/gemma-3-27b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SK", + "model": "google/gemma-3-27b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "005ad3404e59d6023443cb575daa05336842228a", + "submitted_time": "2025-03-01 19:10:19+00:00", + "num_params_billion": 27.43240664, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 67.36999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 68.85, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 67.36999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 67.86333333333333, + "best_prompt": 68.85, + "prompt_id": "p2", + "CPS": 68.17067999999999 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.21, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 50.61, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 51.03, + "stderr": 0.0 + } + ], + "average_accuracy": 50.949999999999996, + "best_prompt": 51.21, + "prompt_id": "p1", + "CPS": 51.076854 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/gemma-3-27b-it_10_SL.json b/e3c_llm_results/google/gemma-3-27b-it_10_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..37a77ebd450c4aabb1937362c0020ecdc0c132dc --- /dev/null +++ b/e3c_llm_results/google/gemma-3-27b-it_10_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 59.561417000000006, + "config": { + "model_name": "google/gemma-3-27b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SL", + "model": "google/gemma-3-27b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "005ad3404e59d6023443cb575daa05336842228a", + "submitted_time": "2025-03-01 19:10:19+00:00", + "num_params_billion": 27.43240664, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 67.5, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 69.17999999999999, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 67.5, + "stderr": 0.0 + } + ], + "average_accuracy": 68.06, + "best_prompt": 69.17999999999999, + "prompt_id": "p2", + "CPS": 68.405184 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.49, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 47.03, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 51.449999999999996, + "stderr": 0.0 + } + ], + "average_accuracy": 49.99, + "best_prompt": 51.49, + "prompt_id": "p1", + "CPS": 50.71765 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-27b-text-it_0_EN.json b/e3c_llm_results/google/medgemma-27b-text-it_0_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..ff6e165be2cc649edeaf18aa977f6b6696c1b862 --- /dev/null +++ b/e3c_llm_results/google/medgemma-27b-text-it_0_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 50.889483999999996, + "config": { + "model_name": "google/medgemma-27b-text-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "EN", + "model": "google/medgemma-27b-text-it", + "base_model": "Gemma3ForCausalLM", + "revision": "6b08c481126ff65a9b8fa5ab4d691b152b8edb5d", + "submitted_time": "2025-05-19 20:53:04+00:00", + "num_params_billion": 27.00900224, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 38.42, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 60.35, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 51.559999999999995, + "stderr": 0.0 + } + ], + "average_accuracy": 50.10999999999999, + "best_prompt": 60.35, + "prompt_id": "p2", + "CPS": 54.170159999999996 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 48.36, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 47.63, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 44.43, + "stderr": 0.0 + } + ], + "average_accuracy": 46.80666666666667, + "best_prompt": 48.36, + "prompt_id": "p1", + "CPS": 47.608808 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-27b-text-it_0_GR.json b/e3c_llm_results/google/medgemma-27b-text-it_0_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..6d2c404a27a9347989f54f4ac0833bdebd120aee --- /dev/null +++ b/e3c_llm_results/google/medgemma-27b-text-it_0_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 50.34454, + "config": { + "model_name": "google/medgemma-27b-text-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "GR", + "model": "google/medgemma-27b-text-it", + "base_model": "Gemma3ForCausalLM", + "revision": "6b08c481126ff65a9b8fa5ab4d691b152b8edb5d", + "submitted_time": "2025-05-19 20:53:04+00:00", + "num_params_billion": 27.00900224, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 53.14, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 61.260000000000005, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 53.14, + "stderr": 0.0 + } + ], + "average_accuracy": 55.84666666666667, + "best_prompt": 61.260000000000005, + "prompt_id": "p2", + "CPS": 57.943792 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 40.69, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 43.32, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 41.97, + "stderr": 0.0 + } + ], + "average_accuracy": 41.99333333333333, + "best_prompt": 43.32, + "prompt_id": "p2", + "CPS": 42.745288 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-27b-text-it_0_IT.json b/e3c_llm_results/google/medgemma-27b-text-it_0_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..ba2d095e7a6e810b9ca77a6a77ddd42bff04b9ad --- /dev/null +++ b/e3c_llm_results/google/medgemma-27b-text-it_0_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 51.99607533333334, + "config": { + "model_name": "google/medgemma-27b-text-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "IT", + "model": "google/medgemma-27b-text-it", + "base_model": "Gemma3ForCausalLM", + "revision": "6b08c481126ff65a9b8fa5ab4d691b152b8edb5d", + "submitted_time": "2025-05-19 20:53:04+00:00", + "num_params_billion": 27.00900224, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 42.61, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 62.12, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 55.82, + "stderr": 0.0 + } + ], + "average_accuracy": 53.51666666666667, + "best_prompt": 62.12, + "prompt_id": "p2", + "CPS": 56.77560933333334 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 40.42, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 49.16, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 46.04, + "stderr": 0.0 + } + ], + "average_accuracy": 45.20666666666667, + "best_prompt": 49.16, + "prompt_id": "p2", + "CPS": 47.21654133333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-27b-text-it_0_PL.json b/e3c_llm_results/google/medgemma-27b-text-it_0_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..1f30f15180aaafee89b21659d43aaad0a2f00407 --- /dev/null +++ b/e3c_llm_results/google/medgemma-27b-text-it_0_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 43.305971666666665, + "config": { + "model_name": "google/medgemma-27b-text-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "PL", + "model": "google/medgemma-27b-text-it", + "base_model": "Gemma3ForCausalLM", + "revision": "6b08c481126ff65a9b8fa5ab4d691b152b8edb5d", + "submitted_time": "2025-05-19 20:53:04+00:00", + "num_params_billion": 27.00900224, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 42.16, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 43.03, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 42.16, + "stderr": 0.0 + } + ], + "average_accuracy": 42.449999999999996, + "best_prompt": 43.03, + "prompt_id": "p2", + "CPS": 42.780426 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 43.25, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 44.24, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 42.46, + "stderr": 0.0 + } + ], + "average_accuracy": 43.31666666666666, + "best_prompt": 44.24, + "prompt_id": "p2", + "CPS": 43.83151733333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-27b-text-it_0_SK.json b/e3c_llm_results/google/medgemma-27b-text-it_0_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..da892b37ce4b87a7741604689fae122e3e7802d3 --- /dev/null +++ b/e3c_llm_results/google/medgemma-27b-text-it_0_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 36.36130216666666, + "config": { + "model_name": "google/medgemma-27b-text-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SK", + "model": "google/medgemma-27b-text-it", + "base_model": "Gemma3ForCausalLM", + "revision": "6b08c481126ff65a9b8fa5ab4d691b152b8edb5d", + "submitted_time": "2025-05-19 20:53:04+00:00", + "num_params_billion": 27.00900224, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 29.709999999999997, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 10.66, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 29.709999999999997, + "stderr": 0.0 + } + ], + "average_accuracy": 23.36, + "best_prompt": 29.709999999999997, + "prompt_id": "p1", + "CPS": 27.823414999999997 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 43.95, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 45.31, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 43.95, + "stderr": 0.0 + } + ], + "average_accuracy": 44.403333333333336, + "best_prompt": 45.31, + "prompt_id": "p2", + "CPS": 44.89918933333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-27b-text-it_0_SL.json b/e3c_llm_results/google/medgemma-27b-text-it_0_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..15a0929f20159962ee110f91f9dcfcff66cae370 --- /dev/null +++ b/e3c_llm_results/google/medgemma-27b-text-it_0_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 46.321461, + "config": { + "model_name": "google/medgemma-27b-text-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SL", + "model": "google/medgemma-27b-text-it", + "base_model": "Gemma3ForCausalLM", + "revision": "6b08c481126ff65a9b8fa5ab4d691b152b8edb5d", + "submitted_time": "2025-05-19 20:53:04+00:00", + "num_params_billion": 27.00900224, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 46.75, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 52.38, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 46.75, + "stderr": 0.0 + } + ], + "average_accuracy": 48.626666666666665, + "best_prompt": 52.38, + "prompt_id": "p2", + "CPS": 50.414004 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 41.82, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 42.39, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 41.82, + "stderr": 0.0 + } + ], + "average_accuracy": 42.01, + "best_prompt": 42.39, + "prompt_id": "p2", + "CPS": 42.228918 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-27b-text-it_10_EN.json b/e3c_llm_results/google/medgemma-27b-text-it_10_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..f17f4bcfaa89a0ccec0e26eb196cc71f46930348 --- /dev/null +++ b/e3c_llm_results/google/medgemma-27b-text-it_10_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 59.60748666666666, + "config": { + "model_name": "google/medgemma-27b-text-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "EN", + "model": "google/medgemma-27b-text-it", + "base_model": "Gemma3ForCausalLM", + "revision": "6b08c481126ff65a9b8fa5ab4d691b152b8edb5d", + "submitted_time": "2025-05-19 20:53:04+00:00", + "num_params_billion": 27.00900224, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 63.55, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 61.61, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 64.55, + "stderr": 0.0 + } + ], + "average_accuracy": 63.23666666666666, + "best_prompt": 64.55, + "prompt_id": "p3", + "CPS": 63.70224333333332 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 55.620000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 54.94, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 55.65, + "stderr": 0.0 + } + ], + "average_accuracy": 55.403333333333336, + "best_prompt": 55.65, + "prompt_id": "p3", + "CPS": 55.512730000000005 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-27b-text-it_10_GR.json b/e3c_llm_results/google/medgemma-27b-text-it_10_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..862506ffca374fa15a05b2c7383529a42f4480da --- /dev/null +++ b/e3c_llm_results/google/medgemma-27b-text-it_10_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 62.99263766666667, + "config": { + "model_name": "google/medgemma-27b-text-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "GR", + "model": "google/medgemma-27b-text-it", + "base_model": "Gemma3ForCausalLM", + "revision": "6b08c481126ff65a9b8fa5ab4d691b152b8edb5d", + "submitted_time": "2025-05-19 20:53:04+00:00", + "num_params_billion": 27.00900224, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 68.36, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 68.46, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 68.36, + "stderr": 0.0 + } + ], + "average_accuracy": 68.39333333333333, + "best_prompt": 68.46, + "prompt_id": "p2", + "CPS": 68.41436 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 53.92, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 58.67, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 57.8, + "stderr": 0.0 + } + ], + "average_accuracy": 56.79666666666666, + "best_prompt": 58.67, + "prompt_id": "p2", + "CPS": 57.57091533333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-27b-text-it_10_IT.json b/e3c_llm_results/google/medgemma-27b-text-it_10_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..4df93fec54df7c07e05c799c2bde973a7e48bf8e --- /dev/null +++ b/e3c_llm_results/google/medgemma-27b-text-it_10_IT.json @@ -0,0 +1,63 @@ +{ + "average_CPS": 66.161104, + "config": { + "model_name": "google/medgemma-27b-text-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "IT", + "model": "google/medgemma-27b-text-it", + "base_model": "Gemma3ForCausalLM", + "revision": "6b08c481126ff65a9b8fa5ab4d691b152b8edb5d", + "submitted_time": "2025-05-19 20:53:04+00:00", + "num_params_billion": 27.00900224, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 72.61999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 70.05, + "stderr": 0.0 + } + ], + "average_accuracy": 71.335, + "best_prompt": 72.61999999999999, + "prompt_id": "p1", + "CPS": 71.686833 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 59.19, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 62.35000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 57.26, + "stderr": 0.0 + } + ], + "average_accuracy": 59.6, + "best_prompt": 62.35000000000001, + "prompt_id": "p2", + "CPS": 60.635375 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-27b-text-it_10_PL.json b/e3c_llm_results/google/medgemma-27b-text-it_10_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..687310ea6f0b55027aaed4f5bca0fa501be783b9 --- /dev/null +++ b/e3c_llm_results/google/medgemma-27b-text-it_10_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 64.263205, + "config": { + "model_name": "google/medgemma-27b-text-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "PL", + "model": "google/medgemma-27b-text-it", + "base_model": "Gemma3ForCausalLM", + "revision": "6b08c481126ff65a9b8fa5ab4d691b152b8edb5d", + "submitted_time": "2025-05-19 20:53:04+00:00", + "num_params_billion": 27.00900224, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 68.28999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 67.15, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 68.28999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 67.91, + "best_prompt": 68.28999999999999, + "prompt_id": "p1", + "CPS": 68.030498 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 59.4, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 61.33, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 59.18, + "stderr": 0.0 + } + ], + "average_accuracy": 59.97, + "best_prompt": 61.33, + "prompt_id": "p2", + "CPS": 60.495912000000004 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-27b-text-it_10_SK.json b/e3c_llm_results/google/medgemma-27b-text-it_10_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..b616f992aa614e8edce277364ba149d35bf96e45 --- /dev/null +++ b/e3c_llm_results/google/medgemma-27b-text-it_10_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 61.55467333333333, + "config": { + "model_name": "google/medgemma-27b-text-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SK", + "model": "google/medgemma-27b-text-it", + "base_model": "Gemma3ForCausalLM", + "revision": "6b08c481126ff65a9b8fa5ab4d691b152b8edb5d", + "submitted_time": "2025-05-19 20:53:04+00:00", + "num_params_billion": 27.00900224, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 71.43, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 71.27, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 71.43, + "stderr": 0.0 + } + ], + "average_accuracy": 71.37666666666667, + "best_prompt": 71.43, + "prompt_id": "p1", + "CPS": 71.391904 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.11, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 51.88, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 51.71, + "stderr": 0.0 + } + ], + "average_accuracy": 51.56666666666666, + "best_prompt": 51.88, + "prompt_id": "p2", + "CPS": 51.71744266666666 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-27b-text-it_10_SL.json b/e3c_llm_results/google/medgemma-27b-text-it_10_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..4024c2b5064f2c6fc30e552dd613b311e8510d90 --- /dev/null +++ b/e3c_llm_results/google/medgemma-27b-text-it_10_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 62.13607933333333, + "config": { + "model_name": "google/medgemma-27b-text-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SL", + "model": "google/medgemma-27b-text-it", + "base_model": "Gemma3ForCausalLM", + "revision": "6b08c481126ff65a9b8fa5ab4d691b152b8edb5d", + "submitted_time": "2025-05-19 20:53:04+00:00", + "num_params_billion": 27.00900224, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 69.47, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 67.65, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 69.47, + "stderr": 0.0 + } + ], + "average_accuracy": 68.86333333333333, + "best_prompt": 69.47, + "prompt_id": "p1", + "CPS": 69.04854866666666 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 53.23, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 55.900000000000006, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 54.94, + "stderr": 0.0 + } + ], + "average_accuracy": 54.69, + "best_prompt": 55.900000000000006, + "prompt_id": "p2", + "CPS": 55.22361 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-4b-it_0_EN.json b/e3c_llm_results/google/medgemma-4b-it_0_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..df88f5c4b07c29143e6dbdefaee00b4d2bf91a77 --- /dev/null +++ b/e3c_llm_results/google/medgemma-4b-it_0_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 29.15664266666667, + "config": { + "model_name": "google/medgemma-4b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "EN", + "model": "google/medgemma-4b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "efe6cc02361759b6bd501c654ddb7c9d25ec509d", + "submitted_time": "2025-05-19 20:52:44+00:00", + "num_params_billion": 4.300079472, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 26.35, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 25.03, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 27.37, + "stderr": 0.0 + } + ], + "average_accuracy": 26.25, + "best_prompt": 27.37, + "prompt_id": "p3", + "CPS": 27.063456000000002 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 20.95, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 32.57, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 32.029999999999994, + "stderr": 0.0 + } + ], + "average_accuracy": 28.516666666666666, + "best_prompt": 32.57, + "prompt_id": "p2", + "CPS": 31.249829333333334 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-4b-it_0_GR.json b/e3c_llm_results/google/medgemma-4b-it_0_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..ca6fd8573b8fc354c557b566854465ce92dcbf07 --- /dev/null +++ b/e3c_llm_results/google/medgemma-4b-it_0_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 27.1538555, + "config": { + "model_name": "google/medgemma-4b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "GR", + "model": "google/medgemma-4b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "efe6cc02361759b6bd501c654ddb7c9d25ec509d", + "submitted_time": "2025-05-19 20:52:44+00:00", + "num_params_billion": 4.300079472, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 27.05, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 26.540000000000003, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 27.05, + "stderr": 0.0 + } + ], + "average_accuracy": 26.88, + "best_prompt": 27.05, + "prompt_id": "p1", + "CPS": 27.004015 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 23.810000000000002, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 30.240000000000002, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 7.539999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 20.53, + "best_prompt": 30.240000000000002, + "prompt_id": "p2", + "CPS": 27.303696000000002 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-4b-it_0_IT.json b/e3c_llm_results/google/medgemma-4b-it_0_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..514ce069415cac763ca4a1d6328758c6359ad305 --- /dev/null +++ b/e3c_llm_results/google/medgemma-4b-it_0_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 26.877481500000002, + "config": { + "model_name": "google/medgemma-4b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "IT", + "model": "google/medgemma-4b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "efe6cc02361759b6bd501c654ddb7c9d25ec509d", + "submitted_time": "2025-05-19 20:52:44+00:00", + "num_params_billion": 4.300079472, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 31.569999999999997, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 26.27, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 30.04, + "stderr": 0.0 + } + ], + "average_accuracy": 29.293333333333333, + "best_prompt": 31.569999999999997, + "prompt_id": "p1", + "CPS": 30.851256333333332 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 21.54, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 24.610000000000003, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 6.88, + "stderr": 0.0 + } + ], + "average_accuracy": 17.676666666666666, + "best_prompt": 24.610000000000003, + "prompt_id": "p2", + "CPS": 22.903706666666668 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-4b-it_0_PL.json b/e3c_llm_results/google/medgemma-4b-it_0_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..c4fccc04fc74bd7bc0eb9feea16aea685c9ce998 --- /dev/null +++ b/e3c_llm_results/google/medgemma-4b-it_0_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 17.725084000000003, + "config": { + "model_name": "google/medgemma-4b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "PL", + "model": "google/medgemma-4b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "efe6cc02361759b6bd501c654ddb7c9d25ec509d", + "submitted_time": "2025-05-19 20:52:44+00:00", + "num_params_billion": 4.300079472, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 22.55, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 21.83, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 22.55, + "stderr": 0.0 + } + ], + "average_accuracy": 22.310000000000002, + "best_prompt": 22.55, + "prompt_id": "p1", + "CPS": 22.495880000000003 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 11.5, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 13.139999999999999, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 10.54, + "stderr": 0.0 + } + ], + "average_accuracy": 11.726666666666667, + "best_prompt": 13.139999999999999, + "prompt_id": "p2", + "CPS": 12.954287999999998 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-4b-it_0_SK.json b/e3c_llm_results/google/medgemma-4b-it_0_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..e23534e4628922671b0f0b4dd211d9434739834f --- /dev/null +++ b/e3c_llm_results/google/medgemma-4b-it_0_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 19.074956666666665, + "config": { + "model_name": "google/medgemma-4b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SK", + "model": "google/medgemma-4b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "efe6cc02361759b6bd501c654ddb7c9d25ec509d", + "submitted_time": "2025-05-19 20:52:44+00:00", + "num_params_billion": 4.300079472, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 24.47, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 23.87, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 24.47, + "stderr": 0.0 + } + ], + "average_accuracy": 24.27, + "best_prompt": 24.47, + "prompt_id": "p1", + "CPS": 24.421059999999997 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 11.19, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 13.99, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 11.19, + "stderr": 0.0 + } + ], + "average_accuracy": 12.123333333333333, + "best_prompt": 13.99, + "prompt_id": "p2", + "CPS": 13.728853333333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-4b-it_0_SL.json b/e3c_llm_results/google/medgemma-4b-it_0_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..7a3493c014bbb6fa2a33185ee23da96e1baf2e9c --- /dev/null +++ b/e3c_llm_results/google/medgemma-4b-it_0_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 18.266028000000002, + "config": { + "model_name": "google/medgemma-4b-it", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SL", + "model": "google/medgemma-4b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "efe6cc02361759b6bd501c654ddb7c9d25ec509d", + "submitted_time": "2025-05-19 20:52:44+00:00", + "num_params_billion": 4.300079472, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 25.740000000000002, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 25.580000000000002, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 25.740000000000002, + "stderr": 0.0 + } + ], + "average_accuracy": 25.686666666666667, + "best_prompt": 25.740000000000002, + "prompt_id": "p1", + "CPS": 25.726272 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 9.73, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 10.89, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 9.73, + "stderr": 0.0 + } + ], + "average_accuracy": 10.116666666666667, + "best_prompt": 10.89, + "prompt_id": "p2", + "CPS": 10.805784000000001 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-4b-it_10_EN.json b/e3c_llm_results/google/medgemma-4b-it_10_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..3cf5682456197486d183255d45b2a3960a5ed3b3 --- /dev/null +++ b/e3c_llm_results/google/medgemma-4b-it_10_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 31.656783166666663, + "config": { + "model_name": "google/medgemma-4b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "EN", + "model": "google/medgemma-4b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "efe6cc02361759b6bd501c654ddb7c9d25ec509d", + "submitted_time": "2025-05-19 20:52:44+00:00", + "num_params_billion": 4.300079472, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 48.33, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 50.05, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 49.51, + "stderr": 0.0 + } + ], + "average_accuracy": 49.29666666666666, + "best_prompt": 50.05, + "prompt_id": "p2", + "CPS": 49.672956666666664 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 9.64, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 12.370000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 13.91, + "stderr": 0.0 + } + ], + "average_accuracy": 11.973333333333334, + "best_prompt": 13.91, + "prompt_id": "p3", + "CPS": 13.640609666666666 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-4b-it_10_GR.json b/e3c_llm_results/google/medgemma-4b-it_10_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..2c531814bb9d42699d6b76c7abba643e7e4a6095 --- /dev/null +++ b/e3c_llm_results/google/medgemma-4b-it_10_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 32.8816105, + "config": { + "model_name": "google/medgemma-4b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "GR", + "model": "google/medgemma-4b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "efe6cc02361759b6bd501c654ddb7c9d25ec509d", + "submitted_time": "2025-05-19 20:52:44+00:00", + "num_params_billion": 4.300079472, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 49.1, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 50.39, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 49.1, + "stderr": 0.0 + } + ], + "average_accuracy": 49.53, + "best_prompt": 50.39, + "prompt_id": "p2", + "CPS": 49.956646000000006 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 12.04, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 16.05, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 15.509999999999998, + "stderr": 0.0 + } + ], + "average_accuracy": 14.533333333333331, + "best_prompt": 16.05, + "prompt_id": "p2", + "CPS": 15.806575 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-4b-it_10_IT.json b/e3c_llm_results/google/medgemma-4b-it_10_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..7bfa11ea9d723f1f50e9ac88d06cd9eb75050e1b --- /dev/null +++ b/e3c_llm_results/google/medgemma-4b-it_10_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 37.088972000000005, + "config": { + "model_name": "google/medgemma-4b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "IT", + "model": "google/medgemma-4b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "efe6cc02361759b6bd501c654ddb7c9d25ec509d", + "submitted_time": "2025-05-19 20:52:44+00:00", + "num_params_billion": 4.300079472, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 56.330000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 53.769999999999996, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 53.52, + "stderr": 0.0 + } + ], + "average_accuracy": 54.54, + "best_prompt": 56.330000000000005, + "prompt_id": "p1", + "CPS": 55.321693 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 15.920000000000002, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 19.17, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 17.51, + "stderr": 0.0 + } + ], + "average_accuracy": 17.533333333333335, + "best_prompt": 19.17, + "prompt_id": "p2", + "CPS": 18.856251000000004 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-4b-it_10_PL.json b/e3c_llm_results/google/medgemma-4b-it_10_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..8a258a06ac9f7864f3564fb7afc03ccad764f71c --- /dev/null +++ b/e3c_llm_results/google/medgemma-4b-it_10_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 31.782375333333327, + "config": { + "model_name": "google/medgemma-4b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "PL", + "model": "google/medgemma-4b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "efe6cc02361759b6bd501c654ddb7c9d25ec509d", + "submitted_time": "2025-05-19 20:52:44+00:00", + "num_params_billion": 4.300079472, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.85999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 52.059999999999995, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 51.85999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 51.926666666666655, + "best_prompt": 52.059999999999995, + "prompt_id": "p2", + "CPS": 51.99058666666666 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 11.709999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 9.969999999999999, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 9.969999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 10.549999999999999, + "best_prompt": 11.709999999999999, + "prompt_id": "p1", + "CPS": 11.574163999999998 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-4b-it_10_SK.json b/e3c_llm_results/google/medgemma-4b-it_10_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..2df2912042b4e00a4fd588a7a28d8362acff28f8 --- /dev/null +++ b/e3c_llm_results/google/medgemma-4b-it_10_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 28.978618833333336, + "config": { + "model_name": "google/medgemma-4b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SK", + "model": "google/medgemma-4b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "efe6cc02361759b6bd501c654ddb7c9d25ec509d", + "submitted_time": "2025-05-19 20:52:44+00:00", + "num_params_billion": 4.300079472, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 47.56, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 44.49, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 47.56, + "stderr": 0.0 + } + ], + "average_accuracy": 46.53666666666667, + "best_prompt": 47.56, + "prompt_id": "p1", + "CPS": 47.07330266666667 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 10.95, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 10.09, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 10.0, + "stderr": 0.0 + } + ], + "average_accuracy": 10.346666666666666, + "best_prompt": 10.95, + "prompt_id": "p1", + "CPS": 10.883935 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/google/medgemma-4b-it_10_SL.json b/e3c_llm_results/google/medgemma-4b-it_10_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..12372701522a392836b27245e1a82d59a6ef1bfe --- /dev/null +++ b/e3c_llm_results/google/medgemma-4b-it_10_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 32.7709705, + "config": { + "model_name": "google/medgemma-4b-it", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SL", + "model": "google/medgemma-4b-it", + "base_model": "Gemma3ForConditionalGeneration", + "revision": "efe6cc02361759b6bd501c654ddb7c9d25ec509d", + "submitted_time": "2025-05-19 20:52:44+00:00", + "num_params_billion": 4.300079472, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.17, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 49.55, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 51.17, + "stderr": 0.0 + } + ], + "average_accuracy": 50.629999999999995, + "best_prompt": 51.17, + "prompt_id": "p1", + "CPS": 50.893682 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 11.78, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 11.01, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 15.010000000000002, + "stderr": 0.0 + } + ], + "average_accuracy": 12.6, + "best_prompt": 15.010000000000002, + "prompt_id": "p3", + "CPS": 14.648259000000001 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/meta-llama/.DS_Store b/e3c_llm_results/meta-llama/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..446a5c6c78528e5ea6ca14aabc826a97cfbc6bcc Binary files /dev/null and b/e3c_llm_results/meta-llama/.DS_Store differ diff --git a/e3c_llm_results/meta-llama/.ipynb_checkpoints/Llama-3-1.2-1B-Instruct_5_it-checkpoint.json b/e3c_llm_results/meta-llama/.ipynb_checkpoints/Llama-3-1.2-1B-Instruct_5_it-checkpoint.json new file mode 100644 index 0000000000000000000000000000000000000000..8ab73cb0c201e780e965129ec17f0fd3ba5131be --- /dev/null +++ b/e3c_llm_results/meta-llama/.ipynb_checkpoints/Llama-3-1.2-1B-Instruct_5_it-checkpoint.json @@ -0,0 +1,39 @@ +{ + "average_CPS": 12.479999999999999, + "config": { + "model_name": "meta-llama/Llama-3.2-1B-Instruct", + "num_fewshot": "5", + "LANG":"EN", + "batch_size": 8 + }, + "tasks": { + "RE": { + "prompts": [ + { + "prompt": "prom_1", + "metric": "f1", + "value": 12.479999999999999, + "stderr": null + } + ], + "average_accuracy": 12.479999999999999, + "best_prompt": 12.479999999999999, + "prompt_id": "prom_1", + "CPS": 12.479999999999999 + }, + "NER": { + "prompts": [ + { + "prompt": "prom_1", + "metric": "f1", + "value": 20, + "stderr": null + } + ], + "average_accuracy": 20, + "best_prompt": 20, + "prompt_id": "prom_3", + "CPS": 20 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/meta-llama/.ipynb_checkpoints/Llama-3-1.2-1B-Instruct_5_sl-checkpoint.json b/e3c_llm_results/meta-llama/.ipynb_checkpoints/Llama-3-1.2-1B-Instruct_5_sl-checkpoint.json new file mode 100644 index 0000000000000000000000000000000000000000..95acebe7b56935f0e409c3d991831ed381197fba --- /dev/null +++ b/e3c_llm_results/meta-llama/.ipynb_checkpoints/Llama-3-1.2-1B-Instruct_5_sl-checkpoint.json @@ -0,0 +1,39 @@ +{ + "average_CPS": 5, + "config": { + "model_name": "meta-llama/Llama-3.2-1B-Instruct", + "num_fewshot": "5", + "LANG":"IT", + "batch_size": 8 + }, + "tasks": { + "RE": { + "prompts": [ + { + "prompt": "prom_1", + "metric": "f1", + "value": 5, + "stderr": null + } + ], + "average_accuracy": 5, + "best_prompt": 5, + "prompt_id": "prom_1", + "CPS": 5 + }, + "NER": { + "prompts": [ + { + "prompt": "prom_1", + "metric": "f1", + "value": 25, + "stderr": null + } + ], + "average_accuracy": 25, + "best_prompt": 25, + "prompt_id": "prom_3", + "CPS": 25 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/meta-llama/.ipynb_checkpoints/Llama-3.2-1B-Instruct_5-checkpoint.json b/e3c_llm_results/meta-llama/.ipynb_checkpoints/Llama-3.2-1B-Instruct_5-checkpoint.json new file mode 100644 index 0000000000000000000000000000000000000000..50c48aa8a1f4cefbea9b9b5546e2cb3fb1749154 --- /dev/null +++ b/e3c_llm_results/meta-llama/.ipynb_checkpoints/Llama-3.2-1B-Instruct_5-checkpoint.json @@ -0,0 +1,24 @@ +{ + "average_CPS": 12.479999999999999, + "config": { + "model_name": "meta-llama/Llama-3.2-1B-Instruct", + "num_fewshot": "5", + "batch_size": 8 + }, + "tasks": { + "RE": { + "prompts": [ + { + "prompt": "prom_1", + "metric": "f1", + "value": 12.479999999999999, + "stderr": null + } + ], + "average_accuracy": 12.479999999999999, + "best_prompt": 12.479999999999999, + "prompt_id": "prom_1", + "CPS": 12.479999999999999 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/meta-llama/Llama-3-1.2-1B-Instruct_5_it.json b/e3c_llm_results/meta-llama/Llama-3-1.2-1B-Instruct_5_it.json new file mode 100644 index 0000000000000000000000000000000000000000..95acebe7b56935f0e409c3d991831ed381197fba --- /dev/null +++ b/e3c_llm_results/meta-llama/Llama-3-1.2-1B-Instruct_5_it.json @@ -0,0 +1,39 @@ +{ + "average_CPS": 5, + "config": { + "model_name": "meta-llama/Llama-3.2-1B-Instruct", + "num_fewshot": "5", + "LANG":"IT", + "batch_size": 8 + }, + "tasks": { + "RE": { + "prompts": [ + { + "prompt": "prom_1", + "metric": "f1", + "value": 5, + "stderr": null + } + ], + "average_accuracy": 5, + "best_prompt": 5, + "prompt_id": "prom_1", + "CPS": 5 + }, + "NER": { + "prompts": [ + { + "prompt": "prom_1", + "metric": "f1", + "value": 25, + "stderr": null + } + ], + "average_accuracy": 25, + "best_prompt": 25, + "prompt_id": "prom_3", + "CPS": 25 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/meta-llama/Llama-3-1.2-1B-Instruct_5_sl.json b/e3c_llm_results/meta-llama/Llama-3-1.2-1B-Instruct_5_sl.json new file mode 100644 index 0000000000000000000000000000000000000000..5d6184c793235a6e434f76bd74b25be2aa7acdb0 --- /dev/null +++ b/e3c_llm_results/meta-llama/Llama-3-1.2-1B-Instruct_5_sl.json @@ -0,0 +1,39 @@ +{ + "average_CPS": 5, + "config": { + "model_name": "meta-llama/Llama-3.2-1B-Instruct", + "num_fewshot": "5", + "LANG":"SL", + "batch_size": 8 + }, + "tasks": { + "RE": { + "prompts": [ + { + "prompt": "prom_1", + "metric": "f1", + "value": 8, + "stderr": null + } + ], + "average_accuracy": 8, + "best_prompt": 8, + "prompt_id": "prom_1", + "CPS": 8 + }, + "NER": { + "prompts": [ + { + "prompt": "prom_1", + "metric": "f1", + "value": 28, + "stderr": null + } + ], + "average_accuracy": 28, + "best_prompt": 28, + "prompt_id": "prom_3", + "CPS": 28 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/meta-llama/Llama-3.2-1B-Instruct_5.json b/e3c_llm_results/meta-llama/Llama-3.2-1B-Instruct_5.json new file mode 100644 index 0000000000000000000000000000000000000000..57f929e967ef4193e87800b4c8543700b611a406 --- /dev/null +++ b/e3c_llm_results/meta-llama/Llama-3.2-1B-Instruct_5.json @@ -0,0 +1,24 @@ +{ + "average_CPS": 12.479999999999999, + "config": { + "model_name": "meta-llama/Llama-3.2-1B-Instruct", + "num_fewshot": "5", + "batch_size": 8 + }, + "tasks": { + "RE": { + "prompts": [ + { + "prompt": "prompt-1", + "metric": "f1", + "value": 12.479999999999999, + "stderr": null + } + ], + "average_accuracy": 12.479999999999999, + "best_prompt": 12.479999999999999, + "prompt_id": "prompt-1", + "CPS": 12.479999999999999 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Clinical_0_EN.json b/e3c_llm_results/microsoft/MediPhi-Clinical_0_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..98a6e0f785d0df9fdabc0ce2b7dba10c86eb6341 --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Clinical_0_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 33.28536166666667, + "config": { + "model_name": "microsoft/MediPhi-Clinical", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "EN", + "model": "microsoft/MediPhi-Clinical", + "base_model": "Phi3ForCausalLM", + "revision": "0906e64d321a9c4b058137b34fb3ed6e257e05a0", + "submitted_time": "2025-05-29 20:40:05+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 25.019999999999996, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 30.89, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 27.68, + "stderr": 0.0 + } + ], + "average_accuracy": 27.863333333333333, + "best_prompt": 30.89, + "prompt_id": "p2", + "CPS": 29.955062666666667 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 22.74, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 39.290000000000006, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 35.42, + "stderr": 0.0 + } + ], + "average_accuracy": 32.483333333333334, + "best_prompt": 39.290000000000006, + "prompt_id": "p2", + "CPS": 36.61566066666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Clinical_0_GR.json b/e3c_llm_results/microsoft/MediPhi-Clinical_0_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..6a6a11334dacdc148c4c44e3bfc69cd5bda481c9 --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Clinical_0_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 15.980523333333334, + "config": { + "model_name": "microsoft/MediPhi-Clinical", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "GR", + "model": "microsoft/MediPhi-Clinical", + "base_model": "Phi3ForCausalLM", + "revision": "0906e64d321a9c4b058137b34fb3ed6e257e05a0", + "submitted_time": "2025-05-29 20:40:05+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 16.41, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 18.69, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 16.41, + "stderr": 0.0 + } + ], + "average_accuracy": 17.17, + "best_prompt": 18.69, + "prompt_id": "p2", + "CPS": 18.405912 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 7.359999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 7.779999999999999, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 14.180000000000001, + "stderr": 0.0 + } + ], + "average_accuracy": 9.773333333333333, + "best_prompt": 14.180000000000001, + "prompt_id": "p3", + "CPS": 13.555134666666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Clinical_0_IT.json b/e3c_llm_results/microsoft/MediPhi-Clinical_0_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..bed90daaef3864044a73374247b7f6459e17dbac --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Clinical_0_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 23.758854499999998, + "config": { + "model_name": "microsoft/MediPhi-Clinical", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "IT", + "model": "microsoft/MediPhi-Clinical", + "base_model": "Phi3ForCausalLM", + "revision": "0906e64d321a9c4b058137b34fb3ed6e257e05a0", + "submitted_time": "2025-05-29 20:40:05+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 33.97, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 33.0, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 32.26, + "stderr": 0.0 + } + ], + "average_accuracy": 33.07666666666666, + "best_prompt": 33.97, + "prompt_id": "p1", + "CPS": 33.666534666666664 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 14.89, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 7.359999999999999, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 1.49, + "stderr": 0.0 + } + ], + "average_accuracy": 7.913333333333333, + "best_prompt": 14.89, + "prompt_id": "p1", + "CPS": 13.851174333333335 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Clinical_0_PL.json b/e3c_llm_results/microsoft/MediPhi-Clinical_0_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..2f93e127070f8a5949bf8a835c9f160364e26db9 --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Clinical_0_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 29.002397166666665, + "config": { + "model_name": "microsoft/MediPhi-Clinical", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "PL", + "model": "microsoft/MediPhi-Clinical", + "base_model": "Phi3ForCausalLM", + "revision": "0906e64d321a9c4b058137b34fb3ed6e257e05a0", + "submitted_time": "2025-05-29 20:40:05+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 28.15, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 28.610000000000003, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 28.15, + "stderr": 0.0 + } + ], + "average_accuracy": 28.30333333333333, + "best_prompt": 28.610000000000003, + "prompt_id": "p2", + "CPS": 28.522262666666666 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 21.09, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 29.080000000000002, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 30.61, + "stderr": 0.0 + } + ], + "average_accuracy": 26.926666666666666, + "best_prompt": 30.61, + "prompt_id": "p3", + "CPS": 29.482531666666663 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Clinical_0_SK.json b/e3c_llm_results/microsoft/MediPhi-Clinical_0_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..c3d496d955f633bf0be15550f6669da7fb35d1ab --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Clinical_0_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 21.908259666666666, + "config": { + "model_name": "microsoft/MediPhi-Clinical", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SK", + "model": "microsoft/MediPhi-Clinical", + "base_model": "Phi3ForCausalLM", + "revision": "0906e64d321a9c4b058137b34fb3ed6e257e05a0", + "submitted_time": "2025-05-29 20:40:05+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 25.71, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 29.87, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 25.71, + "stderr": 0.0 + } + ], + "average_accuracy": 27.096666666666668, + "best_prompt": 29.87, + "prompt_id": "p2", + "CPS": 29.041605333333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 15.540000000000001, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.77, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 15.540000000000001, + "stderr": 0.0 + } + ], + "average_accuracy": 10.616666666666667, + "best_prompt": 15.540000000000001, + "prompt_id": "p1", + "CPS": 14.774914 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Clinical_0_SL.json b/e3c_llm_results/microsoft/MediPhi-Clinical_0_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..5c499e998d83a98fe29b6a690c7b6a139a8c41dd --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Clinical_0_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 16.78806766666667, + "config": { + "model_name": "microsoft/MediPhi-Clinical", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SL", + "model": "microsoft/MediPhi-Clinical", + "base_model": "Phi3ForCausalLM", + "revision": "0906e64d321a9c4b058137b34fb3ed6e257e05a0", + "submitted_time": "2025-05-29 20:40:05+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 29.98, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 26.8, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 29.98, + "stderr": 0.0 + } + ], + "average_accuracy": 28.92, + "best_prompt": 29.98, + "prompt_id": "p1", + "CPS": 29.662212000000004 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 3.95, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 1.21, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 3.95, + "stderr": 0.0 + } + ], + "average_accuracy": 3.0366666666666666, + "best_prompt": 3.95, + "prompt_id": "p1", + "CPS": 3.9139233333333334 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Clinical_10_EN.json b/e3c_llm_results/microsoft/MediPhi-Clinical_10_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..172cd2593a629873e135edee8b555b1ff5567d9a --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Clinical_10_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 30.98751216666667, + "config": { + "model_name": "microsoft/MediPhi-Clinical", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "EN", + "model": "microsoft/MediPhi-Clinical", + "base_model": "Phi3ForCausalLM", + "revision": "0906e64d321a9c4b058137b34fb3ed6e257e05a0", + "submitted_time": "2025-05-29 20:40:05+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 50.09, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 49.66, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 50.49, + "stderr": 0.0 + } + ], + "average_accuracy": 50.080000000000005, + "best_prompt": 50.49, + "prompt_id": "p3", + "CPS": 50.282991 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 11.75, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 10.95, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 11.07, + "stderr": 0.0 + } + ], + "average_accuracy": 11.256666666666666, + "best_prompt": 11.75, + "prompt_id": "p1", + "CPS": 11.692033333333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Clinical_10_GR.json b/e3c_llm_results/microsoft/MediPhi-Clinical_10_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..7cc2471b33b1a5ca97d09a4142e483c5b70e63e0 --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Clinical_10_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 20.501029666666668, + "config": { + "model_name": "microsoft/MediPhi-Clinical", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "GR", + "model": "microsoft/MediPhi-Clinical", + "base_model": "Phi3ForCausalLM", + "revision": "0906e64d321a9c4b058137b34fb3ed6e257e05a0", + "submitted_time": "2025-05-29 20:40:05+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 33.75, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 34.03, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 33.75, + "stderr": 0.0 + } + ], + "average_accuracy": 33.843333333333334, + "best_prompt": 34.03, + "prompt_id": "p2", + "CPS": 33.96647733333334 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 4.2700000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 6.81, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 7.109999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 6.063333333333333, + "best_prompt": 7.109999999999999, + "prompt_id": "p3", + "CPS": 7.035582 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Clinical_10_IT.json b/e3c_llm_results/microsoft/MediPhi-Clinical_10_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..ab066b177276ec685bb3bf8c7d615f274fd7b17a --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Clinical_10_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 36.308323, + "config": { + "model_name": "microsoft/MediPhi-Clinical", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "IT", + "model": "microsoft/MediPhi-Clinical", + "base_model": "Phi3ForCausalLM", + "revision": "0906e64d321a9c4b058137b34fb3ed6e257e05a0", + "submitted_time": "2025-05-29 20:40:05+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.949999999999996, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 53.010000000000005, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 52.75, + "stderr": 0.0 + } + ], + "average_accuracy": 52.57, + "best_prompt": 53.010000000000005, + "prompt_id": "p2", + "CPS": 52.776756 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 21.14, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 9.610000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 14.219999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 14.99, + "best_prompt": 21.14, + "prompt_id": "p1", + "CPS": 19.83989 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Clinical_10_PL.json b/e3c_llm_results/microsoft/MediPhi-Clinical_10_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..89a2aff72039f7036b3a0d1bb05567a8843aef7a --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Clinical_10_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 28.317504, + "config": { + "model_name": "microsoft/MediPhi-Clinical", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "PL", + "model": "microsoft/MediPhi-Clinical", + "base_model": "Phi3ForCausalLM", + "revision": "0906e64d321a9c4b058137b34fb3ed6e257e05a0", + "submitted_time": "2025-05-29 20:40:05+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 39.129999999999995, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 41.32, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 39.129999999999995, + "stderr": 0.0 + } + ], + "average_accuracy": 39.85999999999999, + "best_prompt": 41.32, + "prompt_id": "p2", + "CPS": 40.716727999999996 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 12.55, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 12.07, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 16.36, + "stderr": 0.0 + } + ], + "average_accuracy": 13.660000000000002, + "best_prompt": 16.36, + "prompt_id": "p3", + "CPS": 15.91828 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Clinical_10_SK.json b/e3c_llm_results/microsoft/MediPhi-Clinical_10_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..43aa5ad9534745978a76d5a8f0df5e3ef8ae7cc4 --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Clinical_10_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 23.942156333333333, + "config": { + "model_name": "microsoft/MediPhi-Clinical", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SK", + "model": "microsoft/MediPhi-Clinical", + "base_model": "Phi3ForCausalLM", + "revision": "0906e64d321a9c4b058137b34fb3ed6e257e05a0", + "submitted_time": "2025-05-29 20:40:05+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 41.06, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 38.61, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 41.06, + "stderr": 0.0 + } + ], + "average_accuracy": 40.24333333333333, + "best_prompt": 41.06, + "prompt_id": "p1", + "CPS": 40.72467666666667 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 5.09, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 6.0600000000000005, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 7.24, + "stderr": 0.0 + } + ], + "average_accuracy": 6.13, + "best_prompt": 7.24, + "prompt_id": "p3", + "CPS": 7.159636 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Clinical_10_SL.json b/e3c_llm_results/microsoft/MediPhi-Clinical_10_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..10336f2da374f0d4e3497ca52a60c3e897867991 --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Clinical_10_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 24.260621, + "config": { + "model_name": "microsoft/MediPhi-Clinical", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SL", + "model": "microsoft/MediPhi-Clinical", + "base_model": "Phi3ForCausalLM", + "revision": "0906e64d321a9c4b058137b34fb3ed6e257e05a0", + "submitted_time": "2025-05-29 20:40:05+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 40.36, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 39.900000000000006, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 40.36, + "stderr": 0.0 + } + ], + "average_accuracy": 40.20666666666667, + "best_prompt": 40.36, + "prompt_id": "p1", + "CPS": 40.29811466666667 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 8.290000000000001, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 6.74, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 7.42, + "stderr": 0.0 + } + ], + "average_accuracy": 7.483333333333334, + "best_prompt": 8.290000000000001, + "prompt_id": "p1", + "CPS": 8.223127333333334 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Instruct_0_EN.json b/e3c_llm_results/microsoft/MediPhi-Instruct_0_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..22ce1987e27b47151de0819e610f8b7a1847b14b --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Instruct_0_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 29.050603666666667, + "config": { + "model_name": "microsoft/MediPhi-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "EN", + "model": "microsoft/MediPhi-Instruct", + "base_model": "Phi3ForCausalLM", + "revision": "a94ac478e7c246103d55665a0804684042f3b973", + "submitted_time": "2025-07-11 19:28:15+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 7.61, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 24.099999999999998, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 16.25, + "stderr": 0.0 + } + ], + "average_accuracy": 15.986666666666666, + "best_prompt": 24.099999999999998, + "prompt_id": "p2", + "CPS": 22.144686666666665 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 11.35, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 40.06, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 38.04, + "stderr": 0.0 + } + ], + "average_accuracy": 29.816666666666666, + "best_prompt": 40.06, + "prompt_id": "p2", + "CPS": 35.95652066666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Instruct_0_GR.json b/e3c_llm_results/microsoft/MediPhi-Instruct_0_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..da244d4097b0eb2f428c05b2766afa2abe199a9e --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Instruct_0_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 15.261295333333333, + "config": { + "model_name": "microsoft/MediPhi-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "GR", + "model": "microsoft/MediPhi-Instruct", + "base_model": "Phi3ForCausalLM", + "revision": "a94ac478e7c246103d55665a0804684042f3b973", + "submitted_time": "2025-07-11 19:28:15+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 12.94, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 8.9, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 12.94, + "stderr": 0.0 + } + ], + "average_accuracy": 11.593333333333334, + "best_prompt": 12.94, + "prompt_id": "p1", + "CPS": 12.765741333333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 9.62, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 6.7299999999999995, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 19.16, + "stderr": 0.0 + } + ], + "average_accuracy": 11.836666666666666, + "best_prompt": 19.16, + "prompt_id": "p3", + "CPS": 17.75684933333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Instruct_0_IT.json b/e3c_llm_results/microsoft/MediPhi-Instruct_0_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..0841f839a64bc0b9b922147ccea62cf26e0a601b --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Instruct_0_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 27.906489166666667, + "config": { + "model_name": "microsoft/MediPhi-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "IT", + "model": "microsoft/MediPhi-Instruct", + "base_model": "Phi3ForCausalLM", + "revision": "a94ac478e7c246103d55665a0804684042f3b973", + "submitted_time": "2025-07-11 19:28:15+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 8.67, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 24.84, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 27.169999999999998, + "stderr": 0.0 + } + ], + "average_accuracy": 20.226666666666667, + "best_prompt": 27.169999999999998, + "prompt_id": "p3", + "CPS": 25.283496333333332 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 17.119999999999997, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 28.96, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 32.61, + "stderr": 0.0 + } + ], + "average_accuracy": 26.23, + "best_prompt": 32.61, + "prompt_id": "p3", + "CPS": 30.529482 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Instruct_0_PL.json b/e3c_llm_results/microsoft/MediPhi-Instruct_0_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..d310296a6cfd872d87b0ac069ca9720091362f13 --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Instruct_0_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 23.550823, + "config": { + "model_name": "microsoft/MediPhi-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "PL", + "model": "microsoft/MediPhi-Instruct", + "base_model": "Phi3ForCausalLM", + "revision": "a94ac478e7c246103d55665a0804684042f3b973", + "submitted_time": "2025-07-11 19:28:15+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 15.1, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 16.8, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 15.1, + "stderr": 0.0 + } + ], + "average_accuracy": 15.666666666666666, + "best_prompt": 16.8, + "prompt_id": "p2", + "CPS": 16.6096 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 26.83, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 31.259999999999998, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 28.32, + "stderr": 0.0 + } + ], + "average_accuracy": 28.80333333333333, + "best_prompt": 31.259999999999998, + "prompt_id": "p2", + "CPS": 30.492046 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Instruct_0_SK.json b/e3c_llm_results/microsoft/MediPhi-Instruct_0_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..e04423fa8fe8a13c7aa56753c7194c337a73e6ea --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Instruct_0_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 18.487242666666667, + "config": { + "model_name": "microsoft/MediPhi-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SK", + "model": "microsoft/MediPhi-Instruct", + "base_model": "Phi3ForCausalLM", + "revision": "a94ac478e7c246103d55665a0804684042f3b973", + "submitted_time": "2025-07-11 19:28:15+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 16.41, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 20.810000000000002, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 16.41, + "stderr": 0.0 + } + ], + "average_accuracy": 17.87666666666667, + "best_prompt": 20.810000000000002, + "prompt_id": "p2", + "CPS": 20.199573333333337 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 17.76, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 1.1199999999999999, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 17.76, + "stderr": 0.0 + } + ], + "average_accuracy": 12.213333333333333, + "best_prompt": 17.76, + "prompt_id": "p1", + "CPS": 16.774912 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Instruct_0_SL.json b/e3c_llm_results/microsoft/MediPhi-Instruct_0_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..c39fa4962d9c73f16202f4c7c0784db69d3aeb89 --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Instruct_0_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 16.379518, + "config": { + "model_name": "microsoft/MediPhi-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SL", + "model": "microsoft/MediPhi-Instruct", + "base_model": "Phi3ForCausalLM", + "revision": "a94ac478e7c246103d55665a0804684042f3b973", + "submitted_time": "2025-07-11 19:28:15+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 17.580000000000002, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 18.6, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 17.580000000000002, + "stderr": 0.0 + } + ], + "average_accuracy": 17.92, + "best_prompt": 18.6, + "prompt_id": "p2", + "CPS": 18.47352 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 14.46, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 10.84, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 14.46, + "stderr": 0.0 + } + ], + "average_accuracy": 13.253333333333336, + "best_prompt": 14.46, + "prompt_id": "p1", + "CPS": 14.285516000000001 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Instruct_10_EN.json b/e3c_llm_results/microsoft/MediPhi-Instruct_10_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..d35e2e7548790ba37050b7b296ad22e195892857 --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Instruct_10_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 35.686588, + "config": { + "model_name": "microsoft/MediPhi-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "EN", + "model": "microsoft/MediPhi-Instruct", + "base_model": "Phi3ForCausalLM", + "revision": "a94ac478e7c246103d55665a0804684042f3b973", + "submitted_time": "2025-07-11 19:28:15+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 53.56999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 52.27, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 50.629999999999995, + "stderr": 0.0 + } + ], + "average_accuracy": 52.156666666666666, + "best_prompt": 53.56999999999999, + "prompt_id": "p1", + "CPS": 52.81287733333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 14.32, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 18.88, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 18.360000000000003, + "stderr": 0.0 + } + ], + "average_accuracy": 17.186666666666667, + "best_prompt": 18.88, + "prompt_id": "p2", + "CPS": 18.560298666666665 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Instruct_10_GR.json b/e3c_llm_results/microsoft/MediPhi-Instruct_10_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..746a4d6809495e6b0b81e7ed14d30b71aba5f6ef --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Instruct_10_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 18.663691, + "config": { + "model_name": "microsoft/MediPhi-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "GR", + "model": "microsoft/MediPhi-Instruct", + "base_model": "Phi3ForCausalLM", + "revision": "a94ac478e7c246103d55665a0804684042f3b973", + "submitted_time": "2025-07-11 19:28:15+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 28.22, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 29.99, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 28.22, + "stderr": 0.0 + } + ], + "average_accuracy": 28.81, + "best_prompt": 29.99, + "prompt_id": "p2", + "CPS": 29.636117999999996 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 5.76, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 6.74, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 7.7700000000000005, + "stderr": 0.0 + } + ], + "average_accuracy": 6.756666666666667, + "best_prompt": 7.7700000000000005, + "prompt_id": "p3", + "CPS": 7.691264 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Instruct_10_IT.json b/e3c_llm_results/microsoft/MediPhi-Instruct_10_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..df3be99350ebf1b022cb709ae59908e2e428285d --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Instruct_10_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 42.82063783333333, + "config": { + "model_name": "microsoft/MediPhi-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "IT", + "model": "microsoft/MediPhi-Instruct", + "base_model": "Phi3ForCausalLM", + "revision": "a94ac478e7c246103d55665a0804684042f3b973", + "submitted_time": "2025-07-11 19:28:15+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 57.29, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 56.269999999999996, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 57.9, + "stderr": 0.0 + } + ], + "average_accuracy": 57.15333333333333, + "best_prompt": 57.9, + "prompt_id": "p3", + "CPS": 57.467679999999994 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 28.73, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 23.07, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 28.58, + "stderr": 0.0 + } + ], + "average_accuracy": 26.793333333333333, + "best_prompt": 28.73, + "prompt_id": "p1", + "CPS": 28.173595666666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Instruct_10_PL.json b/e3c_llm_results/microsoft/MediPhi-Instruct_10_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..53e1c256501493c36d30438e28047eda273d3b40 --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Instruct_10_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 35.295837, + "config": { + "model_name": "microsoft/MediPhi-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "PL", + "model": "microsoft/MediPhi-Instruct", + "base_model": "Phi3ForCausalLM", + "revision": "a94ac478e7c246103d55665a0804684042f3b973", + "submitted_time": "2025-07-11 19:28:15+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 44.17, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 45.06, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 44.17, + "stderr": 0.0 + } + ], + "average_accuracy": 44.46666666666667, + "best_prompt": 45.06, + "prompt_id": "p2", + "CPS": 44.792644 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 15.25, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 26.86, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 26.619999999999997, + "stderr": 0.0 + } + ], + "average_accuracy": 22.909999999999997, + "best_prompt": 26.86, + "prompt_id": "p2", + "CPS": 25.79903 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Instruct_10_SK.json b/e3c_llm_results/microsoft/MediPhi-Instruct_10_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..ca13dfd0624a5d2a282043f03582c65fef1f6743 --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Instruct_10_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 28.662679833333332, + "config": { + "model_name": "microsoft/MediPhi-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SK", + "model": "microsoft/MediPhi-Instruct", + "base_model": "Phi3ForCausalLM", + "revision": "a94ac478e7c246103d55665a0804684042f3b973", + "submitted_time": "2025-07-11 19:28:15+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 43.269999999999996, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 40.23, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 43.269999999999996, + "stderr": 0.0 + } + ], + "average_accuracy": 42.25666666666666, + "best_prompt": 43.269999999999996, + "prompt_id": "p1", + "CPS": 42.831530666666666 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 10.7, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 13.950000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 14.729999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 13.126666666666665, + "best_prompt": 14.729999999999999, + "prompt_id": "p3", + "CPS": 14.493828999999998 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/microsoft/MediPhi-Instruct_10_SL.json b/e3c_llm_results/microsoft/MediPhi-Instruct_10_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..c6ee1d145af7f243f04d69dd9e274bafc5e23b01 --- /dev/null +++ b/e3c_llm_results/microsoft/MediPhi-Instruct_10_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 29.24573433333333, + "config": { + "model_name": "microsoft/MediPhi-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SL", + "model": "microsoft/MediPhi-Instruct", + "base_model": "Phi3ForCausalLM", + "revision": "a94ac478e7c246103d55665a0804684042f3b973", + "submitted_time": "2025-07-11 19:28:15+00:00", + "num_params_billion": 3.821079552, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 39.73, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 35.64, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 39.73, + "stderr": 0.0 + } + ], + "average_accuracy": 38.36666666666667, + "best_prompt": 39.73, + "prompt_id": "p1", + "CPS": 39.188347666666665 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 11.55, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 14.680000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 20.27, + "stderr": 0.0 + } + ], + "average_accuracy": 15.5, + "best_prompt": 20.27, + "prompt_id": "p3", + "CPS": 19.303121 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_EN.json b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..09b48d5503371b1651be316f6dbb21ba9fce6c34 --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 29.513497333333333, + "config": { + "model_name": "mistralai/Mistral-7B-Instruct-v0.2", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "EN", + "model": "mistralai/Mistral-7B-Instruct-v0.2", + "base_model": "MistralForCausalLM", + "revision": "63a8b081895390a26e140280378bc85ec8bce07a", + "submitted_time": "2023-12-11 13:18:44+00:00", + "num_params_billion": 7.241732096, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 25.290000000000003, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 21.44, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 21.62, + "stderr": 0.0 + } + ], + "average_accuracy": 22.783333333333335, + "best_prompt": 25.290000000000003, + "prompt_id": "p1", + "CPS": 24.656064 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 36.88, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 36.42, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 16.93, + "stderr": 0.0 + } + ], + "average_accuracy": 30.076666666666668, + "best_prompt": 36.88, + "prompt_id": "p1", + "CPS": 34.370930666666666 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_GR.json b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..c0ccb63ae51aedbdaaf0f22e88c8d9b6264517f2 --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 14.120156666666666, + "config": { + "model_name": "mistralai/Mistral-7B-Instruct-v0.2", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "GR", + "model": "mistralai/Mistral-7B-Instruct-v0.2", + "base_model": "MistralForCausalLM", + "revision": "63a8b081895390a26e140280378bc85ec8bce07a", + "submitted_time": "2023-12-11 13:18:44+00:00", + "num_params_billion": 7.241732096, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 16.03, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 19.09, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 16.03, + "stderr": 0.0 + } + ], + "average_accuracy": 17.05, + "best_prompt": 19.09, + "prompt_id": "p2", + "CPS": 18.700564 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 4.32, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 3.4799999999999995, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 9.94, + "stderr": 0.0 + } + ], + "average_accuracy": 5.913333333333333, + "best_prompt": 9.94, + "prompt_id": "p3", + "CPS": 9.539749333333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_IT.json b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..9998fd4cd3f71845aff867c97000e62c1662f5fb --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 19.788279666666668, + "config": { + "model_name": "mistralai/Mistral-7B-Instruct-v0.2", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "IT", + "model": "mistralai/Mistral-7B-Instruct-v0.2", + "base_model": "MistralForCausalLM", + "revision": "63a8b081895390a26e140280378bc85ec8bce07a", + "submitted_time": "2023-12-11 13:18:44+00:00", + "num_params_billion": 7.241732096, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 27.88, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 20.3, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 24.81, + "stderr": 0.0 + } + ], + "average_accuracy": 24.33, + "best_prompt": 27.88, + "prompt_id": "p1", + "CPS": 26.89026 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 13.819999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 1.63, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 1.4000000000000001, + "stderr": 0.0 + } + ], + "average_accuracy": 5.616666666666666, + "best_prompt": 13.819999999999999, + "prompt_id": "p1", + "CPS": 12.686299333333332 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_PL.json b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..770eab4c5a5168471cfeb8615bdb4865175d5457 --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 21.287892, + "config": { + "model_name": "mistralai/Mistral-7B-Instruct-v0.2", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "PL", + "model": "mistralai/Mistral-7B-Instruct-v0.2", + "base_model": "MistralForCausalLM", + "revision": "63a8b081895390a26e140280378bc85ec8bce07a", + "submitted_time": "2023-12-11 13:18:44+00:00", + "num_params_billion": 7.241732096, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 30.240000000000002, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 28.110000000000003, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 30.240000000000002, + "stderr": 0.0 + } + ], + "average_accuracy": 29.53, + "best_prompt": 30.240000000000002, + "prompt_id": "p1", + "CPS": 30.025296 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 8.63, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 12.920000000000002, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 8.63, + "stderr": 0.0 + } + ], + "average_accuracy": 10.06, + "best_prompt": 12.920000000000002, + "prompt_id": "p2", + "CPS": 12.550488000000001 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_SK.json b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..fa66c644a134754eac14f412cc230e113ec12ad9 --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 14.880865666666669, + "config": { + "model_name": "mistralai/Mistral-7B-Instruct-v0.2", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SK", + "model": "mistralai/Mistral-7B-Instruct-v0.2", + "base_model": "MistralForCausalLM", + "revision": "63a8b081895390a26e140280378bc85ec8bce07a", + "submitted_time": "2023-12-11 13:18:44+00:00", + "num_params_billion": 7.241732096, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 21.43, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 21.46, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 21.43, + "stderr": 0.0 + } + ], + "average_accuracy": 21.439999999999998, + "best_prompt": 21.46, + "prompt_id": "p2", + "CPS": 21.455708 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 7.5600000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 8.35, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 7.5600000000000005, + "stderr": 0.0 + } + ], + "average_accuracy": 7.823333333333333, + "best_prompt": 8.35, + "prompt_id": "p2", + "CPS": 8.306023333333334 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_SL.json b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..cbcfa790d9fef4dd5fa4f1df4c3812eb1511dfbe --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_0_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 17.567646000000003, + "config": { + "model_name": "mistralai/Mistral-7B-Instruct-v0.2", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SL", + "model": "mistralai/Mistral-7B-Instruct-v0.2", + "base_model": "MistralForCausalLM", + "revision": "63a8b081895390a26e140280378bc85ec8bce07a", + "submitted_time": "2023-12-11 13:18:44+00:00", + "num_params_billion": 7.241732096, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 17.66, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 19.470000000000002, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 17.66, + "stderr": 0.0 + } + ], + "average_accuracy": 18.263333333333335, + "best_prompt": 19.470000000000002, + "prompt_id": "p2", + "CPS": 19.235062000000003 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 7.66, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 16.950000000000003, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 7.66, + "stderr": 0.0 + } + ], + "average_accuracy": 10.756666666666668, + "best_prompt": 16.950000000000003, + "prompt_id": "p2", + "CPS": 15.900230000000002 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_EN.json b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..901677072102f05031732c689b4caaa13f0f844c --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 43.4870355, + "config": { + "model_name": "mistralai/Mistral-7B-Instruct-v0.2", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "EN", + "model": "mistralai/Mistral-7B-Instruct-v0.2", + "base_model": "MistralForCausalLM", + "revision": "63a8b081895390a26e140280378bc85ec8bce07a", + "submitted_time": "2023-12-11 13:18:44+00:00", + "num_params_billion": 7.241732096, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 47.25, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 47.3, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 48.05, + "stderr": 0.0 + } + ], + "average_accuracy": 47.53333333333333, + "best_prompt": 48.05, + "prompt_id": "p3", + "CPS": 47.801741666666665 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 25.929999999999996, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 40.339999999999996, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 41.48, + "stderr": 0.0 + } + ], + "average_accuracy": 35.916666666666664, + "best_prompt": 41.48, + "prompt_id": "p3", + "CPS": 39.17232933333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_GR.json b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..c781ff1f482edfb2dc8d1f751eae772debd3267f --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 29.209499, + "config": { + "model_name": "mistralai/Mistral-7B-Instruct-v0.2", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "GR", + "model": "mistralai/Mistral-7B-Instruct-v0.2", + "base_model": "MistralForCausalLM", + "revision": "63a8b081895390a26e140280378bc85ec8bce07a", + "submitted_time": "2023-12-11 13:18:44+00:00", + "num_params_billion": 7.241732096, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 34.98, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 36.480000000000004, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 34.98, + "stderr": 0.0 + } + ], + "average_accuracy": 35.48, + "best_prompt": 36.480000000000004, + "prompt_id": "p2", + "CPS": 36.1152 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 10.549999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 23.43, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 21.89, + "stderr": 0.0 + } + ], + "average_accuracy": 18.62333333333333, + "best_prompt": 23.43, + "prompt_id": "p2", + "CPS": 22.303798 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_IT.json b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..517fbc098c7e944fdca6ef2137b37251ff6f5fdc --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 47.367924, + "config": { + "model_name": "mistralai/Mistral-7B-Instruct-v0.2", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "IT", + "model": "mistralai/Mistral-7B-Instruct-v0.2", + "base_model": "MistralForCausalLM", + "revision": "63a8b081895390a26e140280378bc85ec8bce07a", + "submitted_time": "2023-12-11 13:18:44+00:00", + "num_params_billion": 7.241732096, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.470000000000006, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 52.32, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 51.49, + "stderr": 0.0 + } + ], + "average_accuracy": 51.76, + "best_prompt": 52.32, + "prompt_id": "p2", + "CPS": 52.027007999999995 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 30.919999999999998, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 45.300000000000004, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 42.52, + "stderr": 0.0 + } + ], + "average_accuracy": 39.580000000000005, + "best_prompt": 45.300000000000004, + "prompt_id": "p2", + "CPS": 42.70884 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_PL.json b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..75e218546bbd71959c63f68f07b2b9178d4f2ac6 --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 43.31202666666667, + "config": { + "model_name": "mistralai/Mistral-7B-Instruct-v0.2", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "PL", + "model": "mistralai/Mistral-7B-Instruct-v0.2", + "base_model": "MistralForCausalLM", + "revision": "63a8b081895390a26e140280378bc85ec8bce07a", + "submitted_time": "2023-12-11 13:18:44+00:00", + "num_params_billion": 7.241732096, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 49.11, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 50.46000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 49.11, + "stderr": 0.0 + } + ], + "average_accuracy": 49.56, + "best_prompt": 50.46000000000001, + "prompt_id": "p2", + "CPS": 50.005860000000006 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 38.95, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 33.11, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 26.83, + "stderr": 0.0 + } + ], + "average_accuracy": 32.96333333333333, + "best_prompt": 38.95, + "prompt_id": "p1", + "CPS": 36.61819333333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_SK.json b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..9989c0a065bd29a74bb0b47f1a054aff266bd0a9 --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 31.267611000000002, + "config": { + "model_name": "mistralai/Mistral-7B-Instruct-v0.2", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SK", + "model": "mistralai/Mistral-7B-Instruct-v0.2", + "base_model": "MistralForCausalLM", + "revision": "63a8b081895390a26e140280378bc85ec8bce07a", + "submitted_time": "2023-12-11 13:18:44+00:00", + "num_params_billion": 7.241732096, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 40.29, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 37.940000000000005, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 40.29, + "stderr": 0.0 + } + ], + "average_accuracy": 39.50666666666667, + "best_prompt": 40.29, + "prompt_id": "p1", + "CPS": 39.974395 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 21.55, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 19.48, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 22.93, + "stderr": 0.0 + } + ], + "average_accuracy": 21.32, + "best_prompt": 22.93, + "prompt_id": "p3", + "CPS": 22.560827 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_SL.json b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..d06da082bdd17139af094b07794fcd5c2a9c15ff --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-7B-Instruct-v0.2_10_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 31.471755, + "config": { + "model_name": "mistralai/Mistral-7B-Instruct-v0.2", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SL", + "model": "mistralai/Mistral-7B-Instruct-v0.2", + "base_model": "MistralForCausalLM", + "revision": "63a8b081895390a26e140280378bc85ec8bce07a", + "submitted_time": "2023-12-11 13:18:44+00:00", + "num_params_billion": 7.241732096, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 42.04, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 41.74, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 42.04, + "stderr": 0.0 + } + ], + "average_accuracy": 41.94, + "best_prompt": 42.04, + "prompt_id": "p1", + "CPS": 41.99796 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 19.900000000000002, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 19.5, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 21.15, + "stderr": 0.0 + } + ], + "average_accuracy": 20.183333333333334, + "best_prompt": 21.15, + "prompt_id": "p3", + "CPS": 20.94555 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_EN.json b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..10e5f5fcc524655d6416f6c912e97e8c9dc3f984 --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 31.20982683333333, + "config": { + "model_name": "mistralai/Mistral-Nemo-Instruct-2407", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "EN", + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "base_model": "MistralForCausalLM", + "revision": "04d8a90549d23fc6bd7f642064003592df51e9b3", + "submitted_time": "2024-07-17 17:26:49+00:00", + "num_params_billion": 12.2477824, + "language": "en_fr_de_es_it_pt_ru_zh_ja" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 27.67, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 22.99, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 17.48, + "stderr": 0.0 + } + ], + "average_accuracy": 22.713333333333335, + "best_prompt": 27.67, + "prompt_id": "p1", + "CPS": 26.298490333333337 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 36.94, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 34.82, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 32.41, + "stderr": 0.0 + } + ], + "average_accuracy": 34.72333333333333, + "best_prompt": 36.94, + "prompt_id": "p1", + "CPS": 36.12116333333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_GR.json b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..96f5e2cf137d493e100323cf6eae12f5103bc8ec --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 18.510654333333335, + "config": { + "model_name": "mistralai/Mistral-Nemo-Instruct-2407", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "GR", + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "base_model": "MistralForCausalLM", + "revision": "04d8a90549d23fc6bd7f642064003592df51e9b3", + "submitted_time": "2024-07-17 17:26:49+00:00", + "num_params_billion": 12.2477824, + "language": "en_fr_de_es_it_pt_ru_zh_ja" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 7.32, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 6.87, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 7.32, + "stderr": 0.0 + } + ], + "average_accuracy": 7.170000000000001, + "best_prompt": 7.32, + "prompt_id": "p1", + "CPS": 7.30902 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 15.75, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 21.17, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 32.87, + "stderr": 0.0 + } + ], + "average_accuracy": 23.263333333333332, + "best_prompt": 32.87, + "prompt_id": "p3", + "CPS": 29.712288666666666 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_IT.json b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..91b0a3a567b60f79a3aca8cb9c914da759c7da05 --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 26.353595166666665, + "config": { + "model_name": "mistralai/Mistral-Nemo-Instruct-2407", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "IT", + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "base_model": "MistralForCausalLM", + "revision": "04d8a90549d23fc6bd7f642064003592df51e9b3", + "submitted_time": "2024-07-17 17:26:49+00:00", + "num_params_billion": 12.2477824, + "language": "en_fr_de_es_it_pt_ru_zh_ja" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 27.92, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 17.72, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 13.16, + "stderr": 0.0 + } + ], + "average_accuracy": 19.599999999999998, + "best_prompt": 27.92, + "prompt_id": "p1", + "CPS": 25.597056 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 28.49, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 23.84, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 18.61, + "stderr": 0.0 + } + ], + "average_accuracy": 23.646666666666665, + "best_prompt": 28.49, + "prompt_id": "p1", + "CPS": 27.11013433333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_PL.json b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..d9583de11950a2782b559d1116c723eff4a969a9 --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 12.708361833333333, + "config": { + "model_name": "mistralai/Mistral-Nemo-Instruct-2407", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "PL", + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "base_model": "MistralForCausalLM", + "revision": "04d8a90549d23fc6bd7f642064003592df51e9b3", + "submitted_time": "2024-07-17 17:26:49+00:00", + "num_params_billion": 12.2477824, + "language": "en_fr_de_es_it_pt_ru_zh_ja" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 4.83, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 4.390000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 4.83, + "stderr": 0.0 + } + ], + "average_accuracy": 4.683333333333334, + "best_prompt": 4.83, + "prompt_id": "p1", + "CPS": 4.822916 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 21.23, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 16.86, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 16.61, + "stderr": 0.0 + } + ], + "average_accuracy": 18.233333333333334, + "best_prompt": 21.23, + "prompt_id": "p1", + "CPS": 20.593807666666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_SK.json b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..8a39855286f44c49cdbc166b823a8bd7da6b6400 --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 12.570468000000002, + "config": { + "model_name": "mistralai/Mistral-Nemo-Instruct-2407", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SK", + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "base_model": "MistralForCausalLM", + "revision": "04d8a90549d23fc6bd7f642064003592df51e9b3", + "submitted_time": "2024-07-17 17:26:49+00:00", + "num_params_billion": 12.2477824, + "language": "en_fr_de_es_it_pt_ru_zh_ja" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 6.8500000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 8.44, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 6.8500000000000005, + "stderr": 0.0 + } + ], + "average_accuracy": 7.38, + "best_prompt": 8.44, + "prompt_id": "p2", + "CPS": 8.350536 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 16.96, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 13.96, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 16.96, + "stderr": 0.0 + } + ], + "average_accuracy": 15.96, + "best_prompt": 16.96, + "prompt_id": "p1", + "CPS": 16.7904 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_SL.json b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..990d4a209e6d2c60c4106696857ed0db633b4e2b --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_0_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 15.375161166666668, + "config": { + "model_name": "mistralai/Mistral-Nemo-Instruct-2407", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SL", + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "base_model": "MistralForCausalLM", + "revision": "04d8a90549d23fc6bd7f642064003592df51e9b3", + "submitted_time": "2024-07-17 17:26:49+00:00", + "num_params_billion": 12.2477824, + "language": "en_fr_de_es_it_pt_ru_zh_ja" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 8.61, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 8.05, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 8.61, + "stderr": 0.0 + } + ], + "average_accuracy": 8.423333333333334, + "best_prompt": 8.61, + "prompt_id": "p1", + "CPS": 8.593928 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 23.09, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 10.96, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 23.09, + "stderr": 0.0 + } + ], + "average_accuracy": 19.046666666666667, + "best_prompt": 23.09, + "prompt_id": "p1", + "CPS": 22.156394333333335 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_EN.json b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..6766f39c7bd90b083a82b24cc52307680638cfa3 --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 52.2740005, + "config": { + "model_name": "mistralai/Mistral-Nemo-Instruct-2407", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "EN", + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "base_model": "MistralForCausalLM", + "revision": "04d8a90549d23fc6bd7f642064003592df51e9b3", + "submitted_time": "2024-07-17 17:26:49+00:00", + "num_params_billion": 12.2477824, + "language": "en_fr_de_es_it_pt_ru_zh_ja" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 57.769999999999996, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 58.41, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 56.68, + "stderr": 0.0 + } + ], + "average_accuracy": 57.62, + "best_prompt": 58.41, + "prompt_id": "p2", + "CPS": 57.948561 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 34.82, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 50.080000000000005, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 44.49, + "stderr": 0.0 + } + ], + "average_accuracy": 43.13, + "best_prompt": 50.080000000000005, + "prompt_id": "p2", + "CPS": 46.59944 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_GR.json b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..9fdec97b12d8d1492873635e1bfeebdb91048e33 --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 40.65579733333333, + "config": { + "model_name": "mistralai/Mistral-Nemo-Instruct-2407", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "GR", + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "base_model": "MistralForCausalLM", + "revision": "04d8a90549d23fc6bd7f642064003592df51e9b3", + "submitted_time": "2024-07-17 17:26:49+00:00", + "num_params_billion": 12.2477824, + "language": "en_fr_de_es_it_pt_ru_zh_ja" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 50.81, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 49.88, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 50.81, + "stderr": 0.0 + } + ], + "average_accuracy": 50.5, + "best_prompt": 50.81, + "prompt_id": "p1", + "CPS": 50.652489 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 20.29, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 22.96, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 33.23, + "stderr": 0.0 + } + ], + "average_accuracy": 25.49333333333333, + "best_prompt": 33.23, + "prompt_id": "p3", + "CPS": 30.65910566666666 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_IT.json b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..08e21b80d8341ace471b2f0bd68d9635b281304d --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 51.6158585, + "config": { + "model_name": "mistralai/Mistral-Nemo-Instruct-2407", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "IT", + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "base_model": "MistralForCausalLM", + "revision": "04d8a90549d23fc6bd7f642064003592df51e9b3", + "submitted_time": "2024-07-17 17:26:49+00:00", + "num_params_billion": 12.2477824, + "language": "en_fr_de_es_it_pt_ru_zh_ja" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 64.3, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 64.37, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 64.57000000000001, + "stderr": 0.0 + } + ], + "average_accuracy": 64.41333333333334, + "best_prompt": 64.57000000000001, + "prompt_id": "p3", + "CPS": 64.46884033333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 27.08, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 40.99, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 38.6, + "stderr": 0.0 + } + ], + "average_accuracy": 35.556666666666665, + "best_prompt": 40.99, + "prompt_id": "p2", + "CPS": 38.76287666666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_PL.json b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..0858eae30fded0fd8fe8dae0f7e8da84b853d41c --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 36.890603, + "config": { + "model_name": "mistralai/Mistral-Nemo-Instruct-2407", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "PL", + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "base_model": "MistralForCausalLM", + "revision": "04d8a90549d23fc6bd7f642064003592df51e9b3", + "submitted_time": "2024-07-17 17:26:49+00:00", + "num_params_billion": 12.2477824, + "language": "en_fr_de_es_it_pt_ru_zh_ja" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 53.52, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 54.21, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 53.52, + "stderr": 0.0 + } + ], + "average_accuracy": 53.75, + "best_prompt": 54.21, + "prompt_id": "p2", + "CPS": 53.960634 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 18.63, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 18.55, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 20.01, + "stderr": 0.0 + } + ], + "average_accuracy": 19.063333333333333, + "best_prompt": 20.01, + "prompt_id": "p3", + "CPS": 19.820572 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_SK.json b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..c9cf698e283f1f943afe8db6c23ba7b85f5f8a00 --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 35.643439, + "config": { + "model_name": "mistralai/Mistral-Nemo-Instruct-2407", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SK", + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "base_model": "MistralForCausalLM", + "revision": "04d8a90549d23fc6bd7f642064003592df51e9b3", + "submitted_time": "2024-07-17 17:26:49+00:00", + "num_params_billion": 12.2477824, + "language": "en_fr_de_es_it_pt_ru_zh_ja" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 50.24999999999999, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 50.4, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 50.24999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 50.29999999999999, + "best_prompt": 50.4, + "prompt_id": "p2", + "CPS": 50.349599999999995 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 12.370000000000001, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 21.66, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 20.94, + "stderr": 0.0 + } + ], + "average_accuracy": 18.323333333333334, + "best_prompt": 21.66, + "prompt_id": "p2", + "CPS": 20.937278 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_SL.json b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..649e78cc8e120af115d138076515e54552cf01f5 --- /dev/null +++ b/e3c_llm_results/mistralai/Mistral-Nemo-Instruct-2407_10_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 36.596855166666664, + "config": { + "model_name": "mistralai/Mistral-Nemo-Instruct-2407", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SL", + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "base_model": "MistralForCausalLM", + "revision": "04d8a90549d23fc6bd7f642064003592df51e9b3", + "submitted_time": "2024-07-17 17:26:49+00:00", + "num_params_billion": 12.2477824, + "language": "en_fr_de_es_it_pt_ru_zh_ja" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 53.23, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 53.349999999999994, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 53.23, + "stderr": 0.0 + } + ], + "average_accuracy": 53.27, + "best_prompt": 53.349999999999994, + "prompt_id": "p2", + "CPS": 53.30732 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 13.900000000000002, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 20.57, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 17.27, + "stderr": 0.0 + } + ], + "average_accuracy": 17.246666666666666, + "best_prompt": 20.57, + "prompt_id": "p2", + "CPS": 19.886390333333335 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_EN.json b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..2413d2b84467652dbe116bf4931dd1e893652048 --- /dev/null +++ b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 32.621343333333336, + "config": { + "model_name": "tiiuae/Falcon3-10B-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "EN", + "model": "tiiuae/Falcon3-10B-Instruct", + "base_model": "LlamaForCausalLM", + "revision": "8799bc6aec0152757221dc6b272d824642db6202", + "submitted_time": "2024-12-14 05:17:25+00:00", + "num_params_billion": 10.30565376, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 22.7, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 27.089999999999996, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 29.959999999999997, + "stderr": 0.0 + } + ], + "average_accuracy": 26.583333333333332, + "best_prompt": 29.959999999999997, + "prompt_id": "p3", + "CPS": 28.948350666666666 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 21.57, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 38.35, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 38.48, + "stderr": 0.0 + } + ], + "average_accuracy": 32.800000000000004, + "best_prompt": 38.48, + "prompt_id": "p3", + "CPS": 36.294336 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_GR.json b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..a4b66d245149e88cc2acb30299099d44d3eda5e5 --- /dev/null +++ b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 14.248081500000001, + "config": { + "model_name": "tiiuae/Falcon3-10B-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "GR", + "model": "tiiuae/Falcon3-10B-Instruct", + "base_model": "LlamaForCausalLM", + "revision": "8799bc6aec0152757221dc6b272d824642db6202", + "submitted_time": "2024-12-14 05:17:25+00:00", + "num_params_billion": 10.30565376, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 21.3, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 4.95, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 21.3, + "stderr": 0.0 + } + ], + "average_accuracy": 15.850000000000001, + "best_prompt": 21.3, + "prompt_id": "p1", + "CPS": 20.13915 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 4.01, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 2.5, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 8.67, + "stderr": 0.0 + } + ], + "average_accuracy": 5.06, + "best_prompt": 8.67, + "prompt_id": "p3", + "CPS": 8.357013 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_IT.json b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..02b723a4ef36250504e231fbf43a4a1568f4627c --- /dev/null +++ b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 23.062588666666667, + "config": { + "model_name": "tiiuae/Falcon3-10B-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "IT", + "model": "tiiuae/Falcon3-10B-Instruct", + "base_model": "LlamaForCausalLM", + "revision": "8799bc6aec0152757221dc6b272d824642db6202", + "submitted_time": "2024-12-14 05:17:25+00:00", + "num_params_billion": 10.30565376, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 12.61, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 23.27, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 24.44, + "stderr": 0.0 + } + ], + "average_accuracy": 20.106666666666666, + "best_prompt": 24.44, + "prompt_id": "p3", + "CPS": 23.380933333333335 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 24.04, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 16.99, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 14.92, + "stderr": 0.0 + } + ], + "average_accuracy": 18.65, + "best_prompt": 24.04, + "prompt_id": "p1", + "CPS": 22.744244 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_PL.json b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..efc38268895e80d78afe735347e4cec25a212cf0 --- /dev/null +++ b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 19.314392833333333, + "config": { + "model_name": "tiiuae/Falcon3-10B-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "PL", + "model": "tiiuae/Falcon3-10B-Instruct", + "base_model": "LlamaForCausalLM", + "revision": "8799bc6aec0152757221dc6b272d824642db6202", + "submitted_time": "2024-12-14 05:17:25+00:00", + "num_params_billion": 10.30565376, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 24.52, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 23.380000000000003, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 24.52, + "stderr": 0.0 + } + ], + "average_accuracy": 24.14, + "best_prompt": 24.52, + "prompt_id": "p1", + "CPS": 24.426824 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 15.010000000000002, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 1.23, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 12.64, + "stderr": 0.0 + } + ], + "average_accuracy": 9.626666666666667, + "best_prompt": 15.010000000000002, + "prompt_id": "p1", + "CPS": 14.201961666666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_SK.json b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..80f14bb3307e33e3a63e248e58608f69d5bd07f7 --- /dev/null +++ b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 16.691507333333334, + "config": { + "model_name": "tiiuae/Falcon3-10B-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SK", + "model": "tiiuae/Falcon3-10B-Instruct", + "base_model": "LlamaForCausalLM", + "revision": "8799bc6aec0152757221dc6b272d824642db6202", + "submitted_time": "2024-12-14 05:17:25+00:00", + "num_params_billion": 10.30565376, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 27.169999999999998, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 31.78, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 27.169999999999998, + "stderr": 0.0 + } + ], + "average_accuracy": 28.706666666666667, + "best_prompt": 31.78, + "prompt_id": "p2", + "CPS": 30.803294666666666 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 1.43, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 2.6, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 1.43, + "stderr": 0.0 + } + ], + "average_accuracy": 1.82, + "best_prompt": 2.6, + "prompt_id": "p2", + "CPS": 2.57972 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_SL.json b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..fc0d9538afb21063d53669859ec1ca6d971c25ed --- /dev/null +++ b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_0_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 12.605178333333333, + "config": { + "model_name": "tiiuae/Falcon3-10B-Instruct", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SL", + "model": "tiiuae/Falcon3-10B-Instruct", + "base_model": "LlamaForCausalLM", + "revision": "8799bc6aec0152757221dc6b272d824642db6202", + "submitted_time": "2024-12-14 05:17:25+00:00", + "num_params_billion": 10.30565376, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 25.19, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 18.529999999999998, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 25.19, + "stderr": 0.0 + } + ], + "average_accuracy": 22.97, + "best_prompt": 25.19, + "prompt_id": "p1", + "CPS": 24.630782 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.47000000000000003, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.58, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.47000000000000003, + "stderr": 0.0 + } + ], + "average_accuracy": 0.5066666666666667, + "best_prompt": 0.58, + "prompt_id": "p2", + "CPS": 0.5795746666666666 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_EN.json b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..23950e8783aacd0b8a8ec07000e5a327b6aa8f2c --- /dev/null +++ b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 55.751357999999996, + "config": { + "model_name": "tiiuae/Falcon3-10B-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "EN", + "model": "tiiuae/Falcon3-10B-Instruct", + "base_model": "LlamaForCausalLM", + "revision": "8799bc6aec0152757221dc6b272d824642db6202", + "submitted_time": "2024-12-14 05:17:25+00:00", + "num_params_billion": 10.30565376, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 58.4, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 54.21, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 59.28, + "stderr": 0.0 + } + ], + "average_accuracy": 57.29666666666666, + "best_prompt": 59.28, + "prompt_id": "p3", + "CPS": 58.104279999999996 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 43.35, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 55.86, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 55.15, + "stderr": 0.0 + } + ], + "average_accuracy": 51.45333333333334, + "best_prompt": 55.86, + "prompt_id": "p2", + "CPS": 53.398436000000004 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_GR.json b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..6b1b7aec32ec1dd607ebf47e8a0831d9666895a2 --- /dev/null +++ b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 36.36378083333333, + "config": { + "model_name": "tiiuae/Falcon3-10B-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "GR", + "model": "tiiuae/Falcon3-10B-Instruct", + "base_model": "LlamaForCausalLM", + "revision": "8799bc6aec0152757221dc6b272d824642db6202", + "submitted_time": "2024-12-14 05:17:25+00:00", + "num_params_billion": 10.30565376, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 33.45, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 36.55, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 33.45, + "stderr": 0.0 + } + ], + "average_accuracy": 34.483333333333334, + "best_prompt": 36.55, + "prompt_id": "p2", + "CPS": 35.79463333333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 37.49, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 37.55, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 32.68, + "stderr": 0.0 + } + ], + "average_accuracy": 35.906666666666666, + "best_prompt": 37.55, + "prompt_id": "p2", + "CPS": 36.93292833333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_IT.json b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..63e48dbad5f85f4bb69b03b6bd5ae53b7fb7e6c6 --- /dev/null +++ b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 55.480365666666664, + "config": { + "model_name": "tiiuae/Falcon3-10B-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "IT", + "model": "tiiuae/Falcon3-10B-Instruct", + "base_model": "LlamaForCausalLM", + "revision": "8799bc6aec0152757221dc6b272d824642db6202", + "submitted_time": "2024-12-14 05:17:25+00:00", + "num_params_billion": 10.30565376, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 58.209999999999994, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 54.32, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 56.220000000000006, + "stderr": 0.0 + } + ], + "average_accuracy": 56.25, + "best_prompt": 58.209999999999994, + "prompt_id": "p1", + "CPS": 57.069084 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 46.22, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 54.58, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 55.97, + "stderr": 0.0 + } + ], + "average_accuracy": 52.25666666666666, + "best_prompt": 55.97, + "prompt_id": "p3", + "CPS": 53.89164733333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_PL.json b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..47845f40149bf7b1ea6d461838c06615f65e1825 --- /dev/null +++ b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 48.75862866666667, + "config": { + "model_name": "tiiuae/Falcon3-10B-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "PL", + "model": "tiiuae/Falcon3-10B-Instruct", + "base_model": "LlamaForCausalLM", + "revision": "8799bc6aec0152757221dc6b272d824642db6202", + "submitted_time": "2024-12-14 05:17:25+00:00", + "num_params_billion": 10.30565376, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 43.04, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 41.23, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 43.04, + "stderr": 0.0 + } + ], + "average_accuracy": 42.43666666666667, + "best_prompt": 43.04, + "prompt_id": "p1", + "CPS": 42.78032533333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.29, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 55.71, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 54.89000000000001, + "stderr": 0.0 + } + ], + "average_accuracy": 53.96333333333334, + "best_prompt": 55.71, + "prompt_id": "p2", + "CPS": 54.736932 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_SK.json b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..035aa9f55baa6e42440e9f396d4557a58d41b12b --- /dev/null +++ b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 44.8562175, + "config": { + "model_name": "tiiuae/Falcon3-10B-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SK", + "model": "tiiuae/Falcon3-10B-Instruct", + "base_model": "LlamaForCausalLM", + "revision": "8799bc6aec0152757221dc6b272d824642db6202", + "submitted_time": "2024-12-14 05:17:25+00:00", + "num_params_billion": 10.30565376, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 45.45, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 41.160000000000004, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 45.45, + "stderr": 0.0 + } + ], + "average_accuracy": 44.02, + "best_prompt": 45.45, + "prompt_id": "p1", + "CPS": 44.800065000000004 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 37.5, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 46.949999999999996, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 43.38, + "stderr": 0.0 + } + ], + "average_accuracy": 42.61, + "best_prompt": 46.949999999999996, + "prompt_id": "p2", + "CPS": 44.912369999999996 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_SL.json b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..bc1d01f3f803fdbb9dbefcfdea59c98d748cf150 --- /dev/null +++ b/e3c_llm_results/tiiuae/Falcon3-10B-Instruct_10_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 38.88441916666667, + "config": { + "model_name": "tiiuae/Falcon3-10B-Instruct", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SL", + "model": "tiiuae/Falcon3-10B-Instruct", + "base_model": "LlamaForCausalLM", + "revision": "8799bc6aec0152757221dc6b272d824642db6202", + "submitted_time": "2024-12-14 05:17:25+00:00", + "num_params_billion": 10.30565376, + "language": "" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 41.21, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 39.09, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 41.21, + "stderr": 0.0 + } + ], + "average_accuracy": 40.50333333333334, + "best_prompt": 41.21, + "prompt_id": "p1", + "CPS": 40.918782666666665 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 23.23, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 30.12, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 40.63, + "stderr": 0.0 + } + ], + "average_accuracy": 31.326666666666668, + "best_prompt": 40.63, + "prompt_id": "p3", + "CPS": 36.85005566666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/unsloth/phi-4_0_EN.json b/e3c_llm_results/unsloth/phi-4_0_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..13f50870e15d60be47bcbe3372c8f160253be831 --- /dev/null +++ b/e3c_llm_results/unsloth/phi-4_0_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 23.598540333333336, + "config": { + "model_name": "unsloth/phi-4", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "EN", + "model": "unsloth/phi-4", + "base_model": "LlamaForCausalLM", + "revision": "c6220bde10fff762dbd72c3331894aa4cade249d", + "submitted_time": "2025-01-08 21:56:16+00:00", + "num_params_billion": 14.6595072, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 2.52, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 5.72, + "stderr": 0.0 + } + ], + "average_accuracy": 2.7466666666666666, + "best_prompt": 5.72, + "prompt_id": "p3", + "CPS": 5.549925333333333 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 40.22, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 42.19, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 40.300000000000004, + "stderr": 0.0 + } + ], + "average_accuracy": 40.903333333333336, + "best_prompt": 42.19, + "prompt_id": "p2", + "CPS": 41.64715533333334 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/unsloth/phi-4_0_GR.json b/e3c_llm_results/unsloth/phi-4_0_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..28b5b848a74b32573aa16eeec5cd9fbc8d9560f1 --- /dev/null +++ b/e3c_llm_results/unsloth/phi-4_0_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 13.214538500000002, + "config": { + "model_name": "unsloth/phi-4", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "GR", + "model": "unsloth/phi-4", + "base_model": "LlamaForCausalLM", + "revision": "c6220bde10fff762dbd72c3331894aa4cade249d", + "submitted_time": "2025-01-08 21:56:16+00:00", + "num_params_billion": 14.6595072, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + } + ], + "average_accuracy": 0.0, + "best_prompt": 0.0, + "prompt_id": "p1", + "CPS": 0.0 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 29.01, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 22.08, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 9.25, + "stderr": 0.0 + } + ], + "average_accuracy": 20.113333333333333, + "best_prompt": 29.01, + "prompt_id": "p1", + "CPS": 26.429077000000003 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/unsloth/phi-4_0_IT.json b/e3c_llm_results/unsloth/phi-4_0_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..34827aa451f82d0a0a881b3dfcdad74347c8e88d --- /dev/null +++ b/e3c_llm_results/unsloth/phi-4_0_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 32.617002166666666, + "config": { + "model_name": "unsloth/phi-4", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "IT", + "model": "unsloth/phi-4", + "base_model": "LlamaForCausalLM", + "revision": "c6220bde10fff762dbd72c3331894aa4cade249d", + "submitted_time": "2025-01-08 21:56:16+00:00", + "num_params_billion": 14.6595072, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 0.0, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 17.24, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 34.28, + "stderr": 0.0 + } + ], + "average_accuracy": 17.173333333333332, + "best_prompt": 34.28, + "prompt_id": "p3", + "CPS": 28.415834666666665 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 33.54, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 37.37, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 36.77, + "stderr": 0.0 + } + ], + "average_accuracy": 35.89333333333334, + "best_prompt": 37.37, + "prompt_id": "p2", + "CPS": 36.81816966666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/unsloth/phi-4_0_PL.json b/e3c_llm_results/unsloth/phi-4_0_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..a87b57aa990def590e05dd8044d0f0c302d89bb9 --- /dev/null +++ b/e3c_llm_results/unsloth/phi-4_0_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 20.92978433333333, + "config": { + "model_name": "unsloth/phi-4", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "PL", + "model": "unsloth/phi-4", + "base_model": "LlamaForCausalLM", + "revision": "c6220bde10fff762dbd72c3331894aa4cade249d", + "submitted_time": "2025-01-08 21:56:16+00:00", + "num_params_billion": 14.6595072, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 2.36, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 3.66, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 2.36, + "stderr": 0.0 + } + ], + "average_accuracy": 2.793333333333333, + "best_prompt": 3.66, + "prompt_id": "p2", + "CPS": 3.62828 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 37.99, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 38.29, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 38.129999999999995, + "stderr": 0.0 + } + ], + "average_accuracy": 38.13666666666666, + "best_prompt": 38.29, + "prompt_id": "p2", + "CPS": 38.231288666666664 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/unsloth/phi-4_0_SK.json b/e3c_llm_results/unsloth/phi-4_0_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..a50726599101df7722e364fe1065962a8e836a80 --- /dev/null +++ b/e3c_llm_results/unsloth/phi-4_0_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 21.629032, + "config": { + "model_name": "unsloth/phi-4", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SK", + "model": "unsloth/phi-4", + "base_model": "LlamaForCausalLM", + "revision": "c6220bde10fff762dbd72c3331894aa4cade249d", + "submitted_time": "2025-01-08 21:56:16+00:00", + "num_params_billion": 14.6595072, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 3.16, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 10.7, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 3.16, + "stderr": 0.0 + } + ], + "average_accuracy": 5.673333333333333, + "best_prompt": 10.7, + "prompt_id": "p2", + "CPS": 10.162146666666667 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 32.519999999999996, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 33.26, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 32.519999999999996, + "stderr": 0.0 + } + ], + "average_accuracy": 32.76666666666666, + "best_prompt": 33.26, + "prompt_id": "p2", + "CPS": 33.09591733333333 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/unsloth/phi-4_0_SL.json b/e3c_llm_results/unsloth/phi-4_0_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..683c18670b665b52f01bd0d957a2f5d06ae32fe3 --- /dev/null +++ b/e3c_llm_results/unsloth/phi-4_0_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 28.7078975, + "config": { + "model_name": "unsloth/phi-4", + "num_fewshot": "0", + "batch_size": 1, + "LANG": "SL", + "model": "unsloth/phi-4", + "base_model": "LlamaForCausalLM", + "revision": "c6220bde10fff762dbd72c3331894aa4cade249d", + "submitted_time": "2025-01-08 21:56:16+00:00", + "num_params_billion": 14.6595072, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 28.7, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 9.81, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 28.7, + "stderr": 0.0 + } + ], + "average_accuracy": 22.403333333333332, + "best_prompt": 28.7, + "prompt_id": "p1", + "CPS": 26.892856666666667 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 32.09, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 17.44, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 32.09, + "stderr": 0.0 + } + ], + "average_accuracy": 27.206666666666667, + "best_prompt": 32.09, + "prompt_id": "p1", + "CPS": 30.522938333333336 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/unsloth/phi-4_10_EN.json b/e3c_llm_results/unsloth/phi-4_10_EN.json new file mode 100644 index 0000000000000000000000000000000000000000..2eca9798eb6ce904c24156d5df79987b7590d523 --- /dev/null +++ b/e3c_llm_results/unsloth/phi-4_10_EN.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 57.6138785, + "config": { + "model_name": "unsloth/phi-4", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "EN", + "model": "unsloth/phi-4", + "base_model": "LlamaForCausalLM", + "revision": "c6220bde10fff762dbd72c3331894aa4cade249d", + "submitted_time": "2025-01-08 21:56:16+00:00", + "num_params_billion": 14.6595072, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 60.980000000000004, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 57.11000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 61.41, + "stderr": 0.0 + } + ], + "average_accuracy": 59.833333333333336, + "best_prompt": 61.41, + "prompt_id": "p3", + "CPS": 60.441769 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 49.120000000000005, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 56.26, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 55.54, + "stderr": 0.0 + } + ], + "average_accuracy": 53.64000000000001, + "best_prompt": 56.26, + "prompt_id": "p2", + "CPS": 54.785988 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/unsloth/phi-4_10_GR.json b/e3c_llm_results/unsloth/phi-4_10_GR.json new file mode 100644 index 0000000000000000000000000000000000000000..484557f271eb1144b8930243fdc0e9e5e56a0a5b --- /dev/null +++ b/e3c_llm_results/unsloth/phi-4_10_GR.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 55.776253, + "config": { + "model_name": "unsloth/phi-4", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "GR", + "model": "unsloth/phi-4", + "base_model": "LlamaForCausalLM", + "revision": "c6220bde10fff762dbd72c3331894aa4cade249d", + "submitted_time": "2025-01-08 21:56:16+00:00", + "num_params_billion": 14.6595072, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 57.17, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 56.11000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 57.17, + "stderr": 0.0 + } + ], + "average_accuracy": 56.81666666666667, + "best_prompt": 57.17, + "prompt_id": "p1", + "CPS": 56.96799933333334 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 49.35, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 52.61, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 56.779999999999994, + "stderr": 0.0 + } + ], + "average_accuracy": 52.913333333333334, + "best_prompt": 56.779999999999994, + "prompt_id": "p3", + "CPS": 54.58450666666666 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/unsloth/phi-4_10_IT.json b/e3c_llm_results/unsloth/phi-4_10_IT.json new file mode 100644 index 0000000000000000000000000000000000000000..f5f563b3c379658ff5dbd08e0e2083c1ff6a853b --- /dev/null +++ b/e3c_llm_results/unsloth/phi-4_10_IT.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 62.7742775, + "config": { + "model_name": "unsloth/phi-4", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "IT", + "model": "unsloth/phi-4", + "base_model": "LlamaForCausalLM", + "revision": "c6220bde10fff762dbd72c3331894aa4cade249d", + "submitted_time": "2025-01-08 21:56:16+00:00", + "num_params_billion": 14.6595072, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 66.47, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 67.32000000000001, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 68.97, + "stderr": 0.0 + } + ], + "average_accuracy": 67.58666666666666, + "best_prompt": 68.97, + "prompt_id": "p3", + "CPS": 68.01591499999999 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 56.08, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 58.199999999999996, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 56.879999999999995, + "stderr": 0.0 + } + ], + "average_accuracy": 57.053333333333335, + "best_prompt": 58.199999999999996, + "prompt_id": "p2", + "CPS": 57.53264 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/unsloth/phi-4_10_PL.json b/e3c_llm_results/unsloth/phi-4_10_PL.json new file mode 100644 index 0000000000000000000000000000000000000000..0f726576d99c14670bfd06a98251a5c7c9a315d2 --- /dev/null +++ b/e3c_llm_results/unsloth/phi-4_10_PL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 56.63946383333333, + "config": { + "model_name": "unsloth/phi-4", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "PL", + "model": "unsloth/phi-4", + "base_model": "LlamaForCausalLM", + "revision": "c6220bde10fff762dbd72c3331894aa4cade249d", + "submitted_time": "2025-01-08 21:56:16+00:00", + "num_params_billion": 14.6595072, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 55.489999999999995, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 53.239999999999995, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 55.489999999999995, + "stderr": 0.0 + } + ], + "average_accuracy": 54.73999999999999, + "best_prompt": 55.489999999999995, + "prompt_id": "p1", + "CPS": 55.07382499999999 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 54.230000000000004, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 57.599999999999994, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 59.72, + "stderr": 0.0 + } + ], + "average_accuracy": 57.18333333333334, + "best_prompt": 59.72, + "prompt_id": "p3", + "CPS": 58.20510266666667 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/unsloth/phi-4_10_SK.json b/e3c_llm_results/unsloth/phi-4_10_SK.json new file mode 100644 index 0000000000000000000000000000000000000000..eae9b8f7dc903b7485ef003c47312654ce4c5036 --- /dev/null +++ b/e3c_llm_results/unsloth/phi-4_10_SK.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 54.49931766666667, + "config": { + "model_name": "unsloth/phi-4", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SK", + "model": "unsloth/phi-4", + "base_model": "LlamaForCausalLM", + "revision": "c6220bde10fff762dbd72c3331894aa4cade249d", + "submitted_time": "2025-01-08 21:56:16+00:00", + "num_params_billion": 14.6595072, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 55.61000000000001, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 54.49, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 55.61000000000001, + "stderr": 0.0 + } + ], + "average_accuracy": 55.23666666666667, + "best_prompt": 55.61000000000001, + "prompt_id": "p1", + "CPS": 55.40238933333334 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.06, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 49.94, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 55.410000000000004, + "stderr": 0.0 + } + ], + "average_accuracy": 52.13666666666666, + "best_prompt": 55.410000000000004, + "prompt_id": "p3", + "CPS": 53.596246 + } + } +} \ No newline at end of file diff --git a/e3c_llm_results/unsloth/phi-4_10_SL.json b/e3c_llm_results/unsloth/phi-4_10_SL.json new file mode 100644 index 0000000000000000000000000000000000000000..78e30ae98107c63ec0916e5a5b8d272498025244 --- /dev/null +++ b/e3c_llm_results/unsloth/phi-4_10_SL.json @@ -0,0 +1,69 @@ +{ + "average_CPS": 55.04669683333333, + "config": { + "model_name": "unsloth/phi-4", + "num_fewshot": "10", + "batch_size": 1, + "LANG": "SL", + "model": "unsloth/phi-4", + "base_model": "LlamaForCausalLM", + "revision": "c6220bde10fff762dbd72c3331894aa4cade249d", + "submitted_time": "2025-01-08 21:56:16+00:00", + "num_params_billion": 14.6595072, + "language": "en" + }, + "tasks": { + "NER": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 55.86, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 55.58, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 55.86, + "stderr": 0.0 + } + ], + "average_accuracy": 55.76666666666667, + "best_prompt": 55.86, + "prompt_id": "p1", + "CPS": 55.807864 + }, + "RE": { + "prompts": [ + { + "prompt": "p1", + "metric": "f1", + "value": 51.17, + "stderr": 0.0 + }, + { + "prompt": "p2", + "metric": "f1", + "value": 52.32, + "stderr": 0.0 + }, + { + "prompt": "p3", + "metric": "f1", + "value": 55.78999999999999, + "stderr": 0.0 + } + ], + "average_accuracy": 53.093333333333334, + "best_prompt": 55.78999999999999, + "prompt_id": "p3", + "CPS": 54.28552966666666 + } + } +} \ No newline at end of file diff --git a/example_app.py b/example_app.py new file mode 100644 index 0000000000000000000000000000000000000000..e6e712f9ac66b7f5ae4305c0615540fde9141d85 --- /dev/null +++ b/example_app.py @@ -0,0 +1,324 @@ +import gradio as gr +from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns +import pandas as pd +from apscheduler.schedulers.background import BackgroundScheduler +from huggingface_hub import snapshot_download + +from src.about import ( + CITATION_BUTTON_LABEL, + CITATION_BUTTON_TEXT, + EVALUATION_QUEUE_TEXT, + INTRODUCTION_TEXT, + LLM_BENCHMARKS_TEXT, + TITLE, +) + +from src.tasks import ( + TE_DESCRIPTION, +) + +from src.display.css_html_js import custom_css +from src.display.utils import ( + BENCHMARK_COLS, + COLS, + EVAL_COLS, + EVAL_TYPES, + AutoEvalColumn, + ModelType, + fields, + WeightType, + Precision +) +from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN +from src.populate import get_evaluation_queue_df, get_leaderboard_df +from src.submission.submit import add_new_eval + + +def restart_space(): + API.restart_space(repo_id=REPO_ID) + +### Space initialisation +try: + print(EVAL_REQUESTS_PATH) + snapshot_download( + repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN + ) +except Exception: + restart_space() +try: + print(EVAL_RESULTS_PATH) + snapshot_download( + repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN + ) +except Exception: + restart_space() + + +LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS) + +( + finished_eval_queue_df, + running_eval_queue_df, + pending_eval_queue_df, +) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS) + +def init_leaderboard(dataframe): + print(dataframe) + if dataframe is None or dataframe.empty: + raise ValueError("Leaderboard DataFrame is empty or None.") + return Leaderboard( + value=dataframe, + datatype=[c.type for c in fields(AutoEvalColumn)], + select_columns=SelectColumns( + default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default], + cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden], + label="Select Columns to Display:", + ), + search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name], + hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden], + filter_columns=[ + ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"), + ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"), + ColumnFilter( + AutoEvalColumn.params.name, + type="slider", + min=0.01, + max=150, + label="Select the number of parameters (B)", + ), + ColumnFilter( + AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True + ), + ], + bool_checkboxgroup_label="Hide models", + interactive=False, + ) + + +def init_leaderboard2(dataframe, default_selection=None, hidden_columns=None): + + print("entrato===============================================") + + if dataframe is None or dataframe.empty: + raise ValueError("Leaderboard DataFrame is empty or None.") + return Leaderboard( + value=dataframe, + datatype=[c.type for c in fields(AutoEvalColumn)], + select_columns=SelectColumns( + default_selection=default_selection or [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default], + cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden], + label="Select Columns to Display:", + ), + search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name], + hide_columns=hidden_columns or [c.name for c in fields(AutoEvalColumn) if c.hidden], + filter_columns=[ + ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"), + ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"), + ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0.01, max=150, label="Select the number of parameters (B)"), + ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True), + ], + bool_checkboxgroup_label="Hide models", + interactive=False, + ) + + +demo = gr.Blocks(css=custom_css) +with demo: + gr.HTML(TITLE) + gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") + + with gr.Tabs(elem_classes="tab-buttons") as tabs: + with gr.TabItem("🏅 EVALITA-LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0): + #leaderboard = init_leaderboard(LEADERBOARD_DF) + + leaderboard = init_leaderboard2( + LEADERBOARD_DF, + default_selection=['T', 'Model', "Average ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"], + hidden_columns=[col for col in LEADERBOARD_DF.columns if + col not in ['T', 'Model', "Average ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL" ]] + ) + + + with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2): + gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") + + with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3): + with gr.Column(): + with gr.Row(): + gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text") + + with gr.Column(): + with gr.Accordion( + f"✅ Finished Evaluations ({len(finished_eval_queue_df)})", + open=False, + ): + with gr.Row(): + finished_eval_table = gr.components.Dataframe( + value=finished_eval_queue_df, + headers=EVAL_COLS, + datatype=EVAL_TYPES, + row_count=5, + ) + with gr.Accordion( + f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})", + open=False, + ): + with gr.Row(): + running_eval_table = gr.components.Dataframe( + value=running_eval_queue_df, + headers=EVAL_COLS, + datatype=EVAL_TYPES, + row_count=5, + ) + + with gr.Accordion( + f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})", + open=False, + ): + with gr.Row(): + pending_eval_table = gr.components.Dataframe( + value=pending_eval_queue_df, + headers=EVAL_COLS, + datatype=EVAL_TYPES, + row_count=5, + ) + with gr.Row(): + gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text") + + with gr.Row(): + with gr.Column(): + model_name_textbox = gr.Textbox(label="Model name") + revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main") + model_type = gr.Dropdown( + choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown], + label="Model type", + multiselect=False, + value=None, + interactive=True, + ) + + with gr.Column(): + precision = gr.Dropdown( + choices=[i.value.name for i in Precision if i != Precision.Unknown], + label="Precision", + multiselect=False, + value="float16", + interactive=True, + ) + weight_type = gr.Dropdown( + choices=[i.value.name for i in WeightType], + label="Weights type", + multiselect=False, + value="Original", + interactive=True, + ) + base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)") + + submit_button = gr.Button("Submit Eval") + submission_result = gr.Markdown() + submit_button.click( + add_new_eval, + [ + model_name_textbox, + base_model_name_textbox, + revision_name_textbox, + precision, + weight_type, + model_type, + ], + submission_result, + ) + + + with gr.TabItem("TE", elem_id="llm-benchmark-tab-table", id=4): + gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text") + #leaderboard = init_leaderboard(LEADERBOARD_DF) + + LEADERBOARD_DF_TE = LEADERBOARD_DF.rename(columns={"TE Prompt Average": "Prompt Average", + "TE Best Prompt": "Best Prompt", + "TE Best Prompt Id": "Best Prompt Id", + "TE": "Combined Performance"}) + + leaderboard = init_leaderboard2( + LEADERBOARD_DF_TE, + default_selection=['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id'], + hidden_columns=[col for col in LEADERBOARD_DF.columns if + col not in ['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id']] + ) + + + with gr.TabItem("SA", elem_id="llm-benchmark-tab-table", id=5): + gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text") + + LEADERBOARD_DF_SA = LEADERBOARD_DF.rename(columns={"SA Prompt Average": "Prompt Average", + "SA Best Prompt": "Best Prompt", + "SA Best Prompt Id": "Best Prompt Id", + "SA": "Combined Performance"}) + + leaderboard = init_leaderboard2( + LEADERBOARD_DF_SA, + default_selection=['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', + 'Best Prompt Id'], + hidden_columns=[col for col in LEADERBOARD_DF.columns if + col not in ['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', + 'Best Prompt Id']] + ) + + + + + with gr.TabItem("HS", elem_id="llm-benchmark-tab-table", id=6): + gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text") + + LEADERBOARD_DF_HS = LEADERBOARD_DF.rename(columns={"HS Prompt Average": "Prompt Average", + "HS Best Prompt": "Best Prompt", + "HS Best Prompt Id": "Best Prompt Id", + "HS": "Combined Performance"}) + + leaderboard = init_leaderboard2( + LEADERBOARD_DF_HS, + default_selection=['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', + 'Best Prompt Id'], + hidden_columns=[col for col in LEADERBOARD_DF.columns if + col not in ['T', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', + 'Best Prompt Id']] + ) + + + + with gr.TabItem("AT", elem_id="llm-benchmark-tab-table", id=7): + gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text") + + with gr.TabItem("WIC", elem_id="llm-benchmark-tab-table", id=8): + gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text") + + with gr.TabItem("FAQ", elem_id="llm-benchmark-tab-table", id=9): + gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text") + + with gr.TabItem("LS", elem_id="llm-benchmark-tab-table", id=10): + gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text") + + with gr.TabItem("SU", elem_id="llm-benchmark-tab-table", id=11): + gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text") + + with gr.TabItem("NER", elem_id="llm-benchmark-tab-table", id=12): + gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text") + + with gr.TabItem("REL", elem_id="llm-benchmark-tab-table", id=13): + gr.Markdown(TE_DESCRIPTION, elem_classes="markdown-text") + + + with gr.Row(): + with gr.Accordion("📙 Citation", open=False): + citation_button = gr.Textbox( + value=CITATION_BUTTON_TEXT, + label=CITATION_BUTTON_LABEL, + lines=20, + elem_id="citation-button", + show_copy_button=True, + ) + +scheduler = BackgroundScheduler() +scheduler.add_job(restart_space, "interval", seconds=1800) +scheduler.start() +demo.queue(default_concurrency_limit=40).launch() \ No newline at end of file diff --git a/example_app2.py b/example_app2.py new file mode 100644 index 0000000000000000000000000000000000000000..9268e66807d66f4d99c6c97a748691f46972e4e8 --- /dev/null +++ b/example_app2.py @@ -0,0 +1,216 @@ +import gradio as gr +from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns +import pandas as pd +from apscheduler.schedulers.background import BackgroundScheduler +from huggingface_hub import snapshot_download + +from src.about import ( + CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, EVALUATION_QUEUE_TEXT, + INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TITLE +) +from src.tasks import TASK_DESCRIPTIONS, MEASURE_DESCRIPTION +from src.display.css_html_js import custom_css +from src.display.utils import ( + BENCHMARK_COLS, COLS, EVAL_COLS, EVAL_TYPES, AutoEvalColumn, + ModelType, fields, WeightType, Precision +) +from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN +from src.populate import get_evaluation_queue_df, get_leaderboard_df +from src.submission.submit import add_new_eval + + + + +# Define the task icons and names +TASK_ICONS = { + "TE": "📊", # Textual Entailment + "SA": "😃", # Sentiment Analysis + "HS": "⚠️", # Hate Speech + "AT": "🏥", # Admission Test + "WIC": "🔤", # Word in Context + "FAQ": "❓", # Frequently Asked Questions + "LS": "🔄", # Lexical Substitution + "SU": "📝", # Summarization + "NER": "🏷️", # Named Entity Recognition + "REL": "🔗", # Relation Extraction +} + +TASK_NAMES = { + "TE": "Textual Entailment", + "SA": "Sentiment Analysis", + "HS": "Hate Speech", + "AT": "Admission Test", + "WIC": "Word in Context", + "FAQ": "Frequently Asked Questions", + "LS": "Lexical Substitution", + "SU": "Summarization", + "NER": "Named Entity Recognition", + "REL": "Relation Extraction", +} + + +# Tooltip descriptions for each task +TASK_TOOLTIPS = { + "TE": "Identify logical relationships between two text segments.", + "SA": "Classify the sentiment (positive, negative, neutral) of a text.", + "HS": "Detect hate speech in a text.", + "AT": "Classify whether a clinical statement pertains to an admission test.", + "WIC": "Identify words in context and their meaning.", + "FAQ": "Answer frequently asked questions based on given text.", + "LS": "Identify alternative words in a given context.", + "SU": "Summarize long text into a shorter version.", + "NER": "Identify named entities (e.g., persons, locations, organizations) in text.", + "REL": "Extract and link laboratory test results to the respective tests in clinical narratives.", +} + + + + +def restart_space(): + """Restart the Hugging Face space.""" + API.restart_space(repo_id=REPO_ID) + + +def download_snapshot(repo, local_dir): + """Try to download a snapshot from the Hugging Face Hub, restarting space on failure.""" + try: + print(f"Downloading from {repo} to {local_dir}...") + snapshot_download(repo_id=repo, local_dir=local_dir, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN) + except Exception as e: + print(f"Error downloading {repo}: {e}") + restart_space() + + +# Space initialization +download_snapshot(QUEUE_REPO, EVAL_REQUESTS_PATH) +download_snapshot(RESULTS_REPO, EVAL_RESULTS_PATH) + +# Load leaderboard and evaluation queue data +LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS) +finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS) + + +def init_leaderboard(dataframe, default_selection=None, hidden_columns=None): + """Initialize a leaderboard with specific columns.""" + if dataframe is None or dataframe.empty: + raise ValueError("Leaderboard DataFrame is empty or None.") + + return Leaderboard( + value=dataframe, + datatype=[c.type for c in fields(AutoEvalColumn)], + select_columns=SelectColumns( + default_selection=default_selection or [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default], + cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden], + label="Select Columns to Display:", + ), + search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name], + hide_columns=hidden_columns or [c.name for c in fields(AutoEvalColumn) if c.hidden], + filter_columns=[ + #ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"), + ColumnFilter(AutoEvalColumn.fewshot_type.name, type="checkboxgroup", label="Few-Shot Learning (FS)"), + #ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"), + ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0.01, max=150, label="Select the number of parameters (B)"), + #ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True), + ], + bool_checkboxgroup_label="Hide models", + interactive=False, + ) + + +def prepare_leaderboard_df(df, task_prefix): + """Rename columns for a specific task to a standard format.""" + return df.rename(columns={ + f"{task_prefix} Prompt Average": "Prompt Average", + f"{task_prefix} Best Prompt": "Best Prompt", + f"{task_prefix} Best Prompt Id": "Best Prompt Id", + task_prefix: "Combined Performance" + }) + + +demo = gr.Blocks(css=custom_css) +with demo: + gr.HTML(TITLE) + gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") + + with gr.Tabs(elem_classes="tab-buttons") as tabs: + # Main leaderboard tab + with gr.TabItem("🏅 EVALITA-LLM Benchmark", elem_id="llm-benchmark-tab-table"): + leaderboard = init_leaderboard( + LEADERBOARD_DF, + default_selection=['FS', 'Model', "Avg. Combined Performance ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"], + hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in + ['FS', 'Model', "Avg. Combined Performance ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]] + ) + + # About tab + with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table"): + gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") + + ''' + # Submission tab + with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table"): + gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text") + + for queue_name, queue_df in [ + ("✅ Finished Evaluations", finished_eval_queue_df), + ("🔄 Running Evaluation Queue", running_eval_queue_df), + ("⏳ Pending Evaluation Queue", pending_eval_queue_df) + ]: + with gr.Accordion(f"{queue_name} ({len(queue_df)})", open=False): + gr.components.Dataframe(value=queue_df, headers=EVAL_COLS, datatype=EVAL_TYPES, row_count=5) + + gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text") + with gr.Row(): + model_name_textbox = gr.Textbox(label="Model name") + revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main") + model_type = gr.Dropdown(choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown], + label="Model type", multiselect=False, interactive=True) + precision = gr.Dropdown(choices=[i.value.name for i in Precision if i != Precision.Unknown], + label="Precision", multiselect=False, value="float16", interactive=True) + weight_type = gr.Dropdown(choices=[i.value.name for i in WeightType], + label="Weights type", multiselect=False, value="Original", interactive=True) + base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)") + + submit_button = gr.Button("Submit Eval") + submission_result = gr.Markdown() + submit_button.click( + add_new_eval, + [model_name_textbox, base_model_name_textbox, revision_name_textbox, precision, weight_type, model_type], + submission_result, + ) + ''' + + # Task-specific leaderboards + for task in ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]: + + with gr.TabItem(f"{TASK_ICONS[task]}{task}", elem_id="llm-benchmark-tab-table"): + + task_description = TASK_DESCRIPTIONS.get(task, "Description not available.") + + + + + gr.Markdown(task_description, elem_classes="markdown-text") + + + gr.Markdown(MEASURE_DESCRIPTION, elem_classes="markdown-text") + + + + leaderboard = init_leaderboard( + prepare_leaderboard_df(LEADERBOARD_DF, task), + default_selection=['FS', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id'], + hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in + ['FS', 'Model', 'Combined Performance', 'Prompt Average', 'Best Prompt', 'Best Prompt Id']] + ) + + # Citation section + with gr.Accordion("📙 Citation", open=False): + gr.Textbox(value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, lines=20, elem_id="citation-button", show_copy_button=True) + +# Background job to restart space +scheduler = BackgroundScheduler() +scheduler.add_job(restart_space, "interval", seconds=1800) +scheduler.start() + +demo.queue(default_concurrency_limit=40).launch() \ No newline at end of file diff --git a/get_model_info.py b/get_model_info.py new file mode 100644 index 0000000000000000000000000000000000000000..1a1f893faac9c892eff482d8027cd1fe724a1c6c --- /dev/null +++ b/get_model_info.py @@ -0,0 +1,128 @@ +""" +MODEL METADATA EXTRACTOR + +This script processes model evaluation output files (input_folder) from the lm-eval-harness library, +extracts model identifiers, retrieves detailed metadata from HuggingFace +and saves the information as structured JSON files (output_folder). + +Input: Directory containing .out files from lm-eval-harness +Output: Directory with JSON files containing model metadata +""" + +# Example input file format (lm-eval-harness output): +''' +hf (pretrained=swap-uniba/LLaMAntino-3-ANITA-8B-Inst-DPO-ITA,trust_remote_code=True), gen_kwargs: (None), limit: None, num_fewshot: 5, batch_size: 1 +| Tasks |Version|Filter|n-shot| Metric | |Value | |Stderr| +|------------------------|------:|------|-----:|--------|---|-----:|---|------| +|evalita-mp | 1|none | |acc |↑ |0.5605|± |0.0052| +... +Job completed +''' + +# Example output JSON format: +''' +{ + "model": "swap-uniba/LLaMAntino-3-ANITA-8B-Inst-DPO-ITA", + "base_model": "LlamaForCausalLM", + "revision": "2b6e46e4c9d341dc8bf8350a167492c880116b66", + "submitted_time": "2024-04-29 09:34:12+00:00", + "num_params_billion": 8.030261248, + "language": "en_it" +} +''' + +import os +import re +import json +from huggingface_hub import HfApi + +# Configures the Hugging Face token (if needed) +# TOKEN = "YOUR_HUGGINGFACE_API_TOKEN" +api = HfApi() + +# Directory paths +# input_folder: Directory containing the output files of the lm-eval-harness library, including model accuracy metrics. +input_folder = "../evalita_llm_models_output/" +# output_folder: Directory where JSON files with model characteristics will be saved. +output_folder = "../evalita_llm_requests/" + +# Creates the output folder if it doesn't exist +os.makedirs(output_folder, exist_ok=True) + +# Regular expression to find the model name +model_pattern = re.compile(r"pretrained=([\w\-./]+)") + +# Scans files in the input folder +for filename in os.listdir(input_folder): + if filename.endswith('.out'): + file_path = os.path.join(input_folder, filename) + + # Reads the file content + with open(file_path, "r", encoding="utf-8") as f: + content = f.read() + + # Extracts the model name + match = model_pattern.search(content) + if match: + model_name = match.group(1) + print(f"Processing model: {model_name}") + + try: + # Retrieves model information from HuggingFace + model_info = api.model_info(model_name) + + # Calculates the number of parameters in billions, if available + num_params = None + if model_info.safetensors and "BF16" in model_info.safetensors.parameters: + num_params = model_info.safetensors.parameters["BF16"] / 1e9 # Convert to billions + + # Extracts and concatenates languages + language = "_".join(model_info.card_data.get("language", [])) if model_info.card_data else "" + + #print(model_info) + + # Builds the dictionary with required metadata + model_data = { + "model": model_name, + "base_model": model_info.config.get("architectures", [""])[0] if model_info.config else "", + "revision": model_info.sha, + # "precision": "bfloat16", # If available, replace with real value + # "weight_type": "Original", + # "status": "FINISHED", + "submitted_time": str(model_info.created_at), + # "model_type": "pretrained", + # "likes": model_info.likes, + # "params": model_info.safetensors_size_in_bytes / 1e9 if model_info.safetensors_size_in_bytes else None, + # "license": model_info.license, + # "private": model_info.private, + "num_params_billion": num_params, # Number of parameters in billions + "language": language, # Extracted language + } + + # Separates the model_name into two parts: directory name and file name + if "/" in model_name: + dir_name, file_name = model_name.split("/", 1) + else: + dir_name, file_name = model_name, model_name # If no "/", use the same name + + # Creates the folder for saving the produced json files + model_output_folder = os.path.join(output_folder, dir_name) + os.makedirs(model_output_folder, exist_ok=True) + + # Saves the JSON file in the appropriate folder + output_file = os.path.join(model_output_folder, f"{file_name}.json") + + # Check if the file already exists + if os.path.exists(output_file): + print(f"File {output_file} already exists. Skipping...") + continue + + with open(output_file, "w", encoding="utf-8") as f: + json.dump(model_data, f, indent=4) + + print(f"Saved metadata for {model_name} in {output_file}") + + except Exception as e: + print(f"Error retrieving info for {model_name}: {e}") + + print("Process finished!") \ No newline at end of file diff --git a/preprocess_models_output.py b/preprocess_models_output.py new file mode 100644 index 0000000000000000000000000000000000000000..553fad40a69c2d25622c091c7490f56e15814422 --- /dev/null +++ b/preprocess_models_output.py @@ -0,0 +1,252 @@ +""" +EVALITA LLM EVALUATION PROCESSOR + +Transforms raw model evaluation outputs into structured performance reports for leaderboard integration. + +DATA PIPELINE OVERVIEW: + +1. Inputs: + - Evaluation Results: Raw .out files from lm-eval-harness + - Model Metadata: Pre-collected .json files from HuggingFace + +2. Output: + - Comprehensive evaluation reports in JSON format + - Ready for ingestion into the evaluation leaderboard + +-------------------------------------------------------------------- +INPUT SPECIFICATION + +Evaluation Results (.out format): + hf (pretrained=model-org/model-name), num_fewshot: 5, batch_size: 1 + | Task | Metric | Value | Stderr | + |---------------|--------|--------|--------| + | main-task | acc | 0.5605 | 0.0052 | + | - sub-task | acc | 0.4640 | 0.0088 | + | - prompt-1 | acc | 0.3720 | 0.0216 | + +Model Metadata (.json format): + { + "model": "model-org/model-name", + "base_model": "ModelArchitecture", + "revision": "git_commit_hash", + "parameters": 8.03, + "language": "en_it" + } + +-------------------------------------------------------------------- +OUTPUT SPECIFICATION + +Evaluation Report (.json format): + { + "summary_metrics": { + "average_CPS": 41.74, + "num_tasks": 12 + }, + "model_config": { + "identifier": "model-org/model-name", + "architecture": "ModelArchitecture", + "parameters": 8.03, + "evaluation_settings": { + "fewshot": 5, + "batch_size": 1 + } + }, + "task_results": { + "task-name": { + "average_score": 52.60, + "best_prompt": { + "id": "prompt-6", + "score": 66.57 + }, + "prompt_analysis": [ + { + "prompt_id": "prompt-1", + "score": 37.20, + "stderr": 0.0216 + } + ] + } + } + } +""" + +import json +import os +import re +import statistics + +def safe_float(value): + """Safely converts a value to float, returning None if the conversion fails.""" + try: + return float(value) + except ValueError: + return None + + +def calculate_task_metrics(task_info): + """Calculates average accuracy, best prompt accuracy, and CPS for a given task.""" + accuracies = [prompt['value'] for prompt in task_info['prompts'] if prompt['value'] is not None] + + if not accuracies: + return None + + task_info['average_accuracy'] = sum(accuracies) / len(accuracies) + task_info['std_accuracy'] = statistics.stdev(accuracies) if len(accuracies) > 1 else 0.0 + best_prompt_data = max(task_info['prompts'], key=lambda x: x['value']) + task_info['best_prompt'] = best_prompt_data['value'] + task_info['prompt_id'] = best_prompt_data['prompt'] + + # Calculate CPS + avg_acc = task_info['average_accuracy'] + best_acc = task_info['best_prompt'] + task_info['CPS'] = (1 - (best_acc - avg_acc) / 100) * best_acc + + +def extract_data_from_file(file_path): + """Extracts task and prompt data from a specified file.""" + with open(file_path, 'r') as file: + lines = file.readlines() + + tasks_data = {} + current_task = None + + for line in lines: + line = line.strip() + + # Skips empty lines + if not line: + continue + + # Skips header lines + if line.startswith("| Tasks"): + continue + + # Extracts model configuration details + if line.startswith("hf (pretrained="): + start = line.find("pretrained=") + len("pretrained=") + end = line.find(",", start) + pretrained_model = line[start:end] + + num_fewshot_match = re.search(r"num_fewshot:\s*([\w\d]+)", line) + num_fewshot = num_fewshot_match.group(1) if num_fewshot_match else None + + batch_size_match = re.search(r"batch_size:\s*(\d+)", line) + batch_size = int(batch_size_match.group(1)) if batch_size_match else None + + continue + + columns = line.split('|') + if len(columns) != 11: + continue + + task_name = columns[1] + metric = columns[5].strip() + value = safe_float(columns[7]) + stderr = safe_float(columns[9]) + + # Skips normalized accuracy metrics + if metric == "acc_norm": + continue + + # Identifies task and prompt sections in the file + if task_name.startswith(" - "): + task_name = task_name[3:].strip() + current_task = task_name + tasks_data.setdefault(current_task, + {'prompts': [], 'average_accuracy': 0, 'best_prompt': None, 'prompt_id': None, + 'CPS': None}) + + elif task_name.startswith(" - ") and current_task: + prompt_name = task_name[4:].strip() + prompt_data = {'prompt': prompt_name, 'metric': metric, 'value': value * 100, + 'stderr': stderr} + tasks_data[current_task]['prompts'].append(prompt_data) + + # Special handling for evalita NER task to calculate weighted prompt averages + if "evalita NER" in tasks_data: + task_info = tasks_data["evalita NER"] + weight_map = {"ADG prompt-1": 521, "ADG prompt-2": 521, "FIC prompt-1": 1517, "FIC prompt-2": 1517, + "WN prompt-1": 2088, "WN prompt-2": 2088} + + weighted_values = {"prompt-1": 0, "prompt-2": 0} + total_weights = sum(weight_map.values()) + + for prompt in task_info['prompts']: + if prompt['prompt'] in weight_map: + if "prompt-1" in prompt['prompt']: + weighted_values["prompt-1"] += weight_map[prompt['prompt']] * prompt['value'] + elif "prompt-2" in prompt['prompt']: + weighted_values["prompt-2"] += weight_map[prompt['prompt']] * prompt['value'] + + task_info['prompts'] = [ + {"prompt": "prompt-1", "metric": "acc", "value": weighted_values["prompt-1"] / total_weights, + 'stderr': None}, + {"prompt": "prompt-2", "metric": "acc", "value": weighted_values["prompt-2"] / total_weights, + 'stderr': None}] + + # Calculates task metrics for each task + for task_info in tasks_data.values(): + calculate_task_metrics(task_info) + + # Calculates the average CPS across all tasks + tasks_with_cps = [task['CPS'] for task in tasks_data.values() if task['CPS'] is not None] + average_CPS = sum(tasks_with_cps) / len(tasks_with_cps) if tasks_with_cps else 0 + + config = { + "model_name": pretrained_model, + "num_fewshot": num_fewshot, + "batch_size": batch_size + } + + return {'average_CPS': average_CPS, 'config': config, 'tasks': tasks_data} + + +""" +MAIN PROCESSING PIPELINE + +This script executes the complete evaluation data processing workflow: + +1. Input Sources: + - Raw evaluation results (.out files) from: ../evalita_llm_models_output/ + - Model metadata JSON files from: ../evalita_llm_requests/ + +2. Processing Steps: + - Parses evaluation metrics from .out files + - Combines with model metadata + - Calculates aggregated performance statistics + +3. Output: + - Structured JSON results saved to: ../evalita_llm_results/ + - Organized by model organization/name + - Contains complete evaluation results with metadata +""" +directory_in_path = '../evalita_llm_models_output/' +directory_in_requests_path = '../evalita_llm_requests/' +directory_out_results_path = '../evalita_llm_results/' + +for filename in os.listdir(directory_in_path): + if filename.endswith('.out'): + file_path = os.path.join(directory_in_path, filename) + json_output = extract_data_from_file(file_path) + + model_org_name, model_name = json_output['config']['model_name'].split('/') + + + config_file_path = os.path.join(directory_in_requests_path, model_org_name, f"{model_name}.json") + + if os.path.exists(config_file_path): + with open(config_file_path, 'r', encoding='utf-8') as config_file: + additional_config = json.load(config_file) + json_output['config'].update(additional_config) + + + org_folder_path = os.path.join(directory_out_results_path, model_org_name) + os.makedirs(org_folder_path, exist_ok=True) + + file_suffix = f"{json_output['config']['num_fewshot']}" + output_file_path = os.path.join(org_folder_path, f"{model_name}_{file_suffix}.json") + + with open(output_file_path, 'w', newline="\n") as outfile: + json.dump(json_output, outfile, indent=4) + + print(f"File {filename} processed and saved to {output_file_path}") \ No newline at end of file diff --git a/preprocess_models_output_old.py b/preprocess_models_output_old.py new file mode 100644 index 0000000000000000000000000000000000000000..e9b7fe2b199f6da6bff0b380ac99afe0ce9d0314 --- /dev/null +++ b/preprocess_models_output_old.py @@ -0,0 +1,201 @@ +import json +import os +import re + +def safe_float(value): + """Convert a value to float safely. Returns None if conversion fails.""" + try: + return float(value) + except ValueError: + return None + + +def calculate_task_metrics(task_info): + """Calculate average accuracy, best prompt, and CPS for a task.""" + accuracies = [prompt['value'] for prompt in task_info['prompts'] if prompt['value'] is not None] + + if not accuracies: + return None + + task_info['average_accuracy'] = sum(accuracies) / len(accuracies) + best_prompt_data = max(task_info['prompts'], key=lambda x: x['value']) + task_info['best_prompt'] = best_prompt_data['value'] + task_info['prompt_id'] = best_prompt_data['prompt'] + + # Calculate CPS + avg_acc = task_info['average_accuracy'] + best_acc = task_info['best_prompt'] + task_info['CPS'] = (1 - (best_acc - avg_acc) / 100) * best_acc + + +def extract_data_from_file(file_path): + """Extract task and prompt data from the given file.""" + with open(file_path, 'r') as file: + lines = file.readlines() + + tasks_data = {} + current_task = None + + for line in lines: + line = line.strip() + + # Skip irrelevant lines + if not line: + continue + + + if line.startswith("| Tasks"): + continue + + if line.startswith("hf (pretrained="): + + # Estrai la parte dopo "pretrained=" + start = line.find("pretrained=") + len("pretrained=") + end = line.find(",", start) # Trova la virgola successiva + # Estrai la stringa desiderata + pretrained_model = line[start:end] + + # Estrarre num_fewshot + num_fewshot_match = re.search(r"num_fewshot:\s*([\w\d]+)", line) + num_fewshot = num_fewshot_match.group(1) if num_fewshot_match else None + + # Estrarre batch_size + batch_size_match = re.search(r"batch_size:\s*(\d+)", line) + batch_size = int(batch_size_match.group(1)) if batch_size_match else None + + continue + + columns = line.split('|') + if len(columns) != 11: + continue + + task_name = columns[1] + metric = columns[5].strip() + value = safe_float(columns[7]) + stderr = safe_float(columns[9]) + + if metric == "acc_norm": + continue + + # Identify task and prompts + if task_name.startswith(" - "): + task_name = task_name[3:].strip() + current_task = task_name + tasks_data.setdefault(current_task, + {'prompts': [], 'average_accuracy': 0, 'best_prompt': None, 'prompt_id': None, + 'CPS': None}) + + elif task_name.startswith(" - ") and current_task: + prompt_name = task_name[4:].strip() + prompt_data = {'prompt': prompt_name, 'metric': metric, 'value': value * 100, + 'stderr': stderr} + tasks_data[current_task]['prompts'].append(prompt_data) + + # Special handling for evalita NER + if "evalita NER" in tasks_data: + task_info = tasks_data["evalita NER"] + weight_map = {"ADG prompt-1": 521, "ADG prompt-2": 521, "FIC prompt-1": 1517, "FIC prompt-2": 1517, + "WN prompt-1": 2088, "WN prompt-2": 2088} + + weighted_values = {"prompt-1": 0, "prompt-2": 0} + total_weights = sum(weight_map.values()) + + for prompt in task_info['prompts']: + if prompt['prompt'] in weight_map: + if "prompt-1" in prompt['prompt']: + weighted_values["prompt-1"] += weight_map[prompt['prompt']] * prompt['value'] + elif "prompt-2" in prompt['prompt']: + weighted_values["prompt-2"] += weight_map[prompt['prompt']] * prompt['value'] + + task_info['prompts'] = [ + {"prompt": "prompt-1", "metric": "acc", "value": weighted_values["prompt-1"] / total_weights, + 'stderr': None}, + {"prompt": "prompt-2", "metric": "acc", "value": weighted_values["prompt-2"] / total_weights, + 'stderr': None}] + + # Calculate metrics for each task + for task_info in tasks_data.values(): + calculate_task_metrics(task_info) + + # Calculate average CPS + tasks_with_cps = [task['CPS'] for task in tasks_data.values() if task['CPS'] is not None] + average_CPS = sum(tasks_with_cps) / len(tasks_with_cps) if tasks_with_cps else 0 + + config = { + "model_name": pretrained_model, + "num_fewshot": num_fewshot, + "batch_size": batch_size + } + + return {'average_CPS': average_CPS, 'config': config, 'tasks': tasks_data} + + +# Example usage +#file_path = '../evalita_llm_results/models_output/slurm-7769.out' +#json_output = extract_data_from_file(file_path) +#print(json_output) + + +# Directory da cui leggere i file .out +directory_in_path = '../evalita_llm_models_output/' +directory_out_results_path = '../evalita_llm_results/' +directory_out_requests_path = '../evalita_llm_requests/' + +# Itera sui file nella directory +for filename in os.listdir(directory_in_path): + if filename.endswith('.out'): + # Costruisci il percorso completo del file + file_path = os.path.join(directory_in_path, filename) + + # Esegui la funzione extract_data_from_file + json_output = extract_data_from_file(file_path) + + # Estrai model_org_name e model_name da model_name + model_org_name, model_name = json_output['config']['model_name'].split('/') + + + + + + + # Percorso del file JSON di configurazione in ../evalita_llm_requests2/ + config_file_path = os.path.join(directory_out_requests_path, model_org_name, f"{model_name}.json") + + # Se il file esiste, caricalo e aggiorna il dizionario config + if os.path.exists(config_file_path): + with open(config_file_path, 'r', encoding='utf-8') as config_file: + additional_config = json.load(config_file) + + # Aggiorna la configurazione con i nuovi dati + json_output['config'].update(additional_config) + + + + + # Crea il percorso della cartella per model_org_name + org_folder_path = os.path.join(directory_out_results_path, model_org_name) + os.makedirs(org_folder_path, exist_ok=True) # Crea la cartella se non esiste + + # Crea il percorso completo del file JSON + file_suffix = f"{json_output['config']['num_fewshot']}" + output_file_path = os.path.join(org_folder_path, f"{model_name}_{file_suffix}.json") + + # Salva il JSON in un file con ritorni a capo compatibili con Linux + with open(output_file_path, 'w', newline="\n") as outfile: + json.dump(json_output, outfile, indent=4) + + # Stampa il risultato + print(f"File {filename} elaborato e salvato in {output_file_path}") + + + + + + + + + + + + + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..3b4737924b5a7d81c962a4e28b66ac6cdcc3b004 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,13 @@ +[tool.ruff] +# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. +select = ["E", "F"] +ignore = ["E501"] # line too long (black is taking care of this) +line-length = 119 +fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"] + +[tool.isort] +profile = "black" +line_length = 119 + +[tool.black] +line-length = 119 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..46d41ea882da58a810ff984860b8fda48abf8f04 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,17 @@ +APScheduler +black +datasets +gradio +gradio[oauth] +gradio_leaderboard==0.0.13 +gradio_client +huggingface-hub>=0.18.0 +matplotlib +numpy +pandas +python-dateutil +tqdm +transformers +tokenizers>=0.15.0 +sentencepiece +plotly diff --git a/run_instructions.txt b/run_instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..a750b69dfb0a3a63c8ef77fb6bfef3c5bc9b2f2c --- /dev/null +++ b/run_instructions.txt @@ -0,0 +1,42 @@ +Model Evaluation and Leaderboard + +1) Model Evaluation +Before integrating a model into the leaderboard, it must first be evaluated using the lm-eval-harness library in both zero-shot and 5-shot configurations. + +This can be done with the following command: + +lm_eval --model hf --model_args pretrained=google/gemma-3-12b-it \ + --tasks evalita-mp --device cuda:0 --batch_size 1 --trust_remote_code \ + --output_path model_output --num_fewshot 5 -- + +The output generated by the library will include the model's accuracy scores on the benchmark tasks. +This output is written to the standard output and should be saved in a txt file (e.g., slurm-8368.out), which needs to be placed in the + evalita_llm_models_output directory for further processing. + +2) Extracting Model Metadata +To display model details on the leaderboard (e.g., organization/group, model name, and parameter count), metadata must be retrieved from Hugging Face. + +This can be done by running: + +python get_model_info.py + +This script processes the evaluation files from Step 1 and saves each model's metadata in a JSON file within the evalita_llm_requests directory. + +3) Generating Leaderboard Submission File +The leaderboard requires a structured file containing each model’s metadata along with its benchmark accuracy scores. + +To generate this file, run: + +python preprocess_model_output. + +This script combines the accuracy results from Step 1 with the metadata from Step 2 and outputs a JSON file in the evalita_llm_results directory. + +4) Updating the Hugging Face Repository +The evalita_llm_results repository on HuggingFace must be updated with the newly generated files from Step 3. + +5) Running the Leaderboard Application +Finally, execute the leaderboard application by running: + +python app.py + + diff --git a/src/.ipynb_checkpoints/about-checkpoint.py b/src/.ipynb_checkpoints/about-checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..36db643246b90c32a8c8262d87af92c878ba7cfe --- /dev/null +++ b/src/.ipynb_checkpoints/about-checkpoint.py @@ -0,0 +1,198 @@ +from dataclasses import dataclass +from enum import Enum + +@dataclass +class Task: + benchmark: str + metric: str + metric_type: str + col_name: str + +# Select your tasks here +# --------------------------------------------------- +class Tasks(Enum): + # task_key in the json file, metric_key in the json file, name to display in the leaderboard + + task1 = Task("text-entailment_1", "acc", "CPS", "TE") + task2 = Task("text-entailment_2", "acc", "average_accuracy", "TE Prompt Average") + task3 = Task("text-entailment_3", "acc", "std_accuracy", "TE Prompt Std") + task4 = Task("text-entailment_4", "acc", "best_prompt", "TE Best Prompt") + task5 = Task("text-entailment_5", "acc", "prompt_id", "TE Best Prompt Id") + + task6 = Task("sentiment-analysis_1", "acc", "CPS", "SA") + task7 = Task("sentiment-analysis_2", "acc", "average_accuracy", "SA Prompt Average") + task8 = Task("sentiment-analysis_3", "acc", "std_accuracy", "SA STD Accuracy") + task9 = Task("sentiment-analysis_4", "acc", "best_prompt", "SA Best Prompt") + task10 = Task("sentiment-analysis_5", "acc", "prompt_id", "SA Best Prompt Id") + + task11 = Task("hate-speech-detection_1", "acc", "CPS", "HS") + task12 = Task("hate-speech-detection_2", "acc", "average_accuracy", "HS Prompt Average") + task13 = Task("hate-speech-detection_3", "acc", "std_accuracy", "HS Prompt Std") + task14 = Task("hate-speech-detection_4", "acc", "best_prompt", "HS Best Prompt") + task15 = Task("hate-speech-detection_5", "acc", "prompt_id", "HS Best Prompt Id") + + task16 = Task("admission-test_1", "acc", "CPS", "AT") + task17 = Task("admission-test_2", "acc", "average_accuracy", "AT Prompt Average") + task18 = Task("admission-test_3", "acc", "std_accuracy", "AT Prompt Std") + task19 = Task("admission-test_4", "acc", "best_prompt", "AT Best Prompt") + task20 = Task("admission-test_5", "acc", "prompt_id", "AT Best Prompt Id") + + task21 = Task("word-in-context_1", "acc", "CPS", "WIC") + task22 = Task("word-in-context_2", "acc", "average_accuracy", "WIC Prompt Average") + task23 = Task("word-in-context_3", "acc", "std_accuracy", "WIC Prompt Std") + task24 = Task("word-in-context_4", "acc", "best_prompt", "WIC Best Prompt") + task25 = Task("word-in-context_5", "acc", "prompt_id", "WIC Best Prompt Id") + + task26 = Task("faq_1", "acc", "CPS", "FAQ") + task27 = Task("faq_2", "acc", "average_accuracy", "FAQ Prompt Average") + task28 = Task("faq_3", "acc", "std_accuracy", "FAQ Prompt Std") + task29 = Task("faq_4", "acc", "best_prompt", "FAQ Best Prompt") + task30 = Task("faq_5", "acc", "prompt_id", "FAQ Best Prompt Id") + + task31 = Task("lexical-substitution_1", "acc", "CPS", "LS") + task32 = Task("lexical-substitution_2", "acc", "average_accuracy", "LS Prompt Average") + task33 = Task("lexical-substitution_3", "acc", "std_accuracy", "LS Prompt Std") + task34 = Task("lexical-substitution_4", "acc", "best_prompt", "LS Best Prompt") + task35 = Task("lexical-substitution_5", "acc", "prompt_id", "LS Best Prompt Id") + + task36 = Task("summarization-fanpage_1", "acc", "CPS", "SU") + task37 = Task("summarization-fanpage_2", "acc", "average_accuracy", "SU Prompt Average") + task38 = Task("summarization-fanpage_3", "acc", "std_accuracy", "SU Prompt Std") + task39 = Task("summarization-fanpage_4", "acc", "best_prompt", "SU Best Prompt") + task40 = Task("summarization-fanpage_5", "acc", "prompt_id", "SU Best Prompt Id") + + task41 = Task("evalita NER_1", "acc", "CPS", "NER") + task42 = Task("evalita NER_2", "acc", "average_accuracy", "NER Prompt Average") + task43 = Task("evalita NER_3", "acc", "std_accuracy", "NER Prompt Std") + task44 = Task("evalita NER_4", "acc", "best_prompt", "NER Best Prompt") + task45 = Task("evalita NER_5", "acc", "prompt_id", "NER Best Prompt Id") + + task46 = Task("relation-extraction_1", "acc", "CPS", "REL") + task47 = Task("relation-extraction_2", "acc", "average_accuracy", "REL Prompt Average") + task48 = Task("relation-extraction_5", "acc", "std_accuracy", "REL Prompt Std") + task49 = Task("relation-extraction_3", "acc", "best_prompt", "REL Best Prompt") + task50 = Task("relation-extraction_4", "acc", "prompt_id", "REL Best Prompt Id") + + ''' + task0 = Task("TextualEntailment", "acc", "Textual Entailment") + task1 = Task("TextualEntailment_best", "acc", "TextualEntailment Best") + task2 = Task("Sentiment Analysis", "acc", "Sentiment Analysis") + task3 = Task("Sentiment Analysis_best", "acc", "Sentiment Analysis_best") + task4 = Task("Hate Speech", "acc", "Hate Speech") + task5 = Task("Hate Speech_best", "acc", "Hate Speech_best") + task6 = Task("Admission Test", "acc", "Admission Test") + task7 = Task("Admission Test_best", "acc", "Admission Test_best") + task8 = Task("Word in Context", "acc", "Word in Context") + task9 = Task("Word in Context_best", "acc", "Word in Context_best") + task10 = Task("FAQ", "acc", "FAQ") + task11 = Task("FAQ_best", "acc", "FAQ_best") + task12 = Task("Lexical Substitution", "acc", "Lexical Substitution") + task13 = Task("Lexical Substitution_best", "acc", "Lexical Substitution_best") + task14 = Task("Summarization", "acc", "Summarization") + task15 = Task("Summarization_best", "acc", "Summarization_best") + task16 = Task("NER", "acc", "NER") + task17 = Task("NER_best", "acc", "NER_best") + task18 = Task("REL", "acc", "REL") + task19 = Task("REL_best", "acc", "REL_best") + ''' + +# Your leaderboard name +TITLE = """

🚀 EVALITA-LLM Leaderboard 🚀

""" + +# What does your leaderboard evaluate? +INTRODUCTION_TEXT = """ +Evalita-LLM is a benchmark designed to evaluate Large Language Models (LLMs) on Italian tasks. The distinguishing features of Evalita-LLM are the following: (i) **all tasks are native Italian**, avoiding translation issues and potential cultural biases; (ii) the benchmark includes **generative** tasks, enabling more natural interaction with LLMs; (iii) **all tasks are evaluated against multiple prompts**, this way mitigating the model sensitivity to specific prompts and allowing a fairer evaluation. + +**Multiple-choice tasks:** 📊TE (Textual Entailment), 😃SA (Sentiment Analysis), ⚠️HS (Hate Speech Detection), 🏥AT (Admission Test), 🔤WIC (Word in Context), ❓FAQ (Frequently Asked Questions)
+**Generative tasks:** 🔄LS (Lexical Substitution), 📝SU (Summarization), 🏷️NER (Named Entity Recognition), 🔗REL (Relation Extraction) +""" + +# Which evaluations are you running? how can people reproduce what you have? +LLM_BENCHMARKS_TEXT = f""" +### Groups + +- `evalita-mp`: All tasks (perplexity and non-perplexity based). +- `evalita-mp_gen`: Only generative tasks. +- `evalita-mp_mc`: Only multiple-choice tasks. + +#### Tasks + +The following Evalita-LLM tasks can also be evaluated in isolation: + - `evalita-mp_te`: Textual Entailment (TE) + - `evalita-mp_sa`: Sentiment Analysis (SA) + - `evalita-mp_wic`: Word in Context (WIC) + - `evalita-mp_hs`: Hate Speech Detection (HS) + - `evalita-mp_at`: Admission Tests (AT) + - `evalita-mp_faq`: Frequently Asked Questions & Question Answering (FAQ) + - `evalita-mp_sum_fp`: Summarization (SU) + - `evalita-mp_ls`: Lexical Substitution LS) + - `evalita-mp_ner_group`: Named Entity Recognition (NER) + - `evalita-mp_re`: Relation Extraction (REL) + + +### Usage + +```bash + +lm_eval --model hf --model_args pretrained=meta-llama/Llama-2-7b-hf --tasks evalita-mp --device cuda:0 --batch_size 1 +``` + + + + +""" + +EVALUATION_QUEUE_TEXT = """ +## Some good practices before submitting a model + +### 1) Make sure you can load your model and tokenizer using AutoClasses: +```python +from transformers import AutoConfig, AutoModel, AutoTokenizer +config = AutoConfig.from_pretrained("your model name", revision=revision) +model = AutoModel.from_pretrained("your model name", revision=revision) +tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision) +``` +If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded. + +Note: make sure your model is public! +Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted! + +### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index) +It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`! + +### 3) Make sure your model has an open license! +This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗 + +### 4) Fill up your model card +When we add extra information about models to the leaderboard, it will be automatically taken from the model card + +## In case of model failure +If your model is displayed in the `FAILED` category, its execution stopped. +Make sure you have followed the above steps first. +If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task). +""" + +CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results" +CITATION_BUTTON_TEXT = r""" +@misc{magnini2025evalitallmbenchmarkinglargelanguage, + title={Evalita-LLM: Benchmarking Large Language Models on Italian}, + author={Bernardo Magnini and Roberto Zanoli and Michele Resta and Martin Cimmino and Paolo Albano and Marco Madeddu and Viviana Patti}, + year={2025}, + eprint={2502.02289}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2502.02289}, +} +""" diff --git a/src/__pycache__/about.cpython-310.pyc b/src/__pycache__/about.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..07fb890a24f5b10d43ddba71590c8829a47fe002 Binary files /dev/null and b/src/__pycache__/about.cpython-310.pyc differ diff --git a/src/__pycache__/envs.cpython-310.pyc b/src/__pycache__/envs.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da32f55e89a9db55752f2f537198438c037f6f80 Binary files /dev/null and b/src/__pycache__/envs.cpython-310.pyc differ diff --git a/src/__pycache__/populate.cpython-310.pyc b/src/__pycache__/populate.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..19b95e09659629c07f2407287dbcf8c95cd8ee76 Binary files /dev/null and b/src/__pycache__/populate.cpython-310.pyc differ diff --git a/src/__pycache__/tasks.cpython-310.pyc b/src/__pycache__/tasks.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..85db69d4d9a54f7388d0b6d4391d2a4b423c5ce1 Binary files /dev/null and b/src/__pycache__/tasks.cpython-310.pyc differ diff --git a/src/about.py b/src/about.py new file mode 100644 index 0000000000000000000000000000000000000000..2caa79e45f208a2590c114cdc6cb1957431a10fe --- /dev/null +++ b/src/about.py @@ -0,0 +1,209 @@ +from dataclasses import dataclass +from enum import Enum + +@dataclass +class Task: + benchmark: str + metric: str + metric_type: str + col_name: str + +# Select your tasks here +# --------------------------------------------------- +class Tasks(Enum): + # task_key in the json file, metric_key in the json file, name to display in the leaderboard + + #task1 = Task("text-entailment_1", "acc", "CPS", "TE") + #task2 = Task("text-entailment_2", "acc", "average_accuracy", "TE Prompt Average") + #task3 = Task("text-entailment_3", "acc", "std_accuracy", "TE Prompt Std") + #task4 = Task("text-entailment_4", "acc", "best_prompt", "TE Best Prompt") + #task5 = Task("text-entailment_5", "acc", "prompt_id", "TE Best Prompt Id") + + #task6 = Task("sentiment-analysis_1", "acc", "CPS", "SA") + #task7 = Task("sentiment-analysis_2", "acc", "average_accuracy", "SA Prompt Average") + #task8 = Task("sentiment-analysis_3", "acc", "std_accuracy", "SA STD Accuracy") + #task9 = Task("sentiment-analysis_4", "acc", "best_prompt", "SA Best Prompt") + #task10 = Task("sentiment-analysis_5", "acc", "prompt_id", "SA Best Prompt Id") + + #task11 = Task("hate-speech-detection_1", "acc", "CPS", "HS") + #task12 = Task("hate-speech-detection_2", "acc", "average_accuracy", "HS Prompt Average") + #task13 = Task("hate-speech-detection_3", "acc", "std_accuracy", "HS Prompt Std") + #task14 = Task("hate-speech-detection_4", "acc", "best_prompt", "HS Best Prompt") + #task15 = Task("hate-speech-detection_5", "acc", "prompt_id", "HS Best Prompt Id") + + #task16 = Task("admission-test_1", "acc", "CPS", "AT") + #task17 = Task("admission-test_2", "acc", "average_accuracy", "AT Prompt Average") + #task18 = Task("admission-test_3", "acc", "std_accuracy", "AT Prompt Std") + #task19 = Task("admission-test_4", "acc", "best_prompt", "AT Best Prompt") + #task20 = Task("admission-test_5", "acc", "prompt_id", "AT Best Prompt Id") + + #task21 = Task("word-in-context_1", "acc", "CPS", "WIC") + #task22 = Task("word-in-context_2", "acc", "average_accuracy", "WIC Prompt Average") + #task23 = Task("word-in-context_3", "acc", "std_accuracy", "WIC Prompt Std") + #task24 = Task("word-in-context_4", "acc", "best_prompt", "WIC Best Prompt") + #task25 = Task("word-in-context_5", "acc", "prompt_id", "WIC Best Prompt Id") + + #task26 = Task("faq_1", "acc", "CPS", "FAQ") + #task27 = Task("faq_2", "acc", "average_accuracy", "FAQ Prompt Average") + #task28 = Task("faq_3", "acc", "std_accuracy", "FAQ Prompt Std") + #task29 = Task("faq_4", "acc", "best_prompt", "FAQ Best Prompt") + #task30 = Task("faq_5", "acc", "prompt_id", "FAQ Best Prompt Id") + + #task31 = Task("lexical-substitution_1", "acc", "CPS", "LS") + #task32 = Task("lexical-substitution_2", "acc", "average_accuracy", "LS Prompt Average") + #task33 = Task("lexical-substitution_3", "acc", "std_accuracy", "LS Prompt Std") + #task34 = Task("lexical-substitution_4", "acc", "best_prompt", "LS Best Prompt") + #task35 = Task("lexical-substitution_5", "acc", "prompt_id", "LS Best Prompt Id") + + #task36 = Task("summarization-fanpage_1", "acc", "CPS", "SU") + #task37 = Task("summarization-fanpage_2", "acc", "average_accuracy", "SU Prompt Average") + #task38 = Task("summarization-fanpage_3", "acc", "std_accuracy", "SU Prompt Std") + #task39 = Task("summarization-fanpage_4", "acc", "best_prompt", "SU Best Prompt") + #task40 = Task("summarization-fanpage_5", "acc", "prompt_id", "SU Best Prompt Id") + + #task41 = Task("evalita NER_1", "acc", "CPS", "NER") + #task42 = Task("evalita NER_2", "acc", "average_accuracy", "NER Prompt Average") + #task43 = Task("evalita NER_3", "acc", "std_accuracy", "NER Prompt Std") + #task44 = Task("evalita NER_4", "acc", "best_prompt", "NER Best Prompt") + #task45 = Task("evalita NER_5", "acc", "prompt_id", "NER Best Prompt Id") + + #task46 = Task("relation-extraction_1", "acc", "CPS", "REL") + #task47 = Task("relation-extraction_2", "acc", "average_accuracy", "REL Prompt Average") + #task48 = Task("relation-extraction_5", "acc", "std_accuracy", "REL Prompt Std") + #task49 = Task("relation-extraction_3", "acc", "best_prompt", "REL Best Prompt") + #task50 = Task("relation-extraction_4", "acc", "prompt_id", "REL Best Prompt Id") + task1 = Task("RE_1", "acc", "CPS", "REL") + task2 = Task("RE_2", "acc", "average_accuracy", "REL Prompt Average") + #task3 = Task("RE_5", "acc", "stderr", "REL Prompt Std") + task4 = Task("RE_3", "acc", "best_prompt", "REL Best Prompt") + task5 = Task("RE_4", "acc", "prompt_id", "REL Best Prompt Id") + + task6 = Task("NER_1", "acc", "CPS", "NER") + task7 = Task("NER_2", "acc", "average_accuracy", "NER Prompt Average") + #task8 = Task("NER_3", "acc", "stderr", "NER Prompt Average") + task9 = Task("NER_4", "acc", "best_prompt", "NER Prompt Std") + task10 = Task("NER_5", "acc", "prompt_id", "NER Best Prompt Id") + + ''' + task0 = Task("TextualEntailment", "acc", "Textual Entailment") + task1 = Task("TextualEntailment_best", "acc", "TextualEntailment Best") + task2 = Task("Sentiment Analysis", "acc", "Sentiment Analysis") + task3 = Task("Sentiment Analysis_best", "acc", "Sentiment Analysis_best") + task4 = Task("Hate Speech", "acc", "Hate Speech") + task5 = Task("Hate Speech_best", "acc", "Hate Speech_best") + task6 = Task("Admission Test", "acc", "Admission Test") + task7 = Task("Admission Test_best", "acc", "Admission Test_best") + task8 = Task("Word in Context", "acc", "Word in Context") + task9 = Task("Word in Context_best", "acc", "Word in Context_best") + task10 = Task("FAQ", "acc", "FAQ") + task11 = Task("FAQ_best", "acc", "FAQ_best") + task12 = Task("Lexical Substitution", "acc", "Lexical Substitution") + task13 = Task("Lexical Substitution_best", "acc", "Lexical Substitution_best") + task14 = Task("Summarization", "acc", "Summarization") + task15 = Task("Summarization_best", "acc", "Summarization_best") + task16 = Task("NER", "acc", "NER") + task17 = Task("NER_best", "acc", "NER_best") + task18 = Task("REL", "acc", "REL") + task19 = Task("REL_best", "acc", "REL_best") + ''' + +# Your leaderboard name +TITLE = """

🚀 EVALITA-LLM Leaderboard 🚀

""" + +# What does your leaderboard evaluate? +INTRODUCTION_TEXT = """ +Evalita-LLM is a benchmark designed to evaluate Large Language Models (LLMs) on Italian tasks. The distinguishing features of Evalita-LLM are the following: (i) **all tasks are native Italian**, avoiding translation issues and potential cultural biases; (ii) the benchmark includes **generative** tasks, enabling more natural interaction with LLMs; (iii) **all tasks are evaluated against multiple prompts**, this way mitigating the model sensitivity to specific prompts and allowing a fairer evaluation. + +**Multiple-choice tasks:** 📊TE (Textual Entailment), 😃SA (Sentiment Analysis), ⚠️HS (Hate Speech Detection), 🏥AT (Admission Test), 🔤WIC (Word in Context), ❓FAQ (Frequently Asked Questions)
+**Generative tasks:** 🔄LS (Lexical Substitution), 📝SU (Summarization), 🏷️NER (Named Entity Recognition), 🔗REL (Relation Extraction) +""" + +# Which evaluations are you running? how can people reproduce what you have? +LLM_BENCHMARKS_TEXT = f""" +### Groups + +- `evalita-mp`: All tasks (perplexity and non-perplexity based). +- `evalita-mp_gen`: Only generative tasks. +- `evalita-mp_mc`: Only multiple-choice tasks. + +#### Tasks + +The following Evalita-LLM tasks can also be evaluated in isolation: + - `evalita-mp_te`: Textual Entailment (TE) + - `evalita-mp_sa`: Sentiment Analysis (SA) + - `evalita-mp_wic`: Word in Context (WIC) + - `evalita-mp_hs`: Hate Speech Detection (HS) + - `evalita-mp_at`: Admission Tests (AT) + - `evalita-mp_faq`: Frequently Asked Questions & Question Answering (FAQ) + - `evalita-mp_sum_fp`: Summarization (SU) + - `evalita-mp_ls`: Lexical Substitution LS) + - `evalita-mp_ner_group`: Named Entity Recognition (NER) + - `evalita-mp_re`: Relation Extraction (REL) + + +### Usage + +```bash + +lm_eval --model hf --model_args pretrained=meta-llama/Llama-2-7b-hf --tasks evalita-mp --device cuda:0 --batch_size 1 +``` + + + + +""" + +EVALUATION_QUEUE_TEXT = """ +## Some good practices before submitting a model + +### 1) Make sure you can load your model and tokenizer using AutoClasses: +```python +from transformers import AutoConfig, AutoModel, AutoTokenizer +config = AutoConfig.from_pretrained("your model name", revision=revision) +model = AutoModel.from_pretrained("your model name", revision=revision) +tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision) +``` +If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded. + +Note: make sure your model is public! +Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted! + +### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index) +It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`! + +### 3) Make sure your model has an open license! +This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗 + +### 4) Fill up your model card +When we add extra information about models to the leaderboard, it will be automatically taken from the model card + +## In case of model failure +If your model is displayed in the `FAILED` category, its execution stopped. +Make sure you have followed the above steps first. +If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task). +""" + +CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results" +CITATION_BUTTON_TEXT = r""" +@misc{magnini2025evalitallmbenchmarkinglargelanguage, + title={Evalita-LLM: Benchmarking Large Language Models on Italian}, + author={Bernardo Magnini and Roberto Zanoli and Michele Resta and Martin Cimmino and Paolo Albano and Marco Madeddu and Viviana Patti}, + year={2025}, + eprint={2502.02289}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2502.02289}, +} +""" diff --git a/src/display/.ipynb_checkpoints/utils-checkpoint.py b/src/display/.ipynb_checkpoints/utils-checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..f11c07243e402f48a8ba018a8942b2190e5747e7 --- /dev/null +++ b/src/display/.ipynb_checkpoints/utils-checkpoint.py @@ -0,0 +1,188 @@ +from dataclasses import dataclass, make_dataclass +from enum import Enum + +import pandas as pd + +from src.about import Tasks + +def fields(raw_class): + return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"] + + +# These classes are for user facing column names, +# to avoid having to change them all around the code +# when a modif is needed +@dataclass +class ColumnContent: + name: str + type: str + displayed_by_default: bool + hidden: bool = False + never_hidden: bool = False + +## Leaderboard columns +auto_eval_column_dict = [] +# Init +#auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)]) + +auto_eval_column_dict.append(["rank", ColumnContent, ColumnContent("Rank", "number", True, never_hidden=True)]) +auto_eval_column_dict.append(["size_symbol", ColumnContent, ColumnContent("Size", "number", True, never_hidden=True)]) + +auto_eval_column_dict.append(["fewshot_symbol", ColumnContent, ColumnContent("FS", "str", True, never_hidden=True)]) +auto_eval_column_dict.append(["is_5fewshot", ColumnContent, ColumnContent("IS_FS", "bool", True)]) + +auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)]) +#auto_eval_column_dict.append(["fewshot", ColumnContent, ColumnContent("Few-Shot", "str", True)]) + +#Scores +auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Avg. Comb. Perf. ⬆️", "number", True)]) +for task in Tasks: + auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)]) + +# Model information +#auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)]) +auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)]) +auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)]) +#auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)]) +auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)]) +auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)]) +auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)]) +auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)]) +auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)]) +#auto_eval_column_dict.append(["submitted_time", ColumnContent, ColumnContent("Submitted time", "date", False)]) + +# We use make dataclass to dynamically fill the scores from Tasks +AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True) + +## For the queue columns in the submission tab +@dataclass(frozen=True) +class EvalQueueColumn: # Queue column + model = ColumnContent("model", "markdown", True) + revision = ColumnContent("revision", "str", True) + private = ColumnContent("private", "bool", True) + #precision = ColumnContent("precision", "str", True) + weight_type = ColumnContent("weight_type", "str", "Original") + status = ColumnContent("status", "str", True) + +## All the model information that we might need +@dataclass +class ModelDetails: + name: str + display_name: str = "" + symbol: str = "" # emoji + + +class ModelType(Enum): + PT = ModelDetails(name="pretrained", symbol="🟢") + FT = ModelDetails(name="fine-tuned", symbol="🔶") + IFT = ModelDetails(name="instruction-tuned", symbol="⭕") + RL = ModelDetails(name="RL-tuned", symbol="🟦") + Unknown = ModelDetails(name="", symbol="?") + + def to_str(self, separator=" "): + return f"{self.value.symbol}{separator}{self.value.name}" + + @staticmethod + def from_str(type): + if "fine-tuned" in type or "🔶" in type: + return ModelType.FT + if "pretrained" in type or "🟢" in type: + return ModelType.PT + if "RL-tuned" in type or "🟦" in type: + return ModelType.RL + if "instruction-tuned" in type or "⭕" in type: + return ModelType.IFT + return ModelType.Unknown + +@dataclass +class FewShotDetails: + name: str + symbol: str = "" # emoji + +class FewShotType(Enum): + ZS = FewShotDetails(name="zero-shot", symbol="🅾️") + FS = FewShotDetails(name="5-few-shot", symbol="5️⃣") + Unknown = FewShotDetails(name="unknown", symbol="❓") + + def to_str(self, separator=" "): + return f"{self.value.symbol}{separator}{self.value.name}" + + @staticmethod + def from_num_fewshot(is_5fewshot): + """Determines FewShotType based on num_fewshot.""" + if is_5fewshot is False: + return FewShotType.ZS + elif is_5fewshot is True: + return FewShotType.FS + return FewShotType.Unknown + +@dataclass +class SizeDetails: + name: str + symbol: str = "" # emoji + +class SizeType(Enum): + SMALL = SizeDetails(name="small", symbol="🔵") + MEDIUM = SizeDetails(name="medium", symbol="🔵🔵") + LARGE = SizeDetails(name="large", symbol="🔵🔵🔵") + Unknown = SizeDetails(name="unknown", symbol="❓") + + def to_str(self, separator=" "): + return f"{self.value.symbol}{separator}{self.value.name}" + + @staticmethod + def num2type(size): + """Determines FewShotType based on num_fewshot.""" + if size <= 10: + return SizeType.SMALL + elif size > 10 and size <= 50: + return SizeType.MEDIUM + else: + return SizeType.LARGE + +class WeightType(Enum): + Adapter = ModelDetails("Adapter") + Original = ModelDetails("Original") + Delta = ModelDetails("Delta") + +class Precision(Enum): + float16 = ModelDetails("float16") + bfloat16 = ModelDetails("bfloat16") + Unknown = ModelDetails("?") + + def from_str(precision): + if precision in ["torch.float16", "float16"]: + return Precision.float16 + if precision in ["torch.bfloat16", "bfloat16"]: + return Precision.bfloat16 + return Precision.Unknown + +# Column selection +COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden] + +EVAL_COLS = [c.name for c in fields(EvalQueueColumn)] +EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)] + +BENCHMARK_COLS = [t.value.col_name for t in Tasks] + +''' +# Nuovi valori per CPS, AVERAGE, BEST, e ID nella tabella +@dataclass +class NewColumnContent: + name: str + type: str + displayed_by_default: bool + hidden: bool = False + never_hidden: bool = False +''' + +''' +new_column_dict = [] +# Aggiungi CPS, VERAGE, BEST, ID +new_column_dict.append(["CPS", NewColumnContent, NewColumnContent("CPS", "number", True)]) +new_column_dict.append(["AVERAGE", NewColumnContent, NewColumnContent("Average ⬆️", "number", True)]) +new_column_dict.append(["BEST", NewColumnContent, NewColumnContent("Best Performance", "number", True)]) +new_column_dict.append(["ID", NewColumnContent, NewColumnContent("ID", "str", True)]) +NewColumn = make_dataclass("NewColumn", new_column_dict, frozen=True) +NEW_COLS = [c.name for c in fields(NewColumn) if not c.hidden] +''' diff --git a/src/display/__pycache__/css_html_js.cpython-310.pyc b/src/display/__pycache__/css_html_js.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..39e54730b6db47d66cfccfe6a72a27fe1d3d23aa Binary files /dev/null and b/src/display/__pycache__/css_html_js.cpython-310.pyc differ diff --git a/src/display/__pycache__/formatting.cpython-310.pyc b/src/display/__pycache__/formatting.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e183b7463adcf188048f008b51c49e84531113a4 Binary files /dev/null and b/src/display/__pycache__/formatting.cpython-310.pyc differ diff --git a/src/display/__pycache__/utils.cpython-310.pyc b/src/display/__pycache__/utils.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5c7f9272e5f5572d8022b193d6b66762b4d49c24 Binary files /dev/null and b/src/display/__pycache__/utils.cpython-310.pyc differ diff --git a/src/display/css_html_js.py b/src/display/css_html_js.py new file mode 100644 index 0000000000000000000000000000000000000000..721bc2e346c9e2578013986ee477c46a5fe11b17 --- /dev/null +++ b/src/display/css_html_js.py @@ -0,0 +1,122 @@ +custom_css = """ + +.markdown-text { + font-size: 16px !important; +} + +#models-to-add-text { + font-size: 18px !important; +} + +#citation-button span { + font-size: 16px !important; +} + +#citation-button textarea { + font-size: 16px !important; +} + +#citation-button > label > button { + margin: 6px; + transform: scale(1.3); +} + +#leaderboard-table { + margin-top: 15px +} + +#leaderboard-table-lite { + margin-top: 15px +} + +#search-bar-table-box > div:first-child { + background: none; + border: none; +} + +#search-bar { + padding: 0px; +} + +/* Limit the width of the first AutoEvalColumn so that names don't expand too much */ +#leaderboard-table td:nth-child(2), +#leaderboard-table th:nth-child(2) { + max-width: 400px; + overflow: auto; + white-space: nowrap; +} + +.tab-buttons button { + font-size: 20px; +} + +#scale-logo { + border-style: none !important; + box-shadow: none; + display: block; + margin-left: auto; + margin-right: auto; + max-width: 600px; +} + +#scale-logo .download { + display: none; +} +#filter_type{ + border: 0; + padding-left: 0; + padding-top: 0; +} +#filter_type label { + display: flex; +} +#filter_type label > span{ + margin-top: var(--spacing-lg); + margin-right: 0.5em; +} +#filter_type label > .wrap{ + width: 103px; +} +#filter_type label > .wrap .wrap-inner{ + padding: 2px; +} +#filter_type label > .wrap .wrap-inner input{ + width: 1px +} +#filter-columns-type{ + border:0; + padding:0.5; +} +#filter-columns-size{ + border:0; + padding:0.5; +} +#box-filter > .form{ + border: 0 +} + +/* === Added scaling for plots === */ +#line-chart, +#boxplot-task { + max-width: 100%; + width: 100%; + height: auto; + margin: 0 auto; + display: block; +} + +/* nasconde la barra degli strumenti Plotly */ +.modebar { + display: none !important; +} + +""" + +get_window_url_params = """ + function(url_params) { + const params = new URLSearchParams(window.location.search); + url_params = Object.fromEntries(params); + return url_params; + } + """ + diff --git a/src/display/formatting.py b/src/display/formatting.py new file mode 100644 index 0000000000000000000000000000000000000000..ba340b8c51c98be420f01682eedda01099dc92a3 --- /dev/null +++ b/src/display/formatting.py @@ -0,0 +1,30 @@ +def model_hyperlink(link, model_name): + return f'{model_name}' + + +def make_clickable_model(model_name): + link = f"https://huggingface.co/{model_name}" + #Remove author prefix from model names for EVALITA-LLM + model_name = model_name.split("/")[-1] + #print(model_name) + return model_hyperlink(link, model_name) + + +def styled_error(error): + return f"

{error}

" + + +def styled_warning(warn): + return f"

{warn}

" + + +def styled_message(message): + return f"

{message}

" + + +def has_no_nan_values(df, columns): + return df[columns].notna().all(axis=1) + + +def has_nan_values(df, columns): + return df[columns].isna().any(axis=1) diff --git a/src/display/utils.py b/src/display/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ce06546426f490f8841725ec7b93d7b8c3fab089 --- /dev/null +++ b/src/display/utils.py @@ -0,0 +1,189 @@ +from dataclasses import dataclass, make_dataclass +from enum import Enum + +import pandas as pd + +from src.about import Tasks + +def fields(raw_class): + return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"] + + +# These classes are for user facing column names, +# to avoid having to change them all around the code +# when a modif is needed +@dataclass +class ColumnContent: + name: str + type: str + displayed_by_default: bool + hidden: bool = False + never_hidden: bool = False + +## Leaderboard columns +auto_eval_column_dict = [] +# Init +#auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)]) + +auto_eval_column_dict.append(["rank", ColumnContent, ColumnContent("Rank", "number", True, never_hidden=True)]) +auto_eval_column_dict.append(["size_symbol", ColumnContent, ColumnContent("Size", "number", True, never_hidden=True)]) + +auto_eval_column_dict.append(["fewshot_symbol", ColumnContent, ColumnContent("FS", "str", True, never_hidden=True)]) +auto_eval_column_dict.append(["is_5fewshot", ColumnContent, ColumnContent("IS_FS", "bool", True)]) +auto_eval_column_dict.append(["LANG", ColumnContent, ColumnContent("LANG", "str", True, never_hidden=True)]) + +auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)]) +#auto_eval_column_dict.append(["fewshot", ColumnContent, ColumnContent("Few-Shot", "str", True)]) + +#Scores +auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Avg. Comb. Perf. ⬆️", "number", True)]) +for task in Tasks: + auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)]) + +# Model information +#auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)]) +auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)]) +auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)]) +#auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)]) +auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)]) +auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)]) +auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)]) +auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)]) +auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)]) +#auto_eval_column_dict.append(["submitted_time", ColumnContent, ColumnContent("Submitted time", "date", False)]) + +# We use make dataclass to dynamically fill the scores from Tasks +AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True) + +## For the queue columns in the submission tab +@dataclass(frozen=True) +class EvalQueueColumn: # Queue column + model = ColumnContent("model", "markdown", True) + revision = ColumnContent("revision", "str", True) + private = ColumnContent("private", "bool", True) + #precision = ColumnContent("precision", "str", True) + weight_type = ColumnContent("weight_type", "str", "Original") + status = ColumnContent("status", "str", True) + +## All the model information that we might need +@dataclass +class ModelDetails: + name: str + display_name: str = "" + symbol: str = "" # emoji + + +class ModelType(Enum): + PT = ModelDetails(name="pretrained", symbol="🟢") + FT = ModelDetails(name="fine-tuned", symbol="🔶") + IFT = ModelDetails(name="instruction-tuned", symbol="⭕") + RL = ModelDetails(name="RL-tuned", symbol="🟦") + Unknown = ModelDetails(name="", symbol="?") + + def to_str(self, separator=" "): + return f"{self.value.symbol}{separator}{self.value.name}" + + @staticmethod + def from_str(type): + if "fine-tuned" in type or "🔶" in type: + return ModelType.FT + if "pretrained" in type or "🟢" in type: + return ModelType.PT + if "RL-tuned" in type or "🟦" in type: + return ModelType.RL + if "instruction-tuned" in type or "⭕" in type: + return ModelType.IFT + return ModelType.Unknown + +@dataclass +class FewShotDetails: + name: str + symbol: str = "" # emoji + +class FewShotType(Enum): + ZS = FewShotDetails(name="zero-shot", symbol="🅾️") + FS = FewShotDetails(name="10-few-shot", symbol="🔟") + Unknown = FewShotDetails(name="unknown", symbol="❓") + + def to_str(self, separator=" "): + return f"{self.value.symbol}{separator}{self.value.name}" + + @staticmethod + def from_num_fewshot(is_5fewshot): + """Determines FewShotType based on num_fewshot.""" + if is_5fewshot is False: + return FewShotType.ZS + elif is_5fewshot is True: + return FewShotType.FS + return FewShotType.Unknown + +@dataclass +class SizeDetails: + name: str + symbol: str = "" # emoji + +class SizeType(Enum): + SMALL = SizeDetails(name="small", symbol="🔵") + MEDIUM = SizeDetails(name="medium", symbol="🔵🔵") + LARGE = SizeDetails(name="large", symbol="🔵🔵🔵") + Unknown = SizeDetails(name="unknown", symbol="❓") + + def to_str(self, separator=" "): + return f"{self.value.symbol}{separator}{self.value.name}" + + @staticmethod + def num2type(size): + """Determines FewShotType based on num_fewshot.""" + if size <= 10: + return SizeType.SMALL + elif size > 10 and size <= 50: + return SizeType.MEDIUM + else: + return SizeType.LARGE + +class WeightType(Enum): + Adapter = ModelDetails("Adapter") + Original = ModelDetails("Original") + Delta = ModelDetails("Delta") + +class Precision(Enum): + float16 = ModelDetails("float16") + bfloat16 = ModelDetails("bfloat16") + Unknown = ModelDetails("?") + + def from_str(precision): + if precision in ["torch.float16", "float16"]: + return Precision.float16 + if precision in ["torch.bfloat16", "bfloat16"]: + return Precision.bfloat16 + return Precision.Unknown + +# Column selection +COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden] + +EVAL_COLS = [c.name for c in fields(EvalQueueColumn)] +EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)] + +BENCHMARK_COLS = [t.value.col_name for t in Tasks] + +''' +# Nuovi valori per CPS, AVERAGE, BEST, e ID nella tabella +@dataclass +class NewColumnContent: + name: str + type: str + displayed_by_default: bool + hidden: bool = False + never_hidden: bool = False +''' + +''' +new_column_dict = [] +# Aggiungi CPS, VERAGE, BEST, ID +new_column_dict.append(["CPS", NewColumnContent, NewColumnContent("CPS", "number", True)]) +new_column_dict.append(["AVERAGE", NewColumnContent, NewColumnContent("Average ⬆️", "number", True)]) +new_column_dict.append(["BEST", NewColumnContent, NewColumnContent("Best Performance", "number", True)]) +new_column_dict.append(["ID", NewColumnContent, NewColumnContent("ID", "str", True)]) +NewColumn = make_dataclass("NewColumn", new_column_dict, frozen=True) +NEW_COLS = [c.name for c in fields(NewColumn) if not c.hidden] +''' diff --git a/src/leaderboard/.ipynb_checkpoints/read_evals-checkpoint.py b/src/leaderboard/.ipynb_checkpoints/read_evals-checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..dee8db5a6e176e56893d5d1e39b2d009c1835832 --- /dev/null +++ b/src/leaderboard/.ipynb_checkpoints/read_evals-checkpoint.py @@ -0,0 +1,231 @@ +import glob +import json +import math +import os +from dataclasses import dataclass, field + +import dateutil +import numpy as np +from typing import Dict, Union +from datetime import datetime + +#from get_model_info import num_params +from src.display.formatting import make_clickable_model +from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, FewShotType, SizeType +from src.submission.check_validity import is_model_on_hub + + +@dataclass +class EvalResult: + """Represents one full evaluation. Built from a combination of the result and request file for a given run. + """ + eval_name: str # org_model_precision (uid) + full_model: str # org/model (path on hub) + org: str + model: str + revision: str # commit hash, "" if main + #submitted_time: datetime + results: Dict[str, Union[float, int]] # float o int + average_CPS: float + is_5fewshot: bool + fewshot_symbol: FewShotType = FewShotType.Unknown + weight_type: WeightType = WeightType.Original # Original or Adapter + architecture: str = "Unknown" + license: str = "?" + likes: int = 0 + num_params: int = 0 + date: str = "" # submission date of request file + still_on_hub: bool = False + rank: int = 0#str = field(default=0) # nuovo campo con default = 0 + size_symbol: SizeType = SizeType.Unknown + + @classmethod + def init_from_json_file(self, json_filepath): + """Inits the result from the specific model result file""" + with open(json_filepath) as fp: + data = json.load(fp) + + config = data.get("config") + + #average_CPS = f"{data.get('average_CPS'):.2f}" + # Get average_CPS + average_CPS = float(data.get('average_CPS', 0.0)) # 0.0 come valore di default + # Get number of fewshot + fewshot = config.get("num_fewshot", False) + + rank = 0 + + try: + if fewshot == "5": + is_5fewshot = True + else: + is_5fewshot = False + except ValueError: + is_5fewshot = False + # Determine the few-shot type (ZS or FS) based on num_fewshot + fewshot_symbol = FewShotType.from_num_fewshot(is_5fewshot) # Use the new + + # Determine the number of parameters of the models + num_params = int(0) + num_params_billion = config.get("num_params_billion") + if num_params_billion is not None: + num_params = math.ceil(num_params_billion) + + size_symbol = SizeType.num2type(num_params) + + # Get model and org + org_and_model = config.get("model_name", config.get("model_args", None)) + org_and_model = org_and_model.split("/", 1) + + if len(org_and_model) == 1: + org = None + model = org_and_model[0] + #result_key = f"{model}_{precision.value.name}" + result_key = f"{model}_{is_5fewshot}" + else: + org = org_and_model[0] + model = org_and_model[1] + #result_key = f"{org}_{model}_{precision.value.name}" + result_key = f"{org}_{model}_{is_5fewshot}" + full_model = "/".join(org_and_model) + + still_on_hub, _, model_config = is_model_on_hub( + full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False + ) + architecture = "?" + if model_config is not None: + architectures = getattr(model_config, "architectures", None) + if architectures: + architecture = ";".join(architectures) + + # Extract the results of the models + results = {} + for task in Tasks: + task = task.value + + for k, v in data["tasks"].items(): + if task.benchmark[:-2] == k: + if "Best Prompt Id" in task.col_name: + results[task.benchmark] = int(v[task.metric_type][-1:]) + else: + #results[task.benchmark] = f"{v[task.metric_type]:.2f}" # Ensure two decimals for display + results[task.benchmark] = float(v[task.metric_type]) + #value = float(v[task.metric_type]) + #results[task.benchmark] = round(value, 2) # Arrotonda a 2 decimali + + return self( + eval_name=result_key, + full_model=full_model, + org=org, + model=model, + results=results, + average_CPS=average_CPS, + fewshot_symbol=fewshot_symbol, + is_5fewshot=is_5fewshot, + revision= config.get("model_sha", ""), + still_on_hub=still_on_hub, + architecture=architecture, + num_params=num_params, + rank = rank, + size_symbol=size_symbol + #submitted_time=config.get("submitted_time", ""), + ) + + ''' + def update_with_request_file(self, requests_path): + """Finds the relevant request file for the current model and updates info with it""" + request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name) + + try: + with open(request_file, "r") as f: + request = json.load(f) + self.model_type = ModelType.from_str(request.get("model_type", "")) + self.weight_type = WeightType[request.get("weight_type", "Original")] + self.license = request.get("license", "?") + self.likes = request.get("likes", 0) + self.num_params = request.get("params", 0) + self.date = request.get("submitted_time", "") + except Exception: + print(f"Could not find request file for {self.org}/{self.model} with precision + ''' + + def to_dict(self): + """Converts the Eval Result to a dict compatible with our dataframe display""" + average = self.average_CPS + + fewshot_symbol = ( + self.fewshot_symbol.value.symbol if isinstance(self.fewshot_symbol, FewShotType) else "❓" + ) + + size_symbol = ( + self.size_symbol.value.symbol if isinstance(self.size_symbol, SizeType) else "❓" + ) + + data_dict = { + "eval_name": self.eval_name, # not a column, just a save name, + #AutoEvalColumn.precision.name: self.precision.value.name, + #AutoEvalColumn.model_type.name: self.model_type.value.name, + #AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol, + #AutoEvalColumn.model_type.name: self.model_type.value.name if self.model_type else "Unknown", + #AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol if self.model_type else "Unknown", + AutoEvalColumn.fewshot_symbol.name: fewshot_symbol, + AutoEvalColumn.weight_type.name: self.weight_type.value.name, + AutoEvalColumn.architecture.name: self.architecture, + AutoEvalColumn.model.name: make_clickable_model(self.full_model), + AutoEvalColumn.revision.name: self.revision, + AutoEvalColumn.average.name: average, + AutoEvalColumn.is_5fewshot.name: self.is_5fewshot, + AutoEvalColumn.license.name: self.license, + AutoEvalColumn.likes.name: self.likes, + AutoEvalColumn.params.name: self.num_params, + AutoEvalColumn.still_on_hub.name: self.still_on_hub, + AutoEvalColumn.rank.name: self.rank, + AutoEvalColumn.size_symbol.name: size_symbol + } + + for task in Tasks: + data_dict[task.value.col_name] = self.results[task.value.benchmark] + + return data_dict + + +def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]: + """From the path of the results folder root, extract all needed info for results""" + model_result_filepaths = [] + + for root, _, files in os.walk(results_path): + # We should only have json files in model results + if len(files) == 0 or any([not f.endswith(".json") for f in files]): + continue + + # Sort the files by date + try: + files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7]) + except dateutil.parser._parser.ParserError: + files = [files[-1]] + + for file in files: + model_result_filepaths.append(os.path.join(root, file)) + + eval_results = {} + for model_result_filepath in model_result_filepaths: + # Creation of result + eval_result = EvalResult.init_from_json_file(model_result_filepath) + #eval_result.update_with_request_file(requests_path) + + # Store results of same eval together + eval_name = eval_result.eval_name + if eval_name in eval_results.keys(): + eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None}) + else: + eval_results[eval_name] = eval_result + + results = [] + for v in eval_results.values(): + try: + v.to_dict() # we test if the dict version is complete + results.append(v) + except KeyError: # not all eval values present + continue + + return results diff --git a/src/leaderboard/__pycache__/read_evals.cpython-310.pyc b/src/leaderboard/__pycache__/read_evals.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8b537403aed6e9414852a18d2989bb9fc53a7618 Binary files /dev/null and b/src/leaderboard/__pycache__/read_evals.cpython-310.pyc differ diff --git a/src/leaderboard/read_evals.py b/src/leaderboard/read_evals.py new file mode 100644 index 0000000000000000000000000000000000000000..b593d5dad3cf06a0ce767b74eeecd47689ebf936 --- /dev/null +++ b/src/leaderboard/read_evals.py @@ -0,0 +1,236 @@ +import glob +import json +import math +import os +from dataclasses import dataclass, field + +import dateutil +import numpy as np +from typing import Dict, Union +from datetime import datetime + +#from get_model_info import num_params +from src.display.formatting import make_clickable_model +from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, FewShotType, SizeType +from src.submission.check_validity import is_model_on_hub + + +@dataclass +class EvalResult: + """Represents one full evaluation. Built from a combination of the result and request file for a given run. + """ + eval_name: str # org_model_precision (uid) + full_model: str # org/model (path on hub) + org: str + model: str + revision: str # commit hash, "" if main + #submitted_time: datetime + results: Dict[str, Union[float, int]] # float o int + average_CPS: float + is_5fewshot: bool + fewshot_symbol: FewShotType = FewShotType.Unknown + weight_type: WeightType = WeightType.Original # Original or Adapter + architecture: str = "Unknown" + license: str = "?" + likes: int = 0 + Lang:str="EN" + num_params: int = 0 + date: str = "" # submission date of request file + still_on_hub: bool = False + rank: int = 0#str = field(default=0) # nuovo campo con default = 0 + size_symbol: SizeType = SizeType.Unknown + + @classmethod + def init_from_json_file(self, json_filepath): + """Inits the result from the specific model result file""" + with open(json_filepath) as fp: + data = json.load(fp) + + config = data.get("config") + + #average_CPS = f"{data.get('average_CPS'):.2f}" + # Get average_CPS + average_CPS = float(data.get('average_CPS', 0.0)) # 0.0 come valore di default + # Get number of fewshot + fewshot = config.get("num_fewshot", False) + + rank = 0 + Lang=config.get("LANG", "EN") + try: + if fewshot == "10": + is_5fewshot = True + else: + is_5fewshot = False + except ValueError: + is_5fewshot = False + # Determine the few-shot type (ZS or FS) based on num_fewshot + fewshot_symbol = FewShotType.from_num_fewshot(is_5fewshot) # Use the new + + # Determine the number of parameters of the models + num_params = int(0) + num_params_billion = config.get("num_params_billion") + if num_params_billion is not None: + num_params = math.ceil(num_params_billion) + + size_symbol = SizeType.num2type(num_params) + + # Get model and org + org_and_model = config.get("model_name", config.get("model_args", None)) + org_and_model = org_and_model.split("/", 1) + + if len(org_and_model) == 1: + org = None + model = org_and_model[0] + #result_key = f"{model}_{precision.value.name}" + result_key = f"{model}_{is_5fewshot}" + else: + org = org_and_model[0] + model = org_and_model[1] + #result_key = f"{org}_{model}_{precision.value.name}" + result_key = f"{org}_{model}_{is_5fewshot}" + full_model = "/".join(org_and_model) + + still_on_hub, _, model_config = is_model_on_hub( + full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False + ) + architecture = "?" + if model_config is not None: + architectures = getattr(model_config, "architectures", None) + if architectures: + architecture = ";".join(architectures) + + # Extract the results of the models + results = {} + print (data) + for task in Tasks: + task = task.value + + for k, v in data["tasks"].items(): + if task.benchmark[:-2] == k: + if "Best Prompt Id" in task.col_name: + results[task.benchmark] = int(v[task.metric_type][-1:]) + else: + #results[task.benchmark] = f"{v[task.metric_type]:.2f}" # Ensure two decimals for display + #print (v) + results[task.benchmark] = float(v[task.metric_type]) + #value = float(v[task.metric_type]) + #results[task.benchmark] = round(value, 2) # Arrotonda a 2 decimali + print ( "************ End of Reading file ****************") + return self( + eval_name=result_key+"_"+Lang, #result_key, + full_model=full_model, + Lang=Lang, + org=org, + model=model, + results=results, + average_CPS=average_CPS, + fewshot_symbol=fewshot_symbol, + is_5fewshot=is_5fewshot, + revision= config.get("model_sha", ""), + still_on_hub=still_on_hub, + architecture=architecture, + num_params=num_params, + rank = rank, + size_symbol=size_symbol + #submitted_time=config.get("submitted_time", ""), + ) + + ''' + def update_with_request_file(self, requests_path): + """Finds the relevant request file for the current model and updates info with it""" + request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name) + + try: + with open(request_file, "r") as f: + request = json.load(f) + self.model_type = ModelType.from_str(request.get("model_type", "")) + self.weight_type = WeightType[request.get("weight_type", "Original")] + self.license = request.get("license", "?") + self.likes = request.get("likes", 0) + self.num_params = request.get("params", 0) + self.date = request.get("submitted_time", "") + except Exception: + print(f"Could not find request file for {self.org}/{self.model} with precision + ''' + + def to_dict(self): + """Converts the Eval Result to a dict compatible with our dataframe display""" + average = self.average_CPS + + fewshot_symbol = ( + self.fewshot_symbol.value.symbol if isinstance(self.fewshot_symbol, FewShotType) else "❓" + ) + + size_symbol = ( + self.size_symbol.value.symbol if isinstance(self.size_symbol, SizeType) else "❓" + ) + + data_dict = { + "eval_name": self.eval_name, # not a column, just a save name, + #AutoEvalColumn.precision.name: self.precision.value.name, + #AutoEvalColumn.model_type.name: self.model_type.value.name, + #AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol, + #AutoEvalColumn.model_type.name: self.model_type.value.name if self.model_type else "Unknown", + #AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol if self.model_type else "Unknown", + AutoEvalColumn.fewshot_symbol.name: fewshot_symbol, + AutoEvalColumn.weight_type.name: self.weight_type.value.name, + AutoEvalColumn.architecture.name: self.architecture, + AutoEvalColumn.model.name: make_clickable_model(self.full_model), + AutoEvalColumn.revision.name: self.revision, + AutoEvalColumn.average.name: average, + AutoEvalColumn.is_5fewshot.name: self.is_5fewshot, + AutoEvalColumn.license.name: self.license, + AutoEvalColumn.likes.name: self.likes, + AutoEvalColumn.params.name: self.num_params, + AutoEvalColumn.still_on_hub.name: self.still_on_hub, + AutoEvalColumn.rank.name: self.rank, + AutoEvalColumn.size_symbol.name: size_symbol, + AutoEvalColumn.LANG.name:self.Lang + } + + for task in Tasks: + data_dict[task.value.col_name] = self.results[task.value.benchmark] + + return data_dict + + +def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]: + """From the path of the results folder root, extract all needed info for results""" + model_result_filepaths = [] + + for root, _, files in os.walk(results_path): + # We should only have json files in model results + if len(files) == 0 or any([not f.endswith(".json") for f in files]): + continue + + # Sort the files by date + try: + files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7]) + except dateutil.parser._parser.ParserError: + files = [files[-1]] + + for file in files: + model_result_filepaths.append(os.path.join(root, file)) + + eval_results = {} + for model_result_filepath in model_result_filepaths: + # Creation of result + eval_result = EvalResult.init_from_json_file(model_result_filepath) + #eval_result.update_with_request_file(requests_path) + + # Store results of same eval together + eval_name = eval_result.eval_name + if eval_name in eval_results.keys(): + eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None}) + else: + eval_results[eval_name] = eval_result + + results = [] + for v in eval_results.values(): + try: + v.to_dict() # we test if the dict version is complete + results.append(v) + except KeyError: # not all eval values present + continue + + return results diff --git a/src/populate.py b/src/populate.py new file mode 100644 index 0000000000000000000000000000000000000000..5bbaf385b52d6edf173633353b9458b76c868158 --- /dev/null +++ b/src/populate.py @@ -0,0 +1,59 @@ +import json +import os + +import pandas as pd + +from src.display.formatting import has_no_nan_values, make_clickable_model +from src.display.utils import AutoEvalColumn, EvalQueueColumn +from src.leaderboard.read_evals import get_raw_eval_results + + +def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame: + """Creates a dataframe from all the individual experiment results""" + raw_data = get_raw_eval_results(results_path, requests_path) + all_data_json = [v.to_dict() for v in raw_data] + + df = pd.DataFrame.from_records(all_data_json) + df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False) + df = df[cols].round(decimals=2) + #df.to_csv("output.csv", index=False) + + # filter out if any of the benchmarks have not been produced + df = df[has_no_nan_values(df, benchmark_cols)] + return df + + +def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]: + """Creates the different dataframes for the evaluation queues requestes""" + entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")] + all_evals = [] + + for entry in entries: + if ".json" in entry: + file_path = os.path.join(save_path, entry) + with open(file_path) as fp: + data = json.load(fp) + + data[EvalQueueColumn.model.name] = make_clickable_model(data["model"]) + data[EvalQueueColumn.revision.name] = data.get("revision", "main") + + all_evals.append(data) + elif ".md" not in entry: + # this is a folder + sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")] + for sub_entry in sub_entries: + file_path = os.path.join(save_path, entry, sub_entry) + with open(file_path) as fp: + data = json.load(fp) + + data[EvalQueueColumn.model.name] = make_clickable_model(data["model"]) + data[EvalQueueColumn.revision.name] = data.get("revision", "main") + all_evals.append(data) + + pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]] + running_list = [e for e in all_evals if e["status"] == "RUNNING"] + finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"] + df_pending = pd.DataFrame.from_records(pending_list, columns=cols) + df_running = pd.DataFrame.from_records(running_list, columns=cols) + df_finished = pd.DataFrame.from_records(finished_list, columns=cols) + return df_finished[cols], df_running[cols], df_pending[cols] diff --git a/src/submission/__pycache__/check_validity.cpython-310.pyc b/src/submission/__pycache__/check_validity.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..003d1bc369d07a5f69718e741182e92e5a3d16e8 Binary files /dev/null and b/src/submission/__pycache__/check_validity.cpython-310.pyc differ diff --git a/src/submission/__pycache__/submit.cpython-310.pyc b/src/submission/__pycache__/submit.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1b3c40cb1de073d2011bb643428ca80f61b5e5ba Binary files /dev/null and b/src/submission/__pycache__/submit.cpython-310.pyc differ diff --git a/src/submission/check_validity.py b/src/submission/check_validity.py new file mode 100644 index 0000000000000000000000000000000000000000..3c3ce45c4dacd2d600544c87584ee72c81d3b956 --- /dev/null +++ b/src/submission/check_validity.py @@ -0,0 +1,99 @@ +import json +import os +import re +from collections import defaultdict +from datetime import datetime, timedelta, timezone + +import huggingface_hub +from huggingface_hub import ModelCard +from huggingface_hub.hf_api import ModelInfo +from transformers import AutoConfig +from transformers.models.auto.tokenization_auto import AutoTokenizer + +def check_model_card(repo_id: str) -> tuple[bool, str]: + """Checks if the model card and license exist and have been filled""" + try: + card = ModelCard.load(repo_id) + except huggingface_hub.utils.EntryNotFoundError: + return False, "Please add a model card to your model to explain how you trained/fine-tuned it." + + # Enforce license metadata + if card.data.license is None: + if not ("license_name" in card.data and "license_link" in card.data): + return False, ( + "License not found. Please add a license to your model card using the `license` metadata or a" + " `license_name`/`license_link` pair." + ) + + # Enforce card content + if len(card.text) < 200: + return False, "Please add a description to your model card, it is too short." + + return True, "" + +def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]: + """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses.""" + try: + config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token) + if test_tokenizer: + try: + tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token) + except ValueError as e: + return ( + False, + f"uses a tokenizer which is not in a transformers release: {e}", + None + ) + except Exception as e: + return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None) + return True, None, config + + except ValueError: + return ( + False, + "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.", + None + ) + + except Exception as e: + return False, "was not found on hub!", None + + +def get_model_size(model_info: ModelInfo, precision: str): + """Gets the model size from the configuration, or the model name if the configuration does not contain the information.""" + try: + model_size = round(model_info.safetensors["total"] / 1e9, 3) + except (AttributeError, TypeError): + return 0 # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in example_app.py + + size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1 + model_size = size_factor * model_size + return model_size + +def get_model_arch(model_info: ModelInfo): + """Gets the model architecture from the configuration""" + return model_info.config.get("architectures", "Unknown") + +def already_submitted_models(requested_models_dir: str) -> set[str]: + """Gather a list of already submitted models to avoid duplicates""" + depth = 1 + file_names = [] + users_to_submission_dates = defaultdict(list) + + for root, _, files in os.walk(requested_models_dir): + current_depth = root.count(os.sep) - requested_models_dir.count(os.sep) + if current_depth == depth: + for file in files: + if not file.endswith(".json"): + continue + with open(os.path.join(root, file), "r") as f: + info = json.load(f) + file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}") + + # Select organisation + if info["model"].count("/") == 0 or "submitted_time" not in info: + continue + organisation, _ = info["model"].split("/") + users_to_submission_dates[organisation].append(info["submitted_time"]) + + return set(file_names), users_to_submission_dates diff --git a/src/submission/submit.py b/src/submission/submit.py new file mode 100644 index 0000000000000000000000000000000000000000..cac6ea48e803a0af42dabe5226191c769dbec71d --- /dev/null +++ b/src/submission/submit.py @@ -0,0 +1,119 @@ +import json +import os +from datetime import datetime, timezone + +from src.display.formatting import styled_error, styled_message, styled_warning +from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO +from src.submission.check_validity import ( + already_submitted_models, + check_model_card, + get_model_size, + is_model_on_hub, +) + +REQUESTED_MODELS = None +USERS_TO_SUBMISSION_DATES = None + +def add_new_eval( + model: str, + base_model: str, + revision: str, + precision: str, + weight_type: str, + model_type: str, +): + global REQUESTED_MODELS + global USERS_TO_SUBMISSION_DATES + if not REQUESTED_MODELS: + REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH) + + user_name = "" + model_path = model + if "/" in model: + user_name = model.split("/")[0] + model_path = model.split("/")[1] + + precision = precision.split(" ")[0] + current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + if model_type is None or model_type == "": + return styled_error("Please select a model type.") + + # Does the model actually exist? + if revision == "": + revision = "main" + + # Is the model on the hub? + if weight_type in ["Delta", "Adapter"]: + base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True) + if not base_model_on_hub: + return styled_error(f'Base model "{base_model}" {error}') + + if not weight_type == "Adapter": + model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True) + if not model_on_hub: + return styled_error(f'Model "{model}" {error}') + + # Is the model info correctly filled? + try: + model_info = API.model_info(repo_id=model, revision=revision) + except Exception: + return styled_error("Could not get your model information. Please fill it up properly.") + + model_size = get_model_size(model_info=model_info, precision=precision) + + # Were the model card and license filled? + try: + license = model_info.cardData["license"] + except Exception: + return styled_error("Please select a license for your model") + + modelcard_OK, error_msg = check_model_card(model) + if not modelcard_OK: + return styled_error(error_msg) + + # Seems good, creating the eval + print("Adding new eval") + + eval_entry = { + "model": model, + "base_model": base_model, + "revision": revision, + "precision": precision, + "weight_type": weight_type, + "status": "PENDING", + "submitted_time": current_time, + "model_type": model_type, + "likes": model_info.likes, + "params": model_size, + "license": license, + "private": False, + } + + # Check for duplicate submission + if f"{model}_{revision}_{precision}" in REQUESTED_MODELS: + return styled_warning("This model has been already submitted.") + + print("Creating eval file") + OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}" + os.makedirs(OUT_DIR, exist_ok=True) + out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json" + + with open(out_path, "w") as f: + f.write(json.dumps(eval_entry)) + + print("Uploading eval file") + API.upload_file( + path_or_fileobj=out_path, + path_in_repo=out_path.split("eval-queue/")[1], + repo_id=QUEUE_REPO, + repo_type="dataset", + commit_message=f"Add {model} to eval queue", + ) + + # Remove the local file + os.remove(out_path) + + return styled_message( + "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list." + ) diff --git a/src/tasks.py b/src/tasks.py new file mode 100644 index 0000000000000000000000000000000000000000..1b3c7f6764cf25ec0f0a0b3705ed89617235a505 --- /dev/null +++ b/src/tasks.py @@ -0,0 +1,183 @@ +from dataclasses import dataclass +from enum import Enum + +@dataclass +class Task: + benchmark: str + # metric: str + accuracy: str + col_name: str + +NUM_FEWSHOT = 0 # Change with your few shot +# --------------------------------------------------- + +# Your leaderboard name +TITLE = """

🚀 EVALITA-LLM Leaderboard 🚀

""" + +# What does your leaderboard evaluate? +INTRODUCTION_TEXT = """ +Evalita-LLM is a benchmark designed to evaluate Large Language Models (LLMs) on Italian tasks. The distinguishing features of Evalita-LLM are the following: (i) all tasks are native Italian, avoiding translation issues and potential cultural biases; (ii) the benchmark includes generative tasks, enabling more natural interaction with LLMs; (iii) all tasks are evaluated against multiple prompts, this way mitigating the model sensitivity to specific prompts and allowing a fairer evaluation. +""" + +#MEASURE_DESCRIPTION = "Combined Performance = (1 - (Best_Prompt - Prompt_Average) / 100) * Best_Prompt. Prompt Average = accuracy averaged over the six prompts. Best Prompt = accuracy of the best prompt. Prompt ID = ID of the best prompt (see legend above)" +MEASURE_DESCRIPTION = "**Combined Performance** = (1 - (**Best Prompt** - **Prompt Average**) / 100) * **Best Prompt**. **Prompt Average** = accuracy averaged over the assessed prompts. **Best Prompt** = accuracy of the best prompt. **Prompt ID** = ID of the best prompt (see legend above)." +#MEASURE_DESCRIPTION = "**Combined Performance** = (1 - (**Best Prompt** - **Prompt Average**) / 100) * **Best Prompt**. **Prompt Average** = avg. accuracy over prompts. **Best Prompt** = accuracy of best prompt. **Prompt ID** = ID of the best prompt (see legend above)." + +# Tasks Descriptions +TE_DESCRIPTION = """### Textual Entailment (TE) --- *Multiple-choice task* + The input are two sentences: the text (T) and the hypothesis (H). The model has to determine whether the meaning of the hypothesis is logically entailed by the text. + +| # | Prompt | Answer Choices | +|-----|------------|--------------| +| 1 | La frase: '{{text1}}' implica logicamente che la frase: '{{text2}}' sia vera? | ["Sì", "No"] | +| 2 | Devi risolvere un compito di inferenza semantica. La frase: '{{text1}}' implica logicamente che la frase: '{{text2}}' sia vera? | ["Sì", "No"] | +| 3 | La frase: '{{text1}}' implica logicamente che la frase: '{{text2}}' sia vera?\\nA: Sì\\nB: No\\nRisposta: | ["A", "B"] | +| 4 | Devi risolvere un compito di inferenza semantica. La frase: '{{text1}}' implica logicamente che la frase: '{{text2}}' sia vera?\\nA: Sì\\nB: No\\nRisposta: | ["A", "B"] | +| 5 | Frase 1: '{{text1}}' Frase 2: '{{text2}}' | ["La frase 1 implica logicamente che la frase 2 sia vera", "La frase 1 non implica logicamente che la frase 2 sia vera"] | +| 6 | Devi risolvere un compito di inferenza semantica. Frase 1: '{{text1}}' Frase 2: '{{text2}}' | ["La frase 1 implica logicamente che la frase 2 sia vera", "La frase 1 non implica logicamente che la frase 2 sia vera"] | + +**Combined Performance** = (1 - (**Best Prompt** - **Prompt Average**) / 100) * **Best Prompt**. **Prompt Average** = accuracy averaged over the 6 prompts. **Best Prompt** = accuracy of the best prompt. **Prompt ID** = ID of the best prompt (see legend above). + +""" + +SA_DESCRIPTION = """### Sentiment Analysis (SA) --- *Multiple-choice task* + The input is a tweet. The model has to determine the sentiment polarity of the text, categorizing it into one of four classes: positive, negative, neutral, or mixed. + +| # | Prompt | Answer Choices | +|-----|--------------------------------------------------------------------------------|-----------------------------| +| 1 | Qual è il sentiment espresso nel seguente tweet: '{{text}}'? | ["Positivo", "Negativo", "Neutro", "Misto"] | +| 2 | Devi svolgere un compito di analisi del sentiment. Qual è il sentiment espresso nel seguente tweet: '{{text}}'? | ["Positivo", "Negativo", "Neutro", "Misto"] | +| 3 | Qual è il sentiment espresso nel seguente tweet: '{{text}}'?\\nA: Positivo\\nB: Negativo\\nC: Neutro\\nD: Misto\\nRisposta: | ["A", "B", "C", "D"] | +| 4 | Devi svolgere un compito di analisi del sentiment. Qual è il sentiment espresso nel seguente tweet: '{{text}}'?\\nA: Positivo\\nB: Negativo\\nC: Neutro\\nD: \\nRisposta: | ["A", "B", "C", "D"] | +| 5 | Il seguente tweet: '{{text}}' esprime un sentiment | ["Positivo", "Negativo", "Neutro", "Misto"] | +| 6 | Devi svolgere un compito di analisi del sentiment. Il seguente tweet: '{{text}}' esprime un sentiment | ["Positivo", "Negativo", "Neutro", "Misto"] | + +**Combined Performance** = (1 - (**Best Prompt** - **Prompt Average**) / 100) * **Best Prompt**. **Prompt Average** = F1-macro averaged over the 6 prompts. **Best Prompt** = F1-macro of the best prompt. **Prompt ID** = ID of the best prompt (see legend above). + +""" + +HS_DESCRIPTION = """### Hate Speech (HS) --- *Multiple-choice task* + The input is a tweet. The model has to determine whether the text contains hateful content directed towards marginalized or minority groups. The output is a binary classification: hateful or not hateful. + +| # | Prompt | Answer Choices | +|-----|--------------------------------------------------------------------------------|-------------------------------------------------| +| 1 | C'è incitamento all'odio nel seguente tweet: '{{full_text}}'? | ["Falso", "Vero"] | +| 2 | Devi svolgere un compito di identificazione di incitamento all'odio. C'è incitamento all'odio nel seguente tweet: '{{full_text}}'? | ["Falso", "Vero"] | +| 3 | C'è incitamento all'odio nel seguente tweet: '{{full_text}}'?\\nA: Vero\\nB: Falso\\nRisposta: | ["B", "A"] | +| 4 | Devi svolgere un compito di identificazione di incitamento all'odio. C'è incitamento all'odio nel seguente tweet: '{{full_text}}'?\\nA: Vero\\nB: Falso\\nRisposta: | ["B", "A"] | +| 5 | Il tweet: '{{full_text}}' | ["non contiene incitamento all'odio", "contiene incitamento all'odio"] | +| 6 | Devi svolgere un compito di identificazione di incitamento all'odio. Il tweet: '{{full_text}}' | ["non contiene incitamento all'odio", "contiene incitamento all'odio"] | + +**Combined Performance** = (1 - (**Best Prompt** - **Prompt Average**) / 100) * **Best Prompt**. **Prompt Average** = F1-micro averaged over the 6 prompts. **Best Prompt** = F1-micro of the best prompt. **Prompt ID** = ID of the best prompt (see legend above). + +""" + +AT_DESCRIPTION = """### Admission Tests (AT) --- *Multiple-choice task* + The input is a multiple-choice question with five options (A-E) from Italian medical specialty entrance exams, and the model must identify the correct answer. + +| # | Prompt | Answer Choices | +|-----|--------------------------------------------------------------------------------|-----------------------------| +| 1 | Dato il seguente quesito di medicina: '{{Question}}' qual è la risposta corretta? | ["A", "B", "C", "D", "E"] | +| 2 | Devi risolvere un compito di risposte a domande. Dato il seguente quesito di medicina: '{{Question}}' qual è la risposta corretta? | ["A", "B", "C", "D", "E"] | +| 3 | Dato il seguente quesito di medicina: '{{Question}}' qual è la risposta corretta?\\nA: {{A}}\\nB: {{B}}\\nC: {{C}}\\nD: {{D}}\\nE: {{E}}\\nRisposta: | ["A", "B", "C", "D", "E"] | +| 4 | Devi risolvere un compito a scelta multipla. Dato il seguente caso clinico: '{{background}}', qual è la risposta corretta alla domanda: '{{domanda}}'?\\nA: {{A}}\\nB: {{B}}\\nC: {{C}}\\nD: {{D}}\\nE: {{E}}\\nRisposta:Devi risolvere un compito a scelta multipla. Dato il seguente quesito di medicina: '{{Question}}' qual è la risposta corretta?\\nA: {{A}}\\nB: {{B}}\\nC: {{C}}\\nD: {{D}}\\nE: {{E}}\\nRisposta: | ["A", "B", "C", "D", "E"] | +| 5 | Dato il seguente quesito di medicina '{{Question}}' la risposta corretta è: | ["A", "B", "C", "D", "E"] | +| 6 | Devi risolvere un compito di risposte a domande. Dato il seguente quesito di medicina '{{Question}}' la risposta corretta è: | ["A", "B", "C", "D", "E"] | + +**Combined Performance** = (1 - (**Best Prompt** - **Prompt Average**) / 100) * **Best Prompt**. **Prompt Average** = accuracy averaged over the 6 prompts. **Best Prompt** = accuracy of the best prompt. **Prompt ID** = ID of the best prompt (see legend above). + +""" + +WIC_DESCRIPTION = """### Word in Context (WIC) --- *Multiple-choice task* + The input consists of a word (w) and two sentences. The model has to determine whether the word w has the same meaning in both sentences. The output is a binary classification: 1 (same meaning) or 0 (different meaning). + +| # | Prompt | Answer Choices | +|-----|--------------------------------------------------------------------------------|-------------------------------------------------| +| 1 | La parola: '{{sentence1[start1:end1]}}' nella frase: '{{sentence1}}' ha lo stesso significato della parola: '{{sentence2[start2:end2]}}' nella frase: '{{sentence2}}'? | ["No", "Sì"] | +| 2 | Devi determinare se una stessa parola usata in due frasi differenti ha lo stesso significato in entrambi i contesti. La parola: '{{sentence1[start1:end1]}}' nella frase: '{{sentence1}}' ha lo stesso significato della parola: '{{sentence2[start2:end2]}}' nella frase: '{{sentence2}}'? | ["No", "Sì"] | +| 3 | La parola: '{{sentence1[start1:end1]}}' nella frase: '{{sentence1}}' ha lo stesso significato della parola: '{{sentence2[start2:end2]}}' nella frase: '{{sentence2}}'?\\nA: Sì\\nB: No\\nRisposta: | ["B", "A"] | +| 4 | Devi determinare se una stessa parola usata in due frasi differenti ha lo stesso significato in entrambi i contesti. La parola: '{{sentence1[start1:end1]}}' nella frase: '{{sentence1}}' ha lo stesso significato della parola: '{{sentence2[start2:end2]}}' nella frase: '{{sentence2}}'?\\nA: \\nB: No\\nRisposta: | ["B", "A"] | +| 5 | La parola: '{{sentence1[start1:end1]}}' nella frase: '{{sentence1}}' e la parola: '{{sentence2[start2:end2]}}' nella frase: '{{sentence2}}' | ["non hanno lo stesso significato", "hanno lo stesso significato"] | +| 6 | Devi determinare se una stessa parola usata in due frasi differenti ha lo stesso significato in entrambi i contesti. La parola: '{{sentence1[start1:end1]}}' nella frase: '{{sentence1}}' e la parola: '{{sentence2[start2:end2]}}' nella frase: '{{sentence2}}' | ["non hanno lo stesso significato", "hanno lo stesso significato"] | + +**Combined Performance** = (1 - (**Best Prompt** - **Prompt Average**) / 100) * **Best Prompt**. **Prompt Average** = F1-macro averaged over the 6 prompts. **Best Prompt** = F1-macro of the best prompt. **Prompt ID** = ID of the best prompt (see legend above). + +""" + +FAQ_DESCRIPTION = """### Frequently Asked Questions & Question Answering (FAQ) --- *Multiple-choice task* + The input is a user query regarding the water supply service. The model must identify the correct answer from the 4 available options. + +| # | Prompt | Answer Choices | +|-----|--------------------------------------------------------------------------------|-----------------------------| +| 1 | Rispondi alla seguente domanda: '{{question}}' | {{[A, B, C, D]}} | +| 2 | Devi risolvere un compito di risposte a domande. Rispondi alla seguente domanda: '{{question}}' | {{[A, B, C, D]}} | +| 3 | Rispondi alla seguente domanda: '{{question}}'\\nA: {{A}}\\nB: {{B}}\\nC: {{C}}\\nD: {{D}}\\nRisposta: | ["A", "B", "C", "D"] | +| 4 | Devi risolvere un compito a scelta multipla. Rispondi alla seguente domanda: '{{question}}'\\nA: {{A}}\\nB: {{B}}\\nC: {{C}}\\nD: {{D}}\\nRisposta: | ["A", "B", "C", "D"] | +| 5 | La risposta alla domanda: '{{question}}' è: | {{[A, B, C, D]}} | +| 6 | Devi risolvere un compito di risposte a domande. La risposta alla domanda: '{{question}}' è: | {{[A, B, C, D]}} | + +**Combined Performance** = (1 - (**Best Prompt** - **Prompt Average**) / 100) * **Best Prompt**. **Prompt Average** = accuracy averaged over the 6 prompts. **Best Prompt** = accuracy of the best prompt. **Prompt ID** = ID of the best prompt (see legend above). + +""" + +LS_DESCRIPTION = """### Lexical Substitution (LS) --- *Generative task* + The input is a sentence containing a target word (w). The model has to replace the target word w with its most suitable synonyms that are contextually relevant. + +| # | Prompt | +|-----|--------------------------------------------------------------------------------| +| 7 | Trova 10 parole che possono sostituire la parola racchiusa tra i marcatori `` nella seguente frase: '{{context}}', mantenendo lo stesso significato. Elenca i lemmi (forme base) di queste parole, separandoli con una virgola, ad esempio: lemma1, lemma2, lemma3, lemma4, lemma5. Non aggiungere commenti o altro testo. Risposta: | +| 8 | Devi risolvere un compito di sostituzione lessicale. Trova 10 parole che possono sostituire la parola racchiusa tra i marcatori `` nella seguente frase: '{{context}}', mantenendo lo stesso significato. Elenca i lemmi (forme base) di queste parole, separandoli con una virgola, ad esempio: lemma1, lemma2, lemma3, lemma4, lemma5. Non aggiungere commenti o altro testo. Risposta: | + +**Combined Performance** = (1 - (**Best Prompt** - **Prompt Average**) / 100) * **Best Prompt**. **Prompt Average** = F1 averaged over the 2 prompts. **Best Prompt** = F1 of the best prompt. **Prompt ID** = ID of the best prompt (see legend above). + +""" + +SU_DESCRIPTION = """### Summarization (SUM) --- *Generative task* + The input is a news article. The model has to generate a concise summary of the input text, capturing the key information and main points. + +| # | Prompt | +|-----|--------------------------------------------------------------------------------| +| 7 | Riassumi il seguente articolo di giornale: '{{source}}'\\nRiassunto: | +| 8 | Devi risolvere un compito di sintesi automatica del testo. Riassumi il seguente articolo di giornale: '{{source}}'\\nRiassunto: | + +**Combined Performance** = (1 - (**Best Prompt** - **Prompt Average**) / 100) * **Best Prompt**. **Prompt Average** = F1 averaged over the 2 prompts. **Best Prompt** = F1 of the best prompt. **Prompt ID** = ID of the best prompt (see legend above). + +""" + +NER_DESCRIPTION = """### Named Entity Recognition (NER) --- *Generative task* + The input is a sentence. The model has to identify and classify Named Entities into predefined categories such as person, organization, and location. + +| # | Prompt | +|-----|--------------------------------------------------------------------------------| +| 7 | Estrai tutte le entità di tipo PER (persona), LOC (luogo) e ORG (organizzazione) dal testo seguente. Riporta ogni entità con il formato: Entità$Tipo, separando ciascuna coppia con ','. Se non ci sono entità da estrarre, rispondi con '&&NOENT&&'.\\nTesto: '{{text}}'\\nEntità: | +| 8 | Devi svolgere un compito di riconoscimento delle entità nei testi. Estrai tutte le entità di tipo PER (persona), LOC (luogo) e ORG (organizzazione) dal testo seguente. Riporta ogni entità con il formato: Entità$Tipo, separando ciascuna coppia con ','. Se non ci sono entità da estrarre, rispondi con '&&NOENT&&'.\\nTesto: '{{text}}'\\nEntità: | + +**Combined Performance** = (1 - (**Best Prompt** - **Prompt Average**) / 100) * **Best Prompt**. **Prompt Average** = F1 averaged over the 2 prompts. **Best Prompt** = F1 of the best prompt. **Prompt ID** = ID of the best prompt (see legend above). + +""" + +REL_DESCRIPTION = """### Relation Extraction (REL) --- *Generative task* + The input is a sentence of a clinical text. The model must identify and extract relationships between laboratory test results (e.g., blood pressure) and the corresponding tests or procedures that generated them (e.g., blood pressure test). + +| # | Prompt | +|-----|--------------------------------------------------------------------------------| +| 7 | Dato un documento medico devi estrarre tutte le misurazioni degli esami medici presenti. Riporta ogni relazione nel formato: misurazione$esame, separando ciascuna coppia con '%'. Se non ci sono relazioni da estrarre, rispondi con '&&NOREL&&'.\\nTesto: '{{text}}'\\nRelazioni: | +| 8 | Devi svolgere un compito di estrazione di relazioni da documenti medici. Dato un documento medico devi estrarre tutte le misurazioni degli esami medici presenti. Riporta ogni relazione nel formato: misurazione$esame, separando ciascuna coppia con '%'. Se non ci sono relazioni da estrarre, rispondi con '&&NOREL&&'.\\nTesto: '{{text}}'\\nRelazioni: | + +**Combined Performance** = (1 - (**Best Prompt** - **Prompt Average**) / 100) * **Best Prompt**. **Prompt Average** = F1 averaged over the 2 prompts. **Best Prompt** = F1 of the best prompt. **Prompt ID** = ID of the best prompt (see legend above). + +""" + +# Create a dictionary to map task names to their descriptions +TASK_DESCRIPTIONS = { + "TE": TE_DESCRIPTION, + "SA": SA_DESCRIPTION, + "HS": HS_DESCRIPTION, + "AT": AT_DESCRIPTION, + "WIC": WIC_DESCRIPTION, + "FAQ": FAQ_DESCRIPTION, + "LS": LS_DESCRIPTION, + "SU": SU_DESCRIPTION, + "NER": NER_DESCRIPTION, + "REL": REL_DESCRIPTION +} \ No newline at end of file