app.py

import gradio as gr
import plotly.express as px
from pathlib import Path
import pandas as pd
import numpy as np
from langchain_openai import ChatOpenAI
from langchain_experimental.agents.agent_toolkits import create_pandas_dataframe_agent
from langchain.agents.agent_types import AgentType
from langchain_google_genai import ChatGoogleGenerativeAI
import plotly.graph_objects as go

def explain_df(query, df):
    agent = create_pandas_dataframe_agent(
        # ChatOpenAI(
        #     base_url="https://fmapi.swissai.cscs.ch",
        #     temperature=0.01,
        #     model="meta-llama/Llama-3.3-70B-Instruct"
        # ),
        ChatGoogleGenerativeAI(
            model="gemini-1.5-pro",
            temperature=0,
            max_tokens=None,
            timeout=None,
            max_retries=2,
        ),
        df,
        verbose=True,
        allow_dangerous_code=True,
    )
    response = agent.invoke(query)
    return response['output']

    
abs_path = Path(__file__).parent

def parse_model_args(model_args):
    if "deltazip" in model_args:
        model_args = model_args.split("deltazip")[1]
        model_args = model_args.split(",")[0]
        model_args = model_args.strip(".")
        model_args = model_args.replace(".", "/")
    if "espressor/" in model_args:
        model_args = model_args.split("espressor/")[1]
        model_args = model_args.split(",")[0]
        model_args = model_args.strip(".")
        model_args = model_args.replace(".", "/",1)
        model_args = model_args.split("_")[0]
    else:
        model_args = model_args.split(",")[0]
        model_args = model_args.replace("pretrained=", "")
    return model_args

def parse_model_precision(model_args):
    if "espressor" in model_args:
        if 'W8A8_int8' in model_args:
            precision = 'W8A8_int8'
        else:
            precision = model_args.split("_")[-1]
    else:
        precision = "Default"
    return precision

# Any pandas-compatible data
df = pd.read_csv(str(abs_path / "eval_results.csv"))
perf_df = pd.read_csv(str(abs_path / "perfbench_results.csv"))
# take acc only
df = df[df['metric'] == 'acc']
# dedup
df = df.drop_duplicates(subset=['model', 'task'])
# pivot df, such that the column names are model,task,efficiency
# but keep precision in its original place
df['model_physical_size'] = df['model_physical_size'].apply(lambda x: x/1024/1024/1024)

df = df.pivot(index=['model','hf_name','model_physical_size'], columns='task', values='value').reset_index()

df['precision'] = df['model'].apply(lambda x: x.split(":")[-1])
df['model'] = df['model'].apply(lambda x: x.split(":")[0])
df['avg_acc'] = df.filter(like='task_').mean(axis=1)

df = df.rename(columns=lambda x: x.replace('task_', ''))
numeric_columns = df.select_dtypes(include=[np.number]).columns
# remove physical size from numeric columns
numeric_columns = numeric_columns.drop('model_physical_size')
df[numeric_columns] = (df[numeric_columns]*100).round(2)
df['model_physical_size'] = df['model_physical_size'].round(2)

full_df = df.merge(perf_df, left_on='hf_name', right_on='hf_name', how='left')

with gr.Blocks() as demo:
    gr.Markdown("""
    # 🥇 Efficient LLM Leaderboard
    """)
    with gr.Tabs():
        with gr.TabItem("Leaderboard"):
            # ...existing code...
            task_options = [col for col in df.columns if col not in ['model','hf_name','model_physical_size', 'precision']]
            task_options.append("plot_pareto")
            with gr.Row():
                # print pareto or not
                selected_tasks = gr.CheckboxGroup(choices=task_options, label="Select Tasks")
            with gr.Row():
                accuracy_plot = gr.Plot(label="Accuracy Plot")
                line_plot = gr.Plot(label="Average Accuracy vs Model Size")
            with gr.Row():
                throughput_line_plot = gr.Plot(label="Throughput vs Average Accuracy")
                latency_line_plot = gr.Plot(label="Latency vs Average Accuracy")
            with gr.Row():
                data_table = gr.Dataframe(value=df, label="Result Table")

            def update_outputs(selected_tasks):
                if not selected_tasks:
                    return df[['model', 'precision']], None, None
                plot_pareto=False
                if "plot_pareto" in selected_tasks:
                    plot_pareto = True
                    selected_tasks.remove("plot_pareto")
                filtered_df = df[['model', 'precision', 'model_physical_size','hf_name'] + selected_tasks]
                # average accuracy of selected tasks
                filtered_df['avg_accuracy'] = filtered_df[selected_tasks].mean(axis=1)
                
                bar_fig = px.bar(filtered_df, x='model', y='avg_accuracy', color='precision', barmode='group')
                line_fig = px.line(filtered_df, x='model_physical_size', y='avg_accuracy', color='model', symbol='precision')
                pareto_df = filtered_df.sort_values('model_physical_size')
                pareto_df = pareto_df.loc[pareto_df['avg_accuracy'].cummax().drop_duplicates().index]
                # Add Pareto frontier to line_plot
                if plot_pareto:
                    line_fig.add_trace(go.Scatter(
                        x=pareto_df['model_physical_size'],
                        y=pareto_df['avg_accuracy'],
                        mode='lines+markers',
                        name='Pareto Frontier'
                    ))
                
                # set title of bar_fig
                bar_fig.update_layout(title=f'tasks: {", ".join(selected_tasks)}')
                line_fig.update_layout(title=f'tasks: {", ".join(selected_tasks)}')
                with_perf_df = filtered_df.merge(perf_df, left_on='hf_name', right_on='hf_name', how='left')
                throughput_line_fig = px.line(with_perf_df, x='output_throughput', y='avg_accuracy', color='model', symbol='precision')
                latency_line_fig = px.line(with_perf_df, x="avg_e2e_latency", y='avg_accuracy', color='model', symbol='precision')
                
                pareto_df = with_perf_df.sort_values('avg_e2e_latency')
                pareto_df = pareto_df.loc[pareto_df['avg_accuracy'].cummax().drop_duplicates().index]
                if plot_pareto:
                    latency_line_fig.add_trace(go.Scatter(
                        x=pareto_df['avg_e2e_latency'],
                        y=pareto_df['avg_accuracy'],
                        mode='lines+markers',
                        name='Pareto Frontier'
                    ))
                return with_perf_df, bar_fig, line_fig, throughput_line_fig, latency_line_fig
            
            selected_tasks.change(
                fn=update_outputs,
                inputs=selected_tasks, 
                outputs=[data_table, accuracy_plot, line_plot, throughput_line_plot, latency_line_plot]
            )
        with gr.TabItem("Find Model"):
            query_input = gr.Textbox(label="Enter your query", placeholder="Enter your query here")
            response_output = gr.Textbox(label="Response", interactive=False)
            query_input.submit(
                fn=lambda query: explain_df(query, df),
                inputs=query_input,
                outputs=response_output
            )
    
if __name__ == "__main__":
    demo.launch(share=True)