Add data for ARC-AGI and Simple Bench

app.py

@@ -159,6 +159,150 @@ def create_size_for_performance_plot(category_to_display: str,
             gr.Dropdown(choices=list(elo_ratings_for_category.keys()), value=model_to_compare, interactive=True))
 
 
+def create_arc_agi_plot() -> go.Figure:
+    arc_agi_leaderboard = []
+    with open("arc_agi_leaderboard.jsonl", 'r') as file:
+        for line in file:
+            arc_agi_leaderboard.append(json.loads(line))
+
+    models = []
+    with open("models.jsonl", 'r') as file:
+        for line in file:
+            models.append(json.loads(line))
+
+    data = []
+    for entry in arc_agi_leaderboard:
+        model_name = entry['model']
+        score = entry['score']
+        model_info = next((m for m in models if m['Name'] == model_name), None)
+        if model_info:
+            release_date = datetime.strptime(model_info['Release Date'], "%Y-%m-%d")
+            data.append({'model': model_name, 'score': score, 'release_date': release_date})
+        else:
+            print(f"[WARNING] Model '{model_name}' not found in models.jsonl")
+
+    data.sort(key=lambda x: x['release_date'])
+
+    x_dates = [d['release_date'] for d in data]
+    y_scores = []
+    max_score = 0
+    for entry in data:
+        if entry['score'] > max_score:
+            max_score = entry['score']
+        y_scores.append(max_score)
+
+    fig = go.Figure()
+
+    fig.add_trace(go.Scatter(
+        x=x_dates,
+        y=y_scores,
+        mode='lines',
+        line=dict(shape='hv', width=2),
+        name='ARC-AGI Score'
+    ))
+
+    for i, entry in enumerate(data):
+        if i == 0 or y_scores[i] > y_scores[i - 1]:
+            fig.add_trace(go.Scatter(
+                x=[entry['release_date']],
+                y=[entry['score']],
+                mode='markers+text',
+                marker=dict(size=10),
+                text=[entry['model']],
+                textposition="top center",
+                name=entry['model']
+            ))
+
+    fig.update_layout(
+        title='ARC-AGI Score Progression Over Time',
+        xaxis_title='Release Date',
+        yaxis_title='ARC-AGI Score',
+        hovermode='x unified',
+        xaxis=dict(
+            range=[date(2024, 5, 13), date(2024, 9, 17)],
+            type='date'
+        ),
+        yaxis=dict(
+            range=[0, 100]
+        ),
+        height=800
+    )
+
+    return fig
+
+
+def create_simple_bench_plot() -> go.Figure:
+    simple_bench_leaderboard = []
+    with open("simple_bench_leaderboard.jsonl", 'r') as file:
+        for line in file:
+            simple_bench_leaderboard.append(json.loads(line))
+
+    models = []
+    with open("models.jsonl", 'r') as file:
+        for line in file:
+            models.append(json.loads(line))
+
+    data = []
+    for entry in simple_bench_leaderboard:
+        model_name = entry['model']
+        score = entry['score']
+        model_info = next((m for m in models if m['Name'] == model_name), None)
+        if model_info:
+            release_date = datetime.strptime(model_info['Release Date'], "%Y-%m-%d")
+            data.append({'model': model_name, 'score': score, 'release_date': release_date})
+        else:
+            print(f"[WARNING] Model '{model_name}' not found in models.jsonl")
+
+    data.sort(key=lambda x: x['release_date'])
+
+    x_dates = [d['release_date'] for d in data]
+    y_scores = []
+    max_score = 0
+    for entry in data:
+        if entry['score'] > max_score:
+            max_score = entry['score']
+        y_scores.append(max_score)
+
+    fig = go.Figure()
+
+    fig.add_trace(go.Scatter(
+        x=x_dates,
+        y=y_scores,
+        mode='lines',
+        line=dict(shape='hv', width=2),
+        name='Simple Bench Score'
+    ))
+
+    for i, entry in enumerate(data):
+        if i == 0 or y_scores[i] > y_scores[i - 1]:
+            fig.add_trace(go.Scatter(
+                x=[entry['release_date']],
+                y=[entry['score']],
+                mode='markers+text',
+                marker=dict(size=10),
+                text=[entry['model']],
+                textposition="top center",
+                name=entry['model']
+            ))
+
+    fig.update_layout(
+        title='Simple Bench Score Progression Over Time',
+        xaxis_title='Release Date',
+        yaxis_title='Simple Bench Score',
+        hovermode='x unified',
+        xaxis=dict(
+            range=[date(2023, 6, 13), date(2024, 8, 14)],
+            type='date'
+        ),
+        yaxis=dict(
+            range=[0, 100]
+        ),
+        height=800
+    )
+
+    return fig
+
+
 with gr.Blocks() as demo:
     with gr.Tab("Finance"):
         with gr.Tab("Big Five Capex") as big_five_capex_tab:
@@ -188,30 +332,32 @@ with gr.Blocks() as demo:
             )
         with gr.Tab("API Cost for Specific Performance Level", interactive=False):
             api_cost_for_performance_plot: gr.Plot = gr.Plot()
-    with gr.Tab("System Performance Over Time", interactive=False):
-        with gr.Tab("ARC-AGI"):
+    with gr.Tab("System Performance Over Time"):
+        with gr.Tab("ARC-AGI") as arc_agi_tab:
             arc_agi_plot: gr.Plot = gr.Plot()
-        with gr.Tab("BigCodeBench"):
+        with gr.Tab("Simple Bench") as simple_bench_tab:
+            simple_bench_plot: gr.Plot = gr.Plot()
+        with gr.Tab("BigCodeBench", interactive=False):
             bigcodebench_plot: gr.Plot = gr.Plot()
-        with gr.Tab("Codeforces"):
+        with gr.Tab("Codeforces", interactive=False):
             codeforces_plot: gr.Plot = gr.Plot()
-        with gr.Tab("GAIA"):
+        with gr.Tab("GAIA", interactive=False):
             gaia_plot: gr.Plot = gr.Plot()
-        with gr.Tab("GPQA"):
+        with gr.Tab("GPQA", interactive=False):
             gpqa_plot: gr.Plot = gr.Plot()
-        with gr.Tab("HumanEval"):
+        with gr.Tab("HumanEval", interactive=False):
             humaneval_plot: gr.Plot = gr.Plot()
-        with gr.Tab("LMSYS"):
+        with gr.Tab("LMSYS", interactive=False):
             lmsys_plot: gr.Plot = gr.Plot()
-        with gr.Tab("OpenCompass"):
+        with gr.Tab("MATH", interactive=False):
+            math_plot: gr.Plot = gr.Plot()
+        with gr.Tab("OpenCompass", interactive=False):
             opencompass_plot: gr.Plot = gr.Plot()
-        with gr.Tab("SWE-bench"):
+        with gr.Tab("SWE-bench", interactive=False):
             swe_bench_plot: gr.Plot = gr.Plot()
-        with gr.Tab("Simple Bench"):
-            simple_bench_plot: gr.Plot = gr.Plot()
-        with gr.Tab("WebArena"):
+        with gr.Tab("WebArena", interactive=False):
             webarena_plot: gr.Plot = gr.Plot()
-        with gr.Tab("ZeroEval"):
+        with gr.Tab("ZeroEval", interactive=False):
             zeroeval_plot: gr.Plot = gr.Plot()
     with gr.Tab("Frontier Language Model Training Runs", interactive=False):
         with gr.Tab("Street Price of GPUs Used"):
@@ -228,6 +374,8 @@ with gr.Blocks() as demo:
                                       outputs=[size_for_performance_plot,
                                                size_for_performance_category_dropdown,
                                                size_for_performance_comparison_model_dropdown])
+    arc_agi_tab.select(fn=create_arc_agi_plot, outputs=arc_agi_plot)
+    simple_bench_tab.select(fn=create_simple_bench_plot, outputs=simple_bench_plot)
 
 
 if __name__ == "__main__":

arc_agi_leaderboard.jsonl

@@ -0,0 +1,5 @@
+{"model": "o1-preview-2024-09-12", "score": 21}
+{"model": "claude-3-5-sonnet-20240620", "score": 21}
+{"model": "o1-mini-2024-09-12", "score": 13}
+{"model": "gpt-4o-2024-05-13", "score": 9}
+{"model": "gemini-1.5-pro-001", "score": 8}

models.jsonl

@@ -1,3 +1,5 @@
+{"Name": "o1-preview-2024-09-12", "Release Date": "2024-09-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "o1-mini-2024-09-12", "Release Date": "2024-09-12", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "deepseek-v2.5", "Release Date": "2024-09-05", "Total Parameters": 236, "Active Parameters": 236, "API Cost": 0}
 {"Name": "qwen-plus-0828", "Release Date": "2024-08-28", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "qwen-plus-0828", "Release Date": "2024-08-28", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
@@ -37,6 +39,7 @@
 {"Name": "yi-large", "Release Date": "2024-06-16", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "nemotron-4-340b-instruct", "Release Date": "2024-06-14", "Total Parameters": 340, "Active Parameters": 340, "API Cost": 0}
 {"Name": "bard-jan-24-gemini-pro", "Release Date": "2024-01-01", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
+{"Name": "gemini-1.5-pro-001", "Release Date": "2024-02-15", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}
 {"Name": "glm-4-0520", "Release Date": "2024-05-20", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0.63}
 {"Name": "llama-3-70b-instruct", "Release Date": "2024-04-18", "Total Parameters": 70, "Active Parameters": 70, "API Cost": 0}
 {"Name": "claude-3-sonnet-20240229", "Release Date": "2024-02-29", "Total Parameters": 0, "Active Parameters": 0, "API Cost": 0}

simple_bench_leaderboard.jsonl

@@ -0,0 +1,10 @@
+{"model": "claude-3-5-sonnet-20240620", "score": 27}
+{"model": "gpt-4-1106-preview", "score": 26}
+{"model": "claude-3-opus-20240229", "score": 25}
+{"model": "llama-3.1-405b-instruct-fp8", "score": 22}
+{"model": "gemini-1.5-pro-001", "score": 21}
+{"model": "gpt-4-0613", "score": 18}
+{"model": "gpt-4o-2024-05-13", "score": 16}
+{"model": "deepseek-v2-api-0628", "score": 15}
+{"model": "mistral-large-2407", "score": 13}
+{"model": "gpt-4o-mini-2024-07-18", "score": 5}