init raw app

.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+*.DS_Store

README.md

@@ -1,12 +1,11 @@
 ---
-title: AMA Bench
-emoji: 👀
-colorFrom: purple
-colorTo: blue
+title: AMA-Bench Leaderboard
+emoji: 🧠
+colorFrom: blue
+colorTo: purple
 sdk: gradio
-sdk_version: 6.5.1
+sdk_version: 5.23.3
 app_file: app.py
 pinned: false
+license: apache-2.0
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,330 @@
+import gradio as gr
+import pandas as pd
+import json
+import numpy as np
+import plotly.graph_objects as go
+
+# ---------------------------------------------------------------------------
+# Data loading
+# ---------------------------------------------------------------------------
+
+def load_data(path):
+    with open(path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+MODEL_DATA = load_data("data/model_data.json")
+METHOD_DATA = load_data("data/method_data.json")
+
+METRICS = ["Recall", "Causal Inference", "State Updating", "State Abstraction"]
+ALL_METRICS = METRICS + ["Average"]
+
+# ---------------------------------------------------------------------------
+# DataFrame helpers
+# ---------------------------------------------------------------------------
+
+def build_dataframe(data):
+    """Build a pandas DataFrame showing Accuracy (F1) for each metric."""
+    rows = []
+    for entry in data["entries"]:
+        row = {"Method": entry["method"]}
+        if entry.get("category"):
+            row["Category"] = entry["category"]
+        for m in ALL_METRICS:
+            acc = entry["scores"][m]["accuracy"]
+            f1 = entry["scores"][m]["f1"]
+            row[m] = f"{acc:.4f} ({f1:.4f})"
+        # Store raw average accuracy for sorting
+        row["_sort_avg"] = entry["scores"]["Average"]["accuracy"]
+        rows.append(row)
+
+    df = pd.DataFrame(rows)
+    df = df.sort_values("_sort_avg", ascending=False).reset_index(drop=True)
+    df = df.drop(columns=["_sort_avg"])
+    return df
+
+
+def build_chart_dataframe(data):
+    """Build a DataFrame with raw numeric Accuracy values for charting."""
+    rows = []
+    for entry in data["entries"]:
+        row = {"Method": entry["method"]}
+        for m in ALL_METRICS:
+            row[f"{m} (Acc)"] = entry["scores"][m]["accuracy"]
+        row["_sort_avg"] = entry["scores"]["Average"]["accuracy"]
+        rows.append(row)
+
+    df = pd.DataFrame(rows)
+    df = df.sort_values("_sort_avg", ascending=False).reset_index(drop=True)
+    df = df.drop(columns=["_sort_avg"])
+    return df
+
+
+def add_medals(df):
+    """Add medal emojis to the top-3 Method names."""
+    df = df.copy()
+    medals = ["\U0001f947", "\U0001f948", "\U0001f949"]
+    for i in range(min(3, len(df))):
+        df.loc[i, "Method"] = f"{medals[i]} {df.loc[i, 'Method']}"
+    return df
+
+
+# ---------------------------------------------------------------------------
+# Chart helpers
+# ---------------------------------------------------------------------------
+
+BAR_COLORS = ["#636EFA", "#EF553B", "#00CC96", "#AB63FA"]
+
+
+def make_bar_chart(chart_df, title=""):
+    """Create a grouped vertical bar chart showing Accuracy per metric."""
+    fig = go.Figure()
+
+    for i, m in enumerate(METRICS):
+        fig.add_trace(go.Bar(
+            x=chart_df["Method"],
+            y=chart_df[f"{m} (Acc)"],
+            name=m,
+            marker_color=BAR_COLORS[i % len(BAR_COLORS)],
+        ))
+
+    # Wrap long titles to 2 lines
+    if len(title) > 60:
+        mid = len(title) // 2
+        space_pos = title.find(" ", mid)
+        if space_pos == -1:
+            space_pos = title.rfind(" ", 0, mid)
+        if space_pos != -1:
+            title = title[:space_pos] + "<br>" + title[space_pos + 1:]
+
+    fig.update_layout(
+        barmode="group",
+        title=dict(text=title, x=0.5, font=dict(size=14)),
+        yaxis=dict(title="Accuracy", range=[0, 1]),
+        xaxis=dict(tickangle=-45),
+        height=500,
+        margin=dict(l=60, r=40, t=100, b=140),
+        legend=dict(
+            orientation="h", yanchor="bottom", y=1.02,
+            xanchor="center", x=0.5, font=dict(size=12),
+        ),
+        bargap=0.2,
+        bargroupgap=0.05,
+    )
+    return fig
+
+
+# ---------------------------------------------------------------------------
+# Update functions
+# ---------------------------------------------------------------------------
+
+def update_leaderboard(data, top_n):
+    """Return (display_df, bar_fig) for a given data source."""
+    df = build_dataframe(data)
+    chart_df = build_chart_dataframe(data)
+
+    df = df.head(int(top_n))
+    chart_df = chart_df.head(int(top_n))
+
+    display_df = add_medals(df)
+
+    title = data.get("title", "Score Breakdown")
+    bar = make_bar_chart(chart_df, title)
+
+    return display_df, bar
+
+
+def update_model_leaderboard(top_n):
+    return update_leaderboard(MODEL_DATA, top_n)
+
+
+def update_method_leaderboard(top_n):
+    return update_leaderboard(METHOD_DATA, top_n)
+
+
+# ---------------------------------------------------------------------------
+# App
+# ---------------------------------------------------------------------------
+
+CSS = """
+html, body {
+    overflow-y: auto !important;
+    width: 100% !important;
+}
+.gradio-container {
+    max-width: 1200px !important;
+    margin: auto !important;
+}
+.header-banner {
+    background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+    color: white;
+    padding: 24px 32px;
+    border-radius: 12px;
+    margin-bottom: 16px;
+    text-align: center;
+}
+.header-banner h1 { margin: 0 0 8px 0; font-size: 2em; }
+.header-banner p { margin: 0; font-size: 1.1em; opacity: 0.9; }
+.dark .header-banner {
+    background: linear-gradient(135deg, #434190 0%, #553c6b 100%);
+}
+.table-container {
+    border-radius: 8px;
+    box-shadow: 0 2px 10px rgba(0,0,0,0.08);
+}
+.tip-text {
+    font-size: 13px; color: #666; font-style: italic; margin-top: 4px;
+}
+.dark .tip-text { color: #aaa; }
+.metric-note {
+    background: #f0f4ff; padding: 10px 16px; border-radius: 8px;
+    border-left: 4px solid #667eea; margin-bottom: 12px; font-size: 14px;
+}
+.dark .metric-note {
+    background: #2d2d44; border-left-color: #764ba2;
+}
+"""
+
+
+def build_app():
+    with gr.Blocks(css=CSS, title="AMA-Bench Leaderboard") as demo:
+
+        # Header
+        gr.HTML("""
+        <div class="header-banner">
+            <h1>AMA-Bench Leaderboard</h1>
+            <p>Agent Memory Assessment Benchmark &mdash; Evaluating LLMs and Memory Methods on Cognitive Tasks</p>
+        </div>
+        """)
+
+        with gr.Tabs():
+            # ============================================================
+            # Tab 1: Model Leaderboard
+            # ============================================================
+            with gr.Tab("Model Leaderboard"):
+                gr.Markdown("""
+                <div class="metric-note">
+                Comparing <strong>LLM models</strong> across 4 cognitive tasks: Recall, Causal Inference, State Updating, and State Abstraction.
+                Results are reported as <strong>Accuracy (F1)</strong>. Sorted by Average Accuracy.
+                </div>
+                """)
+
+                with gr.Row():
+                    model_top_n = gr.Slider(
+                        minimum=1,
+                        maximum=len(MODEL_DATA["entries"]),
+                        step=1,
+                        value=len(MODEL_DATA["entries"]),
+                        label="Number of models to display",
+                    )
+
+                # Chart
+                with gr.Row():
+                    gr.Markdown("### Data Visualization")
+                model_bar = gr.Plot(label="Score Breakdown")
+                gr.Markdown("*Click a legend entry to isolate that metric. Double-click to add more for comparison.*", elem_classes="tip-text")
+
+                # Table
+                with gr.Row():
+                    gr.Markdown("### Detailed Results")
+                init_model_df, _ = update_model_leaderboard(len(MODEL_DATA["entries"]))
+                model_table = gr.DataFrame(
+                    value=init_model_df,
+                    elem_classes="table-container",
+                    show_row_numbers=True,
+                    show_fullscreen_button=True,
+                    show_search="search",
+                    interactive=False,
+                )
+
+                # Wire events
+                model_top_n.change(
+                    update_model_leaderboard,
+                    inputs=[model_top_n],
+                    outputs=[model_table, model_bar],
+                )
+
+                demo.load(
+                    update_model_leaderboard,
+                    inputs=[model_top_n],
+                    outputs=[model_table, model_bar],
+                )
+
+            # ============================================================
+            # Tab 2: Method Leaderboard
+            # ============================================================
+            with gr.Tab("Method Leaderboard"):
+                gr.Markdown("""
+                <div class="metric-note">
+                Comparing <strong>RAG &amp; Agent Memory methods</strong> (base model: Qwen-32B) across 4 cognitive tasks.
+                Results are reported as <strong>Accuracy (F1)</strong>. Sorted by Average Accuracy.
+                </div>
+                """)
+
+                with gr.Row():
+                    method_top_n = gr.Slider(
+                        minimum=1,
+                        maximum=len(METHOD_DATA["entries"]),
+                        step=1,
+                        value=len(METHOD_DATA["entries"]),
+                        label="Number of methods to display",
+                    )
+
+                # Chart
+                with gr.Row():
+                    gr.Markdown("### Data Visualization")
+                method_bar = gr.Plot(label="Score Breakdown")
+                gr.Markdown("*Click a legend entry to isolate that metric. Double-click to add more for comparison.*", elem_classes="tip-text")
+
+                # Table
+                with gr.Row():
+                    gr.Markdown("### Detailed Results")
+                init_method_df, _ = update_method_leaderboard(len(METHOD_DATA["entries"]))
+                method_table = gr.DataFrame(
+                    value=init_method_df,
+                    elem_classes="table-container",
+                    show_row_numbers=True,
+                    show_fullscreen_button=True,
+                    show_search="search",
+                    interactive=False,
+                )
+
+                # Wire events
+                method_top_n.change(
+                    update_method_leaderboard,
+                    inputs=[method_top_n],
+                    outputs=[method_table, method_bar],
+                )
+
+                demo.load(
+                    update_method_leaderboard,
+                    inputs=[method_top_n],
+                    outputs=[method_table, method_bar],
+                )
+
+            # ============================================================
+            # Tab 3: About
+            # ============================================================
+            with gr.Tab("About"):
+                gr.Markdown("""
+## AMA-Bench: Agent Memory Assessment Benchmark
+
+AMA-Bench evaluates memory capabilities of LLMs and memory-augmented agents across four cognitive dimensions:
+**Recall** (retrieving stored info), **Causal Inference** (cause-and-effect reasoning), **State Updating** (tracking evolving states), and **State Abstraction** (forming higher-level representations).
+
+**Benchmarks** &mdash; We evaluate on two complementary subsets:
+(1) **Real-world Subset:** 2,496 QA pairs.
+(2) **Synthetic Subset:** 1,200 QA pairs stratified across five trajectory lengths (8K, 16K, 32K, 64K, and 128K tokens), with 240 samples per interval.
+
+**Leaderboard Tabs** &mdash; *Model Leaderboard* compares LLM models directly; *Method Leaderboard* compares RAG and Agent Memory methods using Qwen-32B as the base model.
+
+**Metrics** &mdash; Results are reported as **Accuracy (F1)**. 
+---
+*For questions or submissions, please open a discussion in the Community tab.*
+                """)
+
+    return demo
+
+
+if __name__ == "__main__":
+    demo_app = build_app()
+    demo_app.launch(debug=True, show_error=True)

data/method_data.json

@@ -0,0 +1,160 @@
+{
+  "title": "Performance comparison of Agent Memory and RAG methods (base model: Qwen-32B) on real-world subset",
+  "metrics": ["Recall", "Causal Inference", "State Updating", "State Abstraction", "Average"],
+  "entries": [
+    {
+      "method": "BM25",
+      "category": "RAG",
+      "scores": {
+        "Recall": {"accuracy": 0.3301, "f1": 0.1465},
+        "Causal Inference": {"accuracy": 0.4264, "f1": 0.1549},
+        "State Updating": {"accuracy": 0.3450, "f1": 0.1325},
+        "State Abstraction": {"accuracy": 0.2498, "f1": 0.1623},
+        "Average": {"accuracy": 0.3436, "f1": 0.1475}
+      }
+    },
+    {
+      "method": "Qwen3-Emb-4B",
+      "category": "RAG",
+      "scores": {
+        "Recall": {"accuracy": 0.4843, "f1": 0.1590},
+        "Causal Inference": {"accuracy": 0.4974, "f1": 0.1549},
+        "State Updating": {"accuracy": 0.3520, "f1": 0.1353},
+        "State Abstraction": {"accuracy": 0.3011, "f1": 0.1610},
+        "Average": {"accuracy": 0.4227, "f1": 0.1522}
+      }
+    },
+    {
+      "method": "GraphRAG",
+      "category": "RAG",
+      "scores": {
+        "Recall": {"accuracy": 0.3077, "f1": 0.2769},
+        "Causal Inference": {"accuracy": 0.3905, "f1": 0.2634},
+        "State Updating": {"accuracy": 0.3140, "f1": 0.2551},
+        "State Abstraction": {"accuracy": 0.2879, "f1": 0.2588},
+        "Average": {"accuracy": 0.3258, "f1": 0.2650}
+      }
+    },
+    {
+      "method": "HippoRAG2",
+      "category": "RAG",
+      "scores": {
+        "Recall": {"accuracy": 0.4579, "f1": 0.2356},
+        "Causal Inference": {"accuracy": 0.5080, "f1": 0.1966},
+        "State Updating": {"accuracy": 0.4403, "f1": 0.1892},
+        "State Abstraction": {"accuracy": 0.3538, "f1": 0.1785},
+        "Average": {"accuracy": 0.4480, "f1": 0.2048}
+      }
+    },
+    {
+      "method": "MemAgent",
+      "category": "Agent Memory",
+      "scores": {
+        "Recall": {"accuracy": 0.2550, "f1": 0.1489},
+        "Causal Inference": {"accuracy": 0.3380, "f1": 0.1606},
+        "State Updating": {"accuracy": 0.2849, "f1": 0.1432},
+        "State Abstraction": {"accuracy": 0.2202, "f1": 0.1655},
+        "Average": {"accuracy": 0.2768, "f1": 0.1530}
+      }
+    },
+    {
+      "method": "Mem1",
+      "category": "Agent Memory",
+      "scores": {
+        "Recall": {"accuracy": 0.1180, "f1": 0.1857},
+        "Causal Inference": {"accuracy": 0.1427, "f1": 0.1732},
+        "State Updating": {"accuracy": 0.1205, "f1": 0.1659},
+        "State Abstraction": {"accuracy": 0.1080, "f1": 0.2042},
+        "Average": {"accuracy": 0.1229, "f1": 0.1807}
+      }
+    },
+    {
+      "method": "Amem",
+      "category": "Agent Memory",
+      "scores": {
+        "Recall": {"accuracy": 0.3084, "f1": 0.2707},
+        "Causal Inference": {"accuracy": 0.3653, "f1": 0.2731},
+        "State Updating": {"accuracy": 0.3088, "f1": 0.2480},
+        "State Abstraction": {"accuracy": 0.2873, "f1": 0.2953},
+        "Average": {"accuracy": 0.3186, "f1": 0.2695}
+      }
+    },
+    {
+      "method": "Mem0",
+      "category": "Agent Memory",
+      "scores": {
+        "Recall": {"accuracy": 0.2011, "f1": 0.2413},
+        "Causal Inference": {"accuracy": 0.2645, "f1": 0.2443},
+        "State Updating": {"accuracy": 0.2101, "f1": 0.2225},
+        "State Abstraction": {"accuracy": 0.1516, "f1": 0.2241},
+        "Average": {"accuracy": 0.2104, "f1": 0.2343}
+      }
+    },
+    {
+      "method": "MemoRAG",
+      "category": "Agent Memory",
+      "scores": {
+        "Recall": {"accuracy": 0.4708, "f1": 0.1789},
+        "Causal Inference": {"accuracy": 0.5497, "f1": 0.1811},
+        "State Updating": {"accuracy": 0.4257, "f1": 0.1713},
+        "State Abstraction": {"accuracy": 0.3659, "f1": 0.2073},
+        "Average": {"accuracy": 0.4606, "f1": 0.1822}
+      }
+    },
+    {
+      "method": "MemGPT",
+      "category": "Agent Memory",
+      "scores": {
+        "Recall": {"accuracy": 0.3289, "f1": 0.1318},
+        "Causal Inference": {"accuracy": 0.4404, "f1": 0.1475},
+        "State Updating": {"accuracy": 0.2809, "f1": 0.1259},
+        "State Abstraction": {"accuracy": 0.2526, "f1": 0.1431},
+        "Average": {"accuracy": 0.3304, "f1": 0.1359}
+      }
+    },
+    {
+      "method": "Mem-alpha",
+      "category": "Agent Memory",
+      "scores": {
+        "Recall": {"accuracy": 0.2876, "f1": 0.2325},
+        "Causal Inference": {"accuracy": 0.4172, "f1": 0.1993},
+        "State Updating": {"accuracy": 0.3064, "f1": 0.2000},
+        "State Abstraction": {"accuracy": 0.2171, "f1": 0.2135},
+        "Average": {"accuracy": 0.3117, "f1": 0.2130}
+      }
+    },
+    {
+      "method": "MemoryBank",
+      "category": "Agent Memory",
+      "scores": {
+        "Recall": {"accuracy": 0.3231, "f1": 0.3128},
+        "Causal Inference": {"accuracy": 0.4100, "f1": 0.2861},
+        "State Updating": {"accuracy": 0.3006, "f1": 0.2678},
+        "State Abstraction": {"accuracy": 0.3332, "f1": 0.3011},
+        "Average": {"accuracy": 0.3397, "f1": 0.2928}
+      }
+    },
+    {
+      "method": "Simple Mem",
+      "category": "Agent Memory",
+      "scores": {
+        "Recall": {"accuracy": 0.2012, "f1": 0.2039},
+        "Causal Inference": {"accuracy": 0.1884, "f1": 0.1612},
+        "State Updating": {"accuracy": 0.1764, "f1": 0.1594},
+        "State Abstraction": {"accuracy": 0.1373, "f1": 0.1689},
+        "Average": {"accuracy": 0.1811, "f1": 0.1764}
+      }
+    },
+    {
+      "method": "AMA Agent",
+      "category": "Agent Memory",
+      "scores": {
+        "Recall": {"accuracy": 0.6238, "f1": 0.3280},
+        "Causal Inference": {"accuracy": 0.6145, "f1": 0.3103},
+        "State Updating": {"accuracy": 0.5305, "f1": 0.2625},
+        "State Abstraction": {"accuracy": 0.4719, "f1": 0.2825},
+        "Average": {"accuracy": 0.5722, "f1": 0.2992}
+      }
+    }
+  ]
+}

data/model_data.json

@@ -0,0 +1,94 @@
+{
+  "title": "Performance of different models on real-world subset",
+  "metrics": ["Recall", "Causal Inference", "State Updating", "State Abstraction", "Average"],
+  "entries": [
+    {
+      "method": "Claude Haiku 3.5",
+      "category": null,
+      "scores": {
+        "Recall": {"accuracy": 0.4943, "f1": 0.3510},
+        "Causal Inference": {"accuracy": 0.4507, "f1": 0.2792},
+        "State Updating": {"accuracy": 0.4287, "f1": 0.3015},
+        "State Abstraction": {"accuracy": 0.3090, "f1": 0.2648},
+        "Average": {"accuracy": 0.4361, "f1": 0.3067}
+      }
+    },
+    {
+      "method": "GPT-5-mini",
+      "category": null,
+      "scores": {
+        "Recall": {"accuracy": 0.6951, "f1": 0.4010},
+        "Causal Inference": {"accuracy": 0.7157, "f1": 0.3027},
+        "State Updating": {"accuracy": 0.6575, "f1": 0.3288},
+        "State Abstraction": {"accuracy": 0.6235, "f1": 0.3262},
+        "Average": {"accuracy": 0.6784, "f1": 0.3464}
+      }
+    },
+    {
+      "method": "GPT 5.2",
+      "category": null,
+      "scores": {
+        "Recall": {"accuracy": 0.7741, "f1": 0.4758},
+        "Causal Inference": {"accuracy": 0.8047, "f1": 0.3512},
+        "State Updating": {"accuracy": 0.6563, "f1": 0.3686},
+        "State Abstraction": {"accuracy": 0.6037, "f1": 0.3582},
+        "Average": {"accuracy": 0.7226, "f1": 0.3988}
+      }
+    },
+    {
+      "method": "Gemini 2.5 Flash",
+      "category": null,
+      "scores": {
+        "Recall": {"accuracy": 0.5834, "f1": 0.3682},
+        "Causal Inference": {"accuracy": 0.5087, "f1": 0.2628},
+        "State Updating": {"accuracy": 0.5000, "f1": 0.2395},
+        "State Abstraction": {"accuracy": 0.4196, "f1": 0.2361},
+        "Average": {"accuracy": 0.5168, "f1": 0.2878}
+      }
+    },
+    {
+      "method": "Qwen2.5-14B-1M",
+      "category": null,
+      "scores": {
+        "Recall": {"accuracy": 0.5570, "f1": 0.4157},
+        "Causal Inference": {"accuracy": 0.4111, "f1": 0.3209},
+        "State Updating": {"accuracy": 0.4728, "f1": 0.3348},
+        "State Abstraction": {"accuracy": 0.3368, "f1": 0.3560},
+        "Average": {"accuracy": 0.4638, "f1": 0.3622}
+      }
+    },
+    {
+      "method": "Qwen3-32B",
+      "category": null,
+      "scores": {
+        "Recall": {"accuracy": 0.6149, "f1": 0.4074},
+        "Causal Inference": {"accuracy": 0.5178, "f1": 0.3289},
+        "State Updating": {"accuracy": 0.4903, "f1": 0.3334},
+        "State Abstraction": {"accuracy": 0.3657, "f1": 0.3172},
+        "Average": {"accuracy": 0.5181, "f1": 0.3545}
+      }
+    },
+    {
+      "method": "Qwen3-14B",
+      "category": null,
+      "scores": {
+        "Recall": {"accuracy": 0.5675, "f1": 0.3636},
+        "Causal Inference": {"accuracy": 0.4430, "f1": 0.2931},
+        "State Updating": {"accuracy": 0.4502, "f1": 0.3204},
+        "State Abstraction": {"accuracy": 0.3176, "f1": 0.2716},
+        "Average": {"accuracy": 0.4659, "f1": 0.3203}
+      }
+    },
+    {
+      "method": "Qwen3-8B",
+      "category": null,
+      "scores": {
+        "Recall": {"accuracy": 0.5024, "f1": 0.3801},
+        "Causal Inference": {"accuracy": 0.3776, "f1": 0.2830},
+        "State Updating": {"accuracy": 0.3987, "f1": 0.3177},
+        "State Abstraction": {"accuracy": 0.2923, "f1": 0.2792},
+        "Average": {"accuracy": 0.4109, "f1": 0.3240}
+      }
+    }
+  ]
+}

requirements.txt

@@ -0,0 +1,4 @@
+gradio==5.23.3
+pandas>=2.0.0
+plotly>=5.15.0
+numpy>=1.24.0