app.py

import gradio as gr
import pandas as pd

# Paper Retrieval Benchmark Data (from SemanticBench)
retrieval_data = [
    {"Model": "Qwen3-Coder-30B-Q3_K_M", "Type": "Agent", "Hit Rate": 0.80, "MRR": 0.627, "R@1": 0.58, "R@5": 0.66, "R@10": 0.74, "R@20": 0.78, "R@50": 0.80, "Time (s)": 22.2, "Steps": 1.42},
    {"Model": "qwen3-coder:30b", "Type": "Agent", "Hit Rate": 0.80, "MRR": 0.518, "R@1": 0.46, "R@5": 0.52, "R@10": 0.72, "R@20": 0.76, "R@50": 0.80, "Time (s)": 21.1, "Steps": 1.34},
    {"Model": "BM25", "Type": "Baseline", "Hit Rate": 0.78, "MRR": 0.541, "R@1": 0.48, "R@5": 0.60, "R@10": 0.66, "R@20": 0.78, "R@50": 0.78, "Time (s)": None, "Steps": None},
    {"Model": "microcoder-deepseekr1-14.8b", "Type": "Agent", "Hit Rate": 0.73, "MRR": 0.453, "R@1": 0.38, "R@5": 0.46, "R@10": 0.65, "R@20": 0.69, "R@50": 0.73, "Time (s)": 107.4, "Steps": 4.15},
    {"Model": "deepseek-coder-v3:16b", "Type": "Agent", "Hit Rate": 0.66, "MRR": 0.396, "R@1": 0.32, "R@5": 0.46, "R@10": 0.52, "R@20": 0.60, "R@50": 0.66, "Time (s)": 47.9, "Steps": 1.54},
    {"Model": "qwen2.5-coder:3b", "Type": "Agent", "Hit Rate": 0.60, "MRR": 0.366, "R@1": 0.28, "R@5": 0.45, "R@10": 0.53, "R@20": 0.55, "R@50": 0.57, "Time (s)": 210.4, "Steps": 1.51},
    {"Model": "qwen2.5-coder:14b", "Type": "Agent", "Hit Rate": 0.56, "MRR": 0.461, "R@1": 0.41, "R@5": 0.51, "R@10": 0.51, "R@20": 0.56, "R@50": 0.56, "Time (s)": 73.4, "Steps": 1.05},
    {"Model": "Semantic (MiniLM-L6)", "Type": "Baseline", "Hit Rate": 0.54, "MRR": 0.279, "R@1": 0.22, "R@5": 0.32, "R@10": 0.38, "R@20": 0.52, "R@50": 0.54, "Time (s)": None, "Steps": None},
    {"Model": "qwen2.5-coder:7b", "Type": "Agent", "Hit Rate": 0.54, "MRR": 0.311, "R@1": 0.26, "R@5": 0.36, "R@10": 0.40, "R@20": 0.52, "R@50": 0.54, "Time (s)": 59.3, "Steps": 0.84},
    {"Model": "deepseek-coder:33b", "Type": "Agent", "Hit Rate": 0.12, "MRR": 0.087, "R@1": 0.08, "R@5": 0.08, "R@10": 0.12, "R@20": 0.12, "R@50": 0.12, "Time (s)": 180.4, "Steps": 0.14},
    {"Model": "granite-code:34b", "Type": "Agent", "Hit Rate": 0.02, "MRR": 0.010, "R@1": 0.00, "R@5": 0.02, "R@10": 0.02, "R@20": 0.02, "R@50": 0.02, "Time (s)": 111.3, "Steps": 0.04},
]

# RAbench Results (500 queries)
rabench_data = [
    {"Model": "Qwen3-Coder-30B-Q3_K_M", "Type": "Agent", "Hit Rate": 0.98, "MRR": 0.882, "R@1": 0.83, "R@5": 0.93, "R@10": 0.95, "R@20": 0.96, "R@50": 0.97, "Time (s)": 21.53, "Steps": 1.36},
]

# Ablation Study Data
ablation_data = [
    {"Configuration": "Default (Full Agent)", "Queries": 500, "Hit Rate": 0.9818, "MRR": 0.8824, "R@1": 0.8381, "R@5": 0.9312, "Time (s)": 21.54},
    {"Configuration": "With Filters & Offline", "Queries": 50, "Hit Rate": 0.9600, "MRR": 0.8485, "R@1": 0.7800, "R@5": 0.9000, "Time (s)": 22.76},
    {"Configuration": "Offline Only", "Queries": 50, "Hit Rate": 0.9200, "MRR": 0.6476, "R@1": 0.5600, "R@5": 0.7400, "Time (s)": 41.45},
    {"Configuration": "No Mentions", "Queries": 50, "Hit Rate": 0.6400, "MRR": 0.4316, "R@1": 0.3600, "R@5": 0.5200, "Time (s)": 38.35},
    {"Configuration": "Online/Offline Mix", "Queries": 50, "Hit Rate": 0.6200, "MRR": 0.4595, "R@1": 0.4200, "R@5": 0.5000, "Time (s)": 38.50},
]

# Retrieval Baseline Ablations
baseline_ablation_data = [
    {"Configuration": "BM25 Full", "Baseline": "bm25", "Structure": "full", "Hit Rate": 0.96, "MRR": 0.8629, "R@1": 0.80, "R@5": 0.92, "Time (s)": 33.75},
    {"Configuration": "BM25 + Reranker", "Baseline": "bm25+reranker", "Structure": "full", "Hit Rate": 0.96, "MRR": 0.8692, "R@1": 0.80, "R@5": 0.94, "Time (s)": 935.07},
    {"Configuration": "Hybrid Full", "Baseline": "hybrid", "Structure": "full", "Hit Rate": 0.96, "MRR": 0.8620, "R@1": 0.80, "R@5": 0.92, "Time (s)": 31.65},
    {"Configuration": "Semantic Full", "Baseline": "semantic", "Structure": "full", "Hit Rate": 0.94, "MRR": 0.7097, "R@1": 0.62, "R@5": 0.88, "Time (s)": 31.28},
    {"Configuration": "BM25 No Intent", "Baseline": "bm25", "Structure": "no_intent", "Hit Rate": 0.96, "MRR": 0.8554, "R@1": 0.80, "R@5": 0.92, "Time (s)": 31.47},
    {"Configuration": "BM25 Minimal", "Baseline": "bm25", "Structure": "minimal", "Hit Rate": 0.96, "MRR": 0.8420, "R@1": 0.78, "R@5": 0.92, "Time (s)": 33.34},
]

# Dataset Statistics
dataset_stats = [
    {"Conference": "ICLR", "Count": 12},
    {"Conference": "NeurIPS", "Count": 39},
    {"Conference": "ICML", "Count": 13},
    {"Conference": "CVPR", "Count": 13},
    {"Conference": "IROS", "Count": 25},
    {"Conference": "ICRA", "Count": 25},
    {"Conference": "AAAI", "Count": 5},
    {"Conference": "ACL", "Count": 5},
    {"Conference": "ICCV", "Count": 7},
    {"Conference": "EMNLP", "Count": 4},
    {"Conference": "Other", "Count": 144},
]

def create_retrieval_df():
    df = pd.DataFrame(retrieval_data)
    df = df.sort_values("MRR", ascending=False)
    return df

def create_ablation_df():
    return pd.DataFrame(ablation_data)

def create_baseline_ablation_df():
    return pd.DataFrame(baseline_ablation_data)

def create_dataset_df():
    return pd.DataFrame(dataset_stats)

def filter_by_type(model_type):
    df = pd.DataFrame(retrieval_data)
    if model_type != "All":
        df = df[df["Type"] == model_type]
    return df.sort_values("MRR", ascending=False)

with gr.Blocks(title="PC-Bench: Paper Discovery Benchmark") as demo:
    gr.HTML("""
        <div style="text-align: center; margin-bottom: 20px;">
            <h1>PC-Bench: Paper Discovery Benchmark</h1>
            <p style="color: #666;">Evaluating AI agents for academic paper retrieval and analysis</p>
            <p>
                <a href="https://github.com/MAXNORM8650/papercircle" target="_blank">
                    <img src="https://img.shields.io/badge/GitHub-Repository-blue?logo=github" alt="GitHub"/>
                </a>
                <img src="https://img.shields.io/badge/Papers-292-green" alt="Papers"/>
                <img src="https://img.shields.io/badge/Queries-500+-orange" alt="Queries"/>
            </p>
        </div>
    """)

    with gr.Tabs():
        with gr.TabItem("Model Leaderboard"):
            gr.Markdown("### Multi-Agent Paper Retrieval (SemanticBench - 50 queries)")
            gr.Markdown("Models ranked by Mean Reciprocal Rank (MRR). Higher is better.")

            model_filter = gr.Dropdown(
                choices=["All", "Agent", "Baseline"],
                value="All",
                label="Filter by Type"
            )

            leaderboard_table = gr.Dataframe(
                value=create_retrieval_df(),
                headers=["Model", "Type", "Hit Rate", "MRR", "R@1", "R@5", "R@10", "R@20", "R@50", "Time (s)", "Steps"],
                interactive=False,
            )

            model_filter.change(
                fn=filter_by_type,
                inputs=[model_filter],
                outputs=[leaderboard_table]
            )

            gr.Markdown("""
            **Key Findings:**
            - **Qwen3-Coder-30B** achieves best MRR (0.627) with 80% hit rate
            - **BM25 baseline** remains competitive (78% hit rate, 0.541 MRR)
            - Larger models (30B+) consistently outperform smaller variants
            """)

        with gr.TabItem("RAbench Results"):
            gr.Markdown("### Extended Benchmark (RAbench - 500 queries)")
            gr.Markdown("LLM-perturbed natural language queries")

            gr.Dataframe(
                value=pd.DataFrame(rabench_data),
                headers=["Model", "Type", "Hit Rate", "MRR", "R@1", "R@5", "R@10", "R@20", "R@50", "Time (s)", "Steps"],
                interactive=False,
            )

            gr.Markdown("""
            **Observation:** RAbench shows higher performance than SemanticBench,
            suggesting LLM-perturbed queries are easier for multi-agent retrieval.
            """)

        with gr.TabItem("Configuration Ablations"):
            gr.Markdown("### Query Configuration Impact")
            gr.Dataframe(
                value=create_ablation_df(),
                interactive=False,
            )

            gr.Markdown("### Retrieval Baseline Comparison")
            gr.Dataframe(
                value=create_baseline_ablation_df(),
                interactive=False,
            )

            gr.Markdown("""
            **Key Insights:**
            - BM25 + Reranker achieves highest MRR (0.869) but is 28x slower
            - No Intent configuration is fastest while maintaining 96% hit rate
            - Semantic-only retrieval shows significant R@1 drop (0.62 vs 0.80)
            """)

        with gr.TabItem("Dataset"):
            gr.Markdown("### Database Corpus Statistics")
            gr.Markdown("Papers sourced from OpenReview across major ML/CS conferences")

            gr.Dataframe(
                value=create_dataset_df(),
                interactive=False,
            )

            total = sum(d["Count"] for d in dataset_stats)
            gr.Markdown(f"**Total Papers:** {total}")

    gr.Markdown("""
    ---
    ### About

    **Paper Circle** is a multi-agent research pipeline for intelligent paper discovery and analysis.

    **Pipeline:** Query → Intent Agent → Search Agent → Sort Agent → Analysis Agent → Export

    **Metrics:**
    - **MRR** (Mean Reciprocal Rank): Ranking quality
    - **R@K** (Recall at K): Found in top K results
    - **Hit Rate**: Successful retrieval percentage

    Built with [smolagents](https://github.com/huggingface/smolagents) and [LiteLLM](https://github.com/BerriAI/litellm)
    """)

if __name__ == "__main__":
    demo.launch()