app.py

import gradio as gr
from gradio_leaderboard import Leaderboard
import pandas as pd
from huggingface_hub import snapshot_download, create_repo
from huggingface_hub.utils import RepositoryNotFoundError
import os

from src.about import (
    INTRODUCTION_TEXT,
    LLM_BENCHMARKS_TEXT,
    TITLE,
)
from src.display.css_html_js import custom_css
from src.display.utils import (
    BENCHMARK_COLS,
    COLS,
    AutoEvalColumn,
    fields,
)
from src.envs import API, EVAL_RESULTS_PATH, RESULTS_REPO, TOKEN, OWNER
from src.populate import get_leaderboard_df
from src.evaluation.dynamic_eval import run_dynamic_perplexity_eval

def init_leaderboard(dataframe):
    if dataframe is None:
        raise ValueError("Leaderboard DataFrame is None.")
    
    print("\n=== Initializing Leaderboard ===", flush=True)
    print(f"DataFrame shape: {dataframe.shape}", flush=True)
    print(f"DataFrame columns: {dataframe.columns.tolist()}", flush=True)
    
    return Leaderboard(
        value=dataframe,
        select_columns=[c.name for c in fields(AutoEvalColumn) if not c.hidden],
        search_columns=[AutoEvalColumn.model.name],
        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
        filter_columns=[
            AutoEvalColumn.model_type.name,
            AutoEvalColumn.precision.name,
        ],
    )

def refresh_leaderboard():
    import sys
    import traceback
    import pandas as pd
    
    try:
        sys.stderr.write("=== REFRESH LEADERBOARD DEBUG ===\n")
        sys.stderr.write("Refreshing leaderboard data...\n")
        sys.stderr.flush()
        
        # Get fresh leaderboard data
        df = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
        
        sys.stderr.write(f"get_leaderboard_df returned: {type(df)}\n")
        if df is not None:
            sys.stderr.write(f"DataFrame shape: {df.shape}\n")
            sys.stderr.write(f"DataFrame columns: {df.columns.tolist()}\n")
            sys.stderr.write(f"DataFrame empty: {df.empty}\n")
        else:
            sys.stderr.write("DataFrame is None!\n")
        sys.stderr.flush()
        
        # Check if DataFrame is valid for leaderboard
        if df is None:
            sys.stderr.write("DataFrame is None, creating fallback DataFrame\n")
            sys.stderr.flush()
            # Create a fallback DataFrame
            df = create_fallback_dataframe()
            
        elif df.empty:
            sys.stderr.write("DataFrame is empty, creating fallback DataFrame\n")
            sys.stderr.flush()
            # Create a fallback DataFrame for empty case
            df = create_fallback_dataframe()
        
        elif not all(col in df.columns for col in COLS):
            sys.stderr.write(f"DataFrame missing required columns. Has: {df.columns.tolist()}, Needs: {COLS}\n")
            sys.stderr.flush()
            # Create a fallback DataFrame for missing columns
            df = create_fallback_dataframe()
        
        sys.stderr.write(f"Final DataFrame for leaderboard - Shape: {df.shape}, Columns: {df.columns.tolist()}\n")
        sys.stderr.flush()
        
        # Ensure DataFrame has the exact columns expected
        for col in COLS:
            if col not in df.columns:
                sys.stderr.write(f"Adding missing column: {col}\n")
                if col in BENCHMARK_COLS or col == AutoEvalColumn.average.name:
                    df[col] = 0.0
                elif col == AutoEvalColumn.model.name:
                    df[col] = "Unknown Model"
                elif col == AutoEvalColumn.model_type_symbol.name:
                    df[col] = "?"
                else:
                    df[col] = ""
                sys.stderr.flush()
        
        # Reorder columns to match expected order
        df = df[COLS]
        
        sys.stderr.write("Creating leaderboard component...\n")
        sys.stderr.flush()
        
        new_leaderboard = init_leaderboard(df)
        sys.stderr.write("Leaderboard component created successfully\n")
        sys.stderr.flush()
        
        return new_leaderboard
        
    except Exception as e:
        error_msg = str(e)
        traceback_str = traceback.format_exc()
        sys.stderr.write(f"CRITICAL ERROR in refresh_leaderboard: {error_msg}\n")
        sys.stderr.write(f"Traceback: {traceback_str}\n")
        sys.stderr.flush()
        
        # Create emergency fallback leaderboard
        try:
            sys.stderr.write("Creating emergency fallback leaderboard...\n")
            sys.stderr.flush()
            fallback_df = create_fallback_dataframe()
            return init_leaderboard(fallback_df)
        except Exception as fallback_error:
            sys.stderr.write(f"Even fallback failed: {fallback_error}\n")
            sys.stderr.flush()
            raise Exception(f"Complete leaderboard failure: {error_msg}")

def create_fallback_dataframe():
    """Create a minimal valid DataFrame that won't crash the leaderboard"""
    import pandas as pd
    import sys
    
    sys.stderr.write("Creating fallback DataFrame...\n")
    sys.stderr.flush()
    
    # Create minimal valid data
    fallback_data = {col: [] for col in COLS}
    
    # Add one dummy row to prevent leaderboard component from crashing
    dummy_row = {}
    for col in COLS:
        if col in BENCHMARK_COLS or col == AutoEvalColumn.average.name:
            dummy_row[col] = 0.0
        elif col == AutoEvalColumn.model.name:
            dummy_row[col] = "No models evaluated yet"
        elif col == AutoEvalColumn.model_type_symbol.name:
            dummy_row[col] = "?"
        elif col == AutoEvalColumn.precision.name:
            dummy_row[col] = "float16"
        elif col == AutoEvalColumn.model_type.name:
            dummy_row[col] = "pretrained"
        elif col == AutoEvalColumn.weight_type.name:
            dummy_row[col] = "Original"
        elif col == AutoEvalColumn.architecture.name:
            dummy_row[col] = "Unknown"
        elif col == AutoEvalColumn.still_on_hub.name:
            dummy_row[col] = True
        elif col == AutoEvalColumn.license.name:
            dummy_row[col] = "Unknown"
        elif col == AutoEvalColumn.params.name:
            dummy_row[col] = 0.0
        elif col == AutoEvalColumn.likes.name:
            dummy_row[col] = 0.0
        elif col == AutoEvalColumn.revision.name:
            dummy_row[col] = ""
        else:
            dummy_row[col] = ""
    
    df = pd.DataFrame([dummy_row])
    sys.stderr.write(f"Fallback DataFrame created with shape: {df.shape}\n")
    sys.stderr.write(f"Fallback DataFrame columns: {df.columns.tolist()}\n")
    sys.stderr.flush()
    
    return df

def run_perplexity_test(model_name, revision, precision):
    """Run perplexity evaluation on demand."""
    import sys
    import traceback
    
    if not model_name:
        return "Please enter a model name.", None
    
    try:
        # Use stderr for more reliable logging in HF Spaces
        sys.stderr.write(f"\n=== RUNNING PERPLEXITY TEST ===\n")
        sys.stderr.write(f"Model: {model_name}\n")
        sys.stderr.write(f"Revision: {revision}\n")
        sys.stderr.write(f"Precision: {precision}\n")
        sys.stderr.flush()
        
        success, result = run_dynamic_perplexity_eval(model_name, revision, precision)
        sys.stderr.write(f"Evaluation result - Success: {success}, Result: {result}\n")
        sys.stderr.flush()
        
        if success:
            try:
                # Try to refresh leaderboard
                sys.stderr.write("Attempting to refresh leaderboard...\n")
                sys.stderr.flush()
                
                new_leaderboard = refresh_leaderboard()
                
                if new_leaderboard is not None:
                    sys.stderr.write("Leaderboard refresh successful\n")
                    sys.stderr.flush()
                    return f"✅ Perplexity evaluation completed!\nPerplexity: {result:.4f}\n\nResults saved and leaderboard updated.", new_leaderboard
                else:
                    sys.stderr.write("Leaderboard refresh returned None\n")
                    sys.stderr.flush()
                    return f"✅ Perplexity evaluation completed!\nPerplexity: {result:.4f}\n\n⚠️ Results saved but leaderboard update returned None.\n\nPlease refresh the page to see updated results.", None
                    
            except Exception as refresh_error:
                # If leaderboard refresh fails, still show success but don't update leaderboard
                error_msg = str(refresh_error)
                traceback_str = traceback.format_exc()
                sys.stderr.write(f"Leaderboard refresh failed: {error_msg}\n")
                sys.stderr.write(f"Traceback: {traceback_str}\n")
                sys.stderr.flush()
                
                # Check if it's the specific "must have a value set" error
                if "must have a value set" in error_msg.lower():
                    return f"✅ Perplexity evaluation completed!\nPerplexity: {result:.4f}\n\n⚠️ Results saved but leaderboard component failed to update due to data structure issue.\n\n**Please refresh the page** to see your results in the main leaderboard.", None
                else:
                    return f"✅ Perplexity evaluation completed!\nPerplexity: {result:.4f}\n\n⚠️ Results saved but leaderboard refresh failed: {error_msg}\n\nPlease refresh the page to see updated results.", None
        else:
            return f"❌ Evaluation failed: {result}", None
            
    except Exception as e:
        error_msg = str(e)
        traceback_str = traceback.format_exc()
        sys.stderr.write(f"Critical error in run_perplexity_test: {error_msg}\n")
        sys.stderr.write(f"Traceback: {traceback_str}\n")
        sys.stderr.flush()
        return f"❌ Critical error: {error_msg}", None

# Initialize results repository and directory
try:
    # Try to download existing repository
    try:
        snapshot_download(
            repo_id=RESULTS_REPO,
            local_dir=EVAL_RESULTS_PATH,
            repo_type="dataset",
            tqdm_class=None,
            etag_timeout=30,
            token=TOKEN
        )
    except RepositoryNotFoundError:
        # Create the repository if it doesn't exist
        print(f"Creating new results repository: {RESULTS_REPO}")
        create_repo(
            repo_id=RESULTS_REPO,
            repo_type="dataset",
            private=False,
            token=TOKEN
        )
        # Create local directory
        os.makedirs(EVAL_RESULTS_PATH, exist_ok=True)
except Exception as e:
    print(f"Error initializing results: {e}")
    # Ensure local directory exists even if repo operations fail
    os.makedirs(EVAL_RESULTS_PATH, exist_ok=True)

# Get initial leaderboard data
LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)

# Create the Gradio interface
demo = gr.Blocks(css=custom_css)
with demo:
    gr.HTML(TITLE)
    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")

    with gr.Tabs(elem_classes="tab-buttons") as tabs:
        with gr.TabItem("🏅 Leaderboard", elem_id="leaderboard-tab", id=0):
            leaderboard = init_leaderboard(LEADERBOARD_DF)

        with gr.TabItem("📝 About", elem_id="about-tab", id=1):
            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")

        with gr.TabItem("🧪 Test Model", elem_id="test-model-tab", id=2):
            gr.Markdown("## Run Perplexity Test\n\nTest any Hugging Face model for perplexity evaluation.")
            
            with gr.Row():
                with gr.Column():
                    model_name = gr.Textbox(label="Model name", placeholder="openai-community/gpt2")
                    revision = gr.Textbox(label="Revision", placeholder="main", value="main")
                    precision = gr.Dropdown(
                        choices=["float16", "bfloat16"],
                        label="Precision",
                        value="float16"
                    )
                    debug_mode = gr.Checkbox(label="Enable debug mode (more verbose logging)", value=True)
                
                with gr.Column():
                    test_button = gr.Button("🚀 Run Perplexity Test", variant="primary")
                    result = gr.Markdown()
            
            gr.Markdown("""
            ### Tips:
            - Check stderr logs in HF Spaces for detailed debugging information
            - If evaluation succeeds but leaderboard doesn't update, try refreshing the page
            - Example models to test: `openai-community/gpt2`, `EleutherAI/gpt-neo-1.3B`
            """)
            
            test_button.click(
                run_perplexity_test,
                [model_name, revision, precision],
                [result, leaderboard]
            )

demo.queue(default_concurrency_limit=5).launch()