scripts/config.py

"""
Configuration file for leaderboard generation scripts.

⚠️ IMPORTANT: Update the EXCEL_PATH below before running scripts/main.py
Place your Excel file in the project root or update the path accordingly.
"""

from pathlib import Path

# =============================================================================
# PATH CONFIGURATION - UPDATE THESE PATHS FOR YOUR SETUP
# =============================================================================

# Base directory for leaderboard JSON files
# If running from scripts directory, parent directory contains leaderboards/
BASE_DIR = Path(__file__).parent.parent

# ⚠️ UPDATE THIS PATH: Path to the Excel file containing model and task data
# This is the source data file (typically "Clinical Benchmark and LLM.xlsx")
# 
# Examples:
#   EXCEL_PATH = BASE_DIR / "Clinical Benchmark and LLM.xlsx"  # In project root
#   EXCEL_PATH = Path("/Users/yourname/Desktop/Clinical Benchmark and LLM.xlsx")  # Absolute path
#   EXCEL_PATH = BASE_DIR / "data" / "benchmark.xlsx"  # In data subfolder
#
EXCEL_PATH = BASE_DIR / "Clinical Benchmark and LLM.xlsx"

# Output directory for leaderboard JSON files (relative to BASE_DIR)
LEADERBOARDS_DIR = BASE_DIR / "leaderboards"

# Output paths for each leaderboard type (relative to BASE_DIR)
ZERO_SHOT_OUTPUT = "leaderboards/Zero-Shot_leaderboard.json"
FEW_SHOT_OUTPUT = "leaderboards/Few-Shot_leaderboard.json"
COT_OUTPUT = "leaderboards/CoT_leaderboard.json"

# Full paths to leaderboard files (for update_ranks and other operations)
ZERO_SHOT_PATH = BASE_DIR / ZERO_SHOT_OUTPUT
FEW_SHOT_PATH = BASE_DIR / FEW_SHOT_OUTPUT
COT_PATH = BASE_DIR / COT_OUTPUT

# Task information output path
TASK_INFO_PATH = "task_information.json"

# =============================================================================
# LEADERBOARD CONFIGURATION
# =============================================================================

# Configuration for all leaderboards
# Each leaderboard has: name, output_path, and sheet_names from Excel
LEADERBOARD_CONFIGS = [
    {
        'name': 'Zero-Shot',
        'output_path': ZERO_SHOT_OUTPUT,
        'sheet_names': ["B-CLF", "B-EXT", "B-GEN"]
    },
    {
        'name': 'Few-Shot',
        'output_path': FEW_SHOT_OUTPUT,
        'sheet_names': ["B-CLF-5shot", "B-EXT-5shot", "B-GEN-5shot"]
    },
    {
        'name': 'CoT',
        'output_path': COT_OUTPUT,
        'sheet_names': ["B-CLF-CoT", "B-EXT-CoT", "B-GEN-CoT"]
    }
]

# List of all leaderboard paths (for operations that need to iterate through all)
ALL_LEADERBOARD_PATHS = [
    ZERO_SHOT_PATH,
    FEW_SHOT_PATH,
    COT_PATH
]

# =============================================================================
# MODEL CONFIGURATION
# =============================================================================

# Models to exclude from the leaderboards
# These are models that appear in the Models sheet but don't have evaluation data
INVALID_MODELS = [
    "gemma-3-27b-pt",
    "gemma-3-12b-pt",
    "gemma-3-12b-pt-ylab-4-1-1",
    "gemma-3-12b-pt-ylab-8-1-1",
    "gemma-3-12b-pt-ylab-16-1-1",
    "Hulu-Med-7B",  # Missing evaluation data in task sheets
    "Hulu-Med-14B", # Missing data in the task sheets
    "Hulu-Med-32B", # Missing data in the task sheets
]