Spaces:
Sleeping
Sleeping
File size: 3,273 Bytes
577fb61 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | """
Configuration file for leaderboard generation scripts.
⚠️ IMPORTANT: Update the EXCEL_PATH below before running scripts/main.py
Place your Excel file in the project root or update the path accordingly.
"""
from pathlib import Path
# =============================================================================
# PATH CONFIGURATION - UPDATE THESE PATHS FOR YOUR SETUP
# =============================================================================
# Base directory for leaderboard JSON files
# If running from scripts directory, parent directory contains leaderboards/
BASE_DIR = Path(__file__).parent.parent
# ⚠️ UPDATE THIS PATH: Path to the Excel file containing model and task data
# This is the source data file (typically "Clinical Benchmark and LLM.xlsx")
#
# Examples:
# EXCEL_PATH = BASE_DIR / "Clinical Benchmark and LLM.xlsx" # In project root
# EXCEL_PATH = Path("/Users/yourname/Desktop/Clinical Benchmark and LLM.xlsx") # Absolute path
# EXCEL_PATH = BASE_DIR / "data" / "benchmark.xlsx" # In data subfolder
#
EXCEL_PATH = BASE_DIR / "Clinical Benchmark and LLM.xlsx"
# Output directory for leaderboard JSON files (relative to BASE_DIR)
LEADERBOARDS_DIR = BASE_DIR / "leaderboards"
# Output paths for each leaderboard type (relative to BASE_DIR)
ZERO_SHOT_OUTPUT = "leaderboards/Zero-Shot_leaderboard.json"
FEW_SHOT_OUTPUT = "leaderboards/Few-Shot_leaderboard.json"
COT_OUTPUT = "leaderboards/CoT_leaderboard.json"
# Full paths to leaderboard files (for update_ranks and other operations)
ZERO_SHOT_PATH = BASE_DIR / ZERO_SHOT_OUTPUT
FEW_SHOT_PATH = BASE_DIR / FEW_SHOT_OUTPUT
COT_PATH = BASE_DIR / COT_OUTPUT
# Task information output path
TASK_INFO_PATH = "task_information.json"
# =============================================================================
# LEADERBOARD CONFIGURATION
# =============================================================================
# Configuration for all leaderboards
# Each leaderboard has: name, output_path, and sheet_names from Excel
LEADERBOARD_CONFIGS = [
{
'name': 'Zero-Shot',
'output_path': ZERO_SHOT_OUTPUT,
'sheet_names': ["B-CLF", "B-EXT", "B-GEN"]
},
{
'name': 'Few-Shot',
'output_path': FEW_SHOT_OUTPUT,
'sheet_names': ["B-CLF-5shot", "B-EXT-5shot", "B-GEN-5shot"]
},
{
'name': 'CoT',
'output_path': COT_OUTPUT,
'sheet_names': ["B-CLF-CoT", "B-EXT-CoT", "B-GEN-CoT"]
}
]
# List of all leaderboard paths (for operations that need to iterate through all)
ALL_LEADERBOARD_PATHS = [
ZERO_SHOT_PATH,
FEW_SHOT_PATH,
COT_PATH
]
# =============================================================================
# MODEL CONFIGURATION
# =============================================================================
# Models to exclude from the leaderboards
# These are models that appear in the Models sheet but don't have evaluation data
INVALID_MODELS = [
"gemma-3-27b-pt",
"gemma-3-12b-pt",
"gemma-3-12b-pt-ylab-4-1-1",
"gemma-3-12b-pt-ylab-8-1-1",
"gemma-3-12b-pt-ylab-16-1-1",
"Hulu-Med-7B", # Missing evaluation data in task sheets
"Hulu-Med-14B", # Missing data in the task sheets
"Hulu-Med-32B", # Missing data in the task sheets
]
|