import sys import typing from functools import lru_cache from pathlib import Path from huggingface_hub import snapshot_download from loguru import logger from src.envs import settings from src.schemas.display_toml import DisplayToml from src.schemas.meta_toml import MetaToml from src.utils.hf import restart_space if typing.TYPE_CHECKING: from src.schemas.meta_toml import MetaToml_Benchmark if sys.version_info < (3, 11): from tomli import load as toml_load else: from tomllib import load as toml_load # Use cache to avoid downloading the same data multiple times @lru_cache(maxsize=1) def prepare_space(): """Space initialisation""" download_results() download_queue() def download_results(): try: snapshot_download( repo_id=settings.QUEUE_REPO_ID, local_dir=settings.EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=settings.HF_TOKEN.get_secret_value(), ) logger.info("Downloaded eval queue") except Exception as e: logger.error(f"Error downloading eval queue: {e!s}") restart_space(settings.REPO_ID) def download_queue(): try: snapshot_download( repo_id=settings.RESULTS_REPO_ID, local_dir=settings.EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, allow_patterns=["leaderboard/*.toml", "leaderboard/**/*.json"], token=settings.HF_TOKEN.get_secret_value(), ) logger.info("Downloaded eval results") except Exception as e: logger.error(f"Error downloading eval queue: {e!s}") restart_space(settings.REPO_ID) @lru_cache(maxsize=1) def load_meta_toml() -> "MetaToml": prepare_space() meta_toml_path = Path(settings.EVAL_RESULTS_PATH) / "leaderboard" / "meta.toml" logger.info(f'Loading meta.toml from: {meta_toml_path.as_posix()!r}') with meta_toml_path.open("rb") as f: data = toml_load(f) meta_toml = MetaToml.model_validate(data) logger.info("Loaded meta.toml") assert meta_toml is not None, f"Failed to load meta.toml: {meta_toml_path.as_posix()!r}" return meta_toml @lru_cache(maxsize=1) def load_display_toml() -> "DisplayToml": prepare_space() display_toml_path = Path(settings.EVAL_RESULTS_PATH) / "leaderboard" / "display.toml" logger.info(f'Loading display.toml from: {display_toml_path.as_posix()!r}') with display_toml_path.open("rb") as f: data = toml_load(f) display_toml = DisplayToml.model_validate(data) logger.info("Loaded display.toml") assert display_toml is not None, f"Failed to load display.toml: {display_toml_path.as_posix()!r}" return display_toml @lru_cache(maxsize=1) def get_benchmarks() -> list["MetaToml_Benchmark"]: meta_toml = load_meta_toml() benchmarks = list(meta_toml.benchmarks) # Sort with None values at the end, maintaining stable order for None values # Use tuple (is_none, order_value) where is_none=True means it's None # False < True in boolean comparison, so non-None values come first benchmarks.sort(key=lambda x: (x.order is None, x.order if x.order is not None else 0)) logger.info(f"Loaded {len(benchmarks)} benchmarks: titles={[b.title for b in benchmarks]!r}") return benchmarks