Spaces:
Running
Running
File size: 2,296 Bytes
b96b498 64db9d1 b96b498 64db9d1 b96b498 64db9d1 0d22cb0 1178156 64db9d1 b96b498 64db9d1 0d22cb0 1178156 64db9d1 b96b498 64db9d1 b96b498 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | """Disk cache for Benchmark-tab results, keyed by (config_hash, row_index).
Re-running the Benchmark tab with the same config + sample reuses cached
scores instead of re-running retrieval/generation/API calls, mirroring the
checkpointing pattern from Notebooks 5-6.
Gold scores (from RAGBench labels) are stored alongside predicted scores so
that RMSE/AUROC can be recomputed from cache without re-querying the dataset.
"""
import hashlib
import json
import pandas as pd
from src import settings
RESULTS_PATH = settings.APP_CACHE_DIR / "benchmark_results.csv"
COLUMNS = [
"config_hash", "row_index", "question",
# predicted scores
"relevance", "adherence", "completeness", "utilization",
# RAGBench gold labels
"gold_adherence", "gold_relevance", "gold_utilization", "gold_completeness",
# run diagnostics
"rejected", "abstained", "latency_s",
]
def config_hash(dataset: str, index_id: str, config_dict: dict) -> str:
raw = json.dumps({"dataset": dataset, "index_id": index_id, **config_dict}, sort_keys=True)
return hashlib.md5(raw.encode("utf-8")).hexdigest()[:12]
def load_results() -> pd.DataFrame:
if RESULTS_PATH.exists():
df = pd.read_csv(RESULTS_PATH)
# Back-fill columns added after older cache files were written
for col in ["gold_adherence", "gold_relevance", "gold_utilization", "gold_completeness",
"rejected", "abstained", "latency_s"]:
if col not in df.columns:
df[col] = float("nan")
return df
return pd.DataFrame(columns=COLUMNS)
def save_results(df: pd.DataFrame) -> None:
df.to_csv(RESULTS_PATH, index=False)
def get_cached_rows(results_df: pd.DataFrame, chash: str) -> pd.DataFrame:
return results_df[results_df["config_hash"] == chash]
def upsert_rows(results_df: pd.DataFrame, chash: str, new_rows: pd.DataFrame) -> pd.DataFrame:
"""Replace any rows for (chash, row_index) with new_rows, keeping everything else."""
new_keys = set(zip(new_rows["row_index"], [chash] * len(new_rows)))
keep_mask = ~results_df.apply(
lambda r: (r["row_index"], r["config_hash"]) in new_keys, axis=1
)
merged = pd.concat([results_df[keep_mask], new_rows], ignore_index=True)
save_results(merged)
return merged
|