{render_summary(metric)}
{render_nav(active_page)}
{status_html}
{render_page_content(active_page, metric, task, verified, query, query_string)}
'
def dataset_names_for_task(task_filter: str) -> set[str]:
datasets = load_datasets()
if task_filter != "All":
datasets = [dataset for dataset in datasets if dataset.get("task") == task_filter]
return {dataset["name"] for dataset in datasets if dataset.get("official", True)}
def compute_average(record: dict[str, Any], metric: str, dataset_names: set[str]) -> float | None:
scores = record.get("scores", {})
explicit = metric_value(scores.get("average", {}), metric)
if explicit is not None:
return explicit
values = []
for dataset_name, dataset_metrics in scores.get("datasets", {}).items():
if dataset_name in dataset_names:
value = metric_value(dataset_metrics, metric)
if value is not None:
values.append(value)
return sum(values) / len(values) if values else None
def leaderboard_rows(metric: str, task_filter: str, verified_only: bool, query: str) -> tuple[list[dict[str, Any]], set[str]]:
dataset_names = dataset_names_for_task(task_filter)
rows = []
for record in load_results():
if verified_only and not record.get("verified", False):
continue
model_text = f"{record.get('model_id', '')} {record.get('model_name', '')}".lower()
if query and query.lower() not in model_text:
continue
dataset_scores = record.get("scores", {}).get("datasets", {})
row = {
"Rank": None,
"Model": record.get("model_name") or record.get("model_id"),
metric: compute_average(record, metric, dataset_names),
"Model ID": record.get("model_id", ""),
"Organization": record.get("organization", ""),
"Type": record.get("type", ""),
"Verified": "yes" if record.get("verified", False) else "no",
"Date": record.get("date", ""),
"Source URL": record.get("source_url", ""),
}
for dataset_name in sorted(dataset_names):
row[dataset_name] = metric_value(dataset_scores.get(dataset_name, {}), metric)
rows.append(row)
rows.sort(key=lambda row: row[metric] if row[metric] is not None else -1, reverse=True)
for index, row in enumerate(rows, start=1):
row["Rank"] = index
return rows, dataset_names
def format_score(value: Any) -> str:
if value is None:
return ""
try:
return f"{float(value):.4f}"
except (TypeError, ValueError):
return escape(str(value))
def column_class(column: str, metric: str, index: int) -> str:
if index == 0:
return "col-rank sticky-rank"
if index == 1:
return "col-model sticky-model"
if column == metric:
return "col-average sticky-average"
if column == "Model ID":
return "col-model-id"
if column in {"Organization", "Type", "Verified"}:
return "col-meta"
if column == "Date":
return "col-date"
if column == "Source URL":
return "col-source"
return "col-dataset"
def render_table(metric: str, task_filter: str, verified_only: bool, query: str) -> str:
rows, dataset_names = leaderboard_rows(metric, task_filter, verified_only, query)
columns = [
"Rank",
"Model",
metric,
"Model ID",
"Organization",
"Type",
"Verified",
*sorted(dataset_names),
*TRAILING_COLUMNS,
]
headers = "".join(
f'Compare dense retrievers, sparse baselines, and reranker pipelines on official RusBEIR datasets. The default ranking is the macro-average of {escape(metric)}.
| Dataset | Task | Split | Corpus repo | Qrels repo | Origin |
|---|
RusBEIR tasks used for the default macro-average ranking.
Paste JSONL content or upload a ready JSONL file. Each record must include model_id, scores, and at least one numeric metric.
RusBEIR is a Russian BEIR-style benchmark for zero-shot information retrieval. The leaderboard is backed by a plain JSONL file.
Verified rows should point to reproducible logs or a commit with generated retrieval results.
Filter by task family, model name, or verification status. Scores are stored as fractions.
{render_filters(metric, task, verified, query)} {render_table(metric, task, verified, query)}