Hugging Face's logo

Spaces:
hivetrace
/
GuardRateLeaderboard
Running

App Files Community
GuardRateLeaderboard / docs /backend-data-contract_en.md
Anton Malykhin
feat: stabilize HF deploy, locale routing, and bucket caching
c85bc2e
preview code
|
Raw
History Blame Contribute Delete
18.6 kB

Backend bucket data contract

This document defines the current data format in the private HF bucket that the frontend can integrate with. It is a contract for JSON shapes, relationships between entities, and nullable fields. Formula correctness, ranking logic, and business calculations remain the backend's responsibility.

Verified snapshot:

{
    "schema_version": "1.0.0",
    "snapshot_id": "snapshot_2026-07-10",
    "snapshot_date": "2026-07-10",
    "generated_at": "2026-07-10T19:17:38+00:00",
    "model_count": 48,
    "group_count": 19,
    "dataset_count": 44
}

General rules

  • All files use schema_version: "1.0.0".
  • All missing numeric metrics are represented as null; they are not omitted.
  • model_id, group_id, and dataset_id are string identifiers.
  • In the current bucket, model_id includes the run suffix, for example qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1.
  • Display order comes from the rows, model_ids, group_ids, row_ids, and column_ids arrays.
  • title, subtitle, x_axis, y_axis, label, description, and similar translated fields have the shape { "en": "...", "ru": "..." }.
  • subtitle may be present in visualization payloads; the frontend may use or ignore it.
  • manifest.files is the source of paths to payload files.
  • manifest.hashes contains SHA-256 hashes of payload files and must be updated whenever the corresponding JSON changes.

File list 

manifest.json
catalog.json
leaderboard.json
details_matrix.json
drilldown/index.json
visualizations/radar.json
visualizations/scatter.json
visualizations/heatmap.json
visualizations/grouped_bars.json
visualizations/pareto.json
visualizations/performance.json
visualizations/robustness.json

Common types 

type LocalizedString = {
    en: string;
    ru: string;
};

type NullableNumber = number | null;
type MetricDirection = 'higher' | 'lower';
type MetricKey =
    'integral' | 'score' | 'f1' | 'recall' | 'precision' | 'accuracy' | 'fpr' | 'fnr' | 'latency_ms';

type LatencyMs = {
    p50: NullableNumber;
    p95: NullableNumber;
    p99: NullableNumber;
};

type GroupMetrics = {
    score: NullableNumber;
    f1: NullableNumber;
    fpr: NullableNumber;
    fnr: NullableNumber;
};

type FullMetrics = GroupMetrics & {
    recall: NullableNumber;
    precision: NullableNumber;
    accuracy: NullableNumber;
};

manifest.json 

type BucketFileKey =
    | 'catalog'
    | 'leaderboard'
    | 'details_matrix'
    | 'drilldown_index'
    | 'radar'
    | 'scatter'
    | 'heatmap'
    | 'grouped_bars'
    | 'pareto'
    | 'performance'
    | 'robustness';

type Manifest = {
    schema_version: '1.0.0';
    generated_at: string;
    snapshot_id: string;
    snapshot_date: string;
    is_ok: boolean;
    model_count: number;
    group_count: number;
    dataset_count: number;
    files: Record<BucketFileKey, string>;
    hashes: Record<BucketFileKey, string>;
};

{
    "schema_version": "1.0.0",
    "generated_at": "2026-07-10T19:17:38+00:00",
    "snapshot_id": "snapshot_2026-07-10",
    "snapshot_date": "2026-07-10",
    "is_ok": true,
    "model_count": 48,
    "group_count": 19,
    "dataset_count": 44,
    "files": {
        "catalog": "catalog.json",
        "leaderboard": "leaderboard.json",
        "details_matrix": "details_matrix.json",
        "drilldown_index": "drilldown/index.json",
        "radar": "visualizations/radar.json",
        "scatter": "visualizations/scatter.json",
        "heatmap": "visualizations/heatmap.json",
        "grouped_bars": "visualizations/grouped_bars.json",
        "pareto": "visualizations/pareto.json",
        "performance": "visualizations/performance.json",
        "robustness": "visualizations/robustness.json"
    },
    "hashes": {
        "catalog": "6cde39ef39563ec1db551620cdcbd6a8db4751b05a3d886c97453a382f0309fc",
        "leaderboard": "3e1f957fd2e3d994b200b7d3b35f6606c3fc56fad6c1fe8facac72a513b88cc0",
        "details_matrix": "ea9e89936cf59075e78c8f907b8a1b922f8ce1105a56669a829131e1ce56b4ba",
        "drilldown_index": "5cc2ceca5eeb38514a872b91fe301b1481109404547bbcc9a30622eb19b1a4da",
        "radar": "5ce3a62644f768feb3f52bd787993f6281eec935879d9fd04c10c3835028f249",
        "scatter": "f8c5b6cc91a2f586fbfb9a69bcc9b1472363c4cdda61c6501aff9abcc14b0756",
        "heatmap": "d2665f3f4df58ca2d05a138cee5186bf3a16d287d61c459e9b2ecc4e69914b1a",
        "grouped_bars": "678a531e2b1593de9e8bdd45f17d116aa4f5051d4e246f2abdcf634d5d86d7f4",
        "pareto": "0a1b2bad8addd2df35f86a90d4fce59a76f846d031b56278ef4fb029ff425535",
        "performance": "5cecdcb010454e7918250c526eb69b20199ba8317017d9b30938104b97fb77bf",
        "robustness": "362664c44b3acf2b5927de59fac7f8c3ed4492c6321351c21e68c139a250953f"
    }
}

catalog.json 

type BucketModel = {
    model_id: string;
    display_name: string;
    short_name: string;
    org: string;
    family: string;
    guardrail_type: 'llm-judge' | 'classifier' | 'unknown';
    license: string | null;
    params_b: NullableNumber;
    size_label: LocalizedString | null;
    languages: string[];
    note: LocalizedString | null;
    run_date: string;
    hf_url: string | null;
    eval_leak: boolean | null;
};

type BucketGroup = {
    group_id: string;
    label: LocalizedString;
    description: LocalizedString | null;
    what_it_tests: LocalizedString | null;
    languages: string[];
    dataset_ids: string[];
};

type BucketDataset = {
    dataset_id: string;
    group_id: string;
    label: LocalizedString;
    source_name: LocalizedString | null;
    split: LocalizedString | null;
    subset: LocalizedString | null;
    language: string | null;
    is_combined: boolean;
};

type Catalog = {
    schema_version: '1.0.0';
    models: BucketModel[];
    groups: BucketGroup[];
    datasets: BucketDataset[];
    metric_directions: Record<MetricKey, MetricDirection>;
    metric_labels: Partial<Record<MetricKey, LocalizedString>>;
};

{
    "schema_version": "1.0.0",
    "models": [
        {
            "model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
            "display_name": "Qwen3Guard-Gen-0.6B",
            "short_name": "Qwen3Guard-Gen-0.6B",
            "org": "Qwen",
            "family": "Qwen3",
            "guardrail_type": "llm-judge",
            "license": "Apache-2.0",
            "params_b": 0.6,
            "size_label": { "en": "0.6B", "ru": "0.6B" },
            "languages": ["multilingual", "en", "ru", "zh"],
            "note": {
                "en": "Small Qwen3Guard for low-latency; model_size marked as 0.8B in the run, actual checkpoint ~0.6B.",
                "ru": "Малый Qwen3Guard для low-latency; model_size в прогоне помечен 0.8B, фактический чекпойнт ~0.6B."
            },
            "run_date": "2026-06-06",
            "hf_url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-0.6B",
            "eval_leak": null
        }
    ],
    "groups": [
        {
            "group_id": "s_eval",
            "label": { "en": "S-Eval", "ru": "S-Eval" },
            "description": {
                "en": "S-Eval benchmark family.",
                "ru": "Семейство бенчмарков S-Eval."
            },
            "what_it_tests": {
                "en": "Recognition of unsafe intent and robustness to adversarial attacks.",
                "ru": "Распознавание небезопасного намерения и устойчивость к adversarial-атакам."
            },
            "languages": ["en", "ru"],
            "dataset_ids": ["s_eval_base_risk", "s_eval_attack_set"]
        }
    ],
    "datasets": [
        {
            "dataset_id": "s_eval_base_risk",
            "group_id": "s_eval",
            "label": { "en": "S-Eval (base risk)", "ru": "S-Eval (base risk)" },
            "source_name": null,
            "split": null,
            "subset": null,
            "language": "en",
            "is_combined": false
        }
    ],
    "metric_directions": {
        "integral": "higher",
        "score": "higher",
        "f1": "higher",
        "recall": "higher",
        "precision": "higher",
        "accuracy": "higher",
        "fpr": "lower",
        "fnr": "lower",
        "latency_ms": "lower"
    },
    "metric_labels": {
        "score": { "en": "Score", "ru": "Скор" },
        "f1": { "en": "F1", "ru": "F1" },
        "fpr": { "en": "FPR", "ru": "FPR" },
        "fnr": { "en": "FNR", "ru": "FNR" },
        "integral": { "en": "Integral score", "ru": "Интегральный скор" },
        "latency_ms": { "en": "Latency, ms", "ru": "Задержка, мс" }
    }
}

leaderboard.json 

type LeaderboardRow = {
    model_id: string;
    rank: number;
    run_id: string;
    run_date: string;
    integral: NullableNumber;
    min_group: NullableNumber;
    overall_fpr: NullableNumber;
    overall_fnr: NullableNumber;
    overall_f1: NullableNumber;
    latency_ms: LatencyMs;
    groups: Record<string, GroupMetrics>;
};

type Leaderboard = {
    schema_version: '1.0.0';
    rows: LeaderboardRow[];
};

{
    "schema_version": "1.0.0",
    "rows": [
        {
            "model_id": "alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f",
            "rank": 1,
            "run_id": "run_20260701_155019_4a9b279f",
            "run_date": "2026-07-01",
            "integral": 0.7605880393198141,
            "min_group": 0.15654499428022556,
            "overall_fpr": 0.21801750988077712,
            "overall_fnr": 0.12210970263223303,
            "overall_f1": 0.8224670573864103,
            "latency_ms": { "p50": 71.79, "p95": 102.57, "p99": 144.35 },
            "groups": {
                "s_eval": {
                    "score": 0.8761476877122706,
                    "f1": null,
                    "fpr": null,
                    "fnr": 0.11835
                }
            }
        }
    ]
}

details_matrix.json 

type GroupCell = FullMetrics & {
    model_id: string;
    group_id: string;
};

type DatasetCell = GroupCell & {
    dataset_id: string;
    sample_count: number | null;
};

type DetailsMatrix = {
    schema_version: '1.0.0';
    group_cells: GroupCell[];
    dataset_cells: DatasetCell[];
};

{
    "schema_version": "1.0.0",
    "group_cells": [
        {
            "model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
            "group_id": "s_eval",
            "score": 0.6505268251330913,
            "f1": null,
            "fpr": null,
            "fnr": 0.34645000000000004,
            "recall": null,
            "precision": null,
            "accuracy": null
        }
    ],
    "dataset_cells": [
        {
            "model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
            "group_id": "s_eval",
            "dataset_id": "s_eval_base_risk",
            "score": 0.698,
            "f1": null,
            "fpr": null,
            "fnr": 0.302,
            "recall": 0.698,
            "precision": null,
            "accuracy": null,
            "sample_count": 1000
        }
    ]
}

drilldown/index.json 

type DrilldownMetric = 'score' | 'f1' | 'fpr' | 'fnr';

type DrilldownHeatmapRow = {
    model_id: string;
    group_id: string;
    dataset_ids: string[];
    selected_metric: DrilldownMetric;
    metrics: GroupMetrics;
    description: LocalizedString | null;
    prompt_viewer_available: boolean;
    prompt_viewer_path: string | null;
};

type DrilldownIndex = {
    schema_version: '1.0.0';
    heatmap: DrilldownHeatmapRow[];
};

{
    "schema_version": "1.0.0",
    "heatmap": [
        {
            "model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
            "group_id": "s_eval",
            "dataset_ids": ["s_eval_base_risk", "s_eval_attack_set"],
            "selected_metric": "fnr",
            "metrics": {
                "score": 0.6505268251330913,
                "f1": null,
                "fpr": null,
                "fnr": 0.34645000000000004
            },
            "description": {
                "en": "Small Qwen3Guard for low-latency; model_size marked as 0.8B in the run, actual checkpoint ~0.6B.",
                "ru": "Малый Qwen3Guard для low-latency; model_size в прогоне помечен 0.8B, фактический чекпойнт ~0.6B."
            },
            "prompt_viewer_available": false,
            "prompt_viewer_path": null
        }
    ]
}

visualizations/radar.json 

type Radar = {
    schema_version: '1.0.0';
    title: LocalizedString;
    subtitle?: LocalizedString;
    model_ids: string[];
    group_ids: string[];
    default_model_ids: string[];
    values: Record<string, Record<string, NullableNumber>>;
    f1_values: Record<string, Record<string, NullableNumber>>;
};

values and f1_values are indexed as values[group_id][model_id].

{
    "schema_version": "1.0.0",
    "title": { "en": "Model Safety Profile", "ru": "Профиль безопасности модели" },
    "subtitle": { "en": "Group scores", "ru": "Групповые скоры" },
    "model_ids": ["qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1"],
    "group_ids": ["s_eval"],
    "default_model_ids": ["qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1"],
    "values": {
        "s_eval": {
            "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1": 0.6505268251330913
        }
    },
    "f1_values": {
        "s_eval": {
            "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1": null
        }
    }
}

visualizations/scatter.json 

type ScatterRowId = 'fpr' | 'fnr' | 'overall';

type ScatterPoint = {
    model_id: string;
    fpr: NullableNumber;
    fnr: NullableNumber;
    tooltip: string | null;
};

type Scatter = {
    schema_version: '1.0.0';
    title: LocalizedString;
    x_axis: LocalizedString;
    y_axis: LocalizedString;
    points: ScatterPoint[];
    row_ids: ScatterRowId[];
    values: Record<ScatterRowId, Record<string, NullableNumber>>;
};

values is indexed as values[row_id][model_id].

{
    "schema_version": "1.0.0",
    "title": { "en": "FPR vs FNR (overall)", "ru": "FPR vs FNR (overall)" },
    "x_axis": { "en": "FPR", "ru": "FPR" },
    "y_axis": { "en": "FNR", "ru": "FNR" },
    "points": [
        {
            "model_id": "alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f",
            "fpr": 0.21801750988077712,
            "fnr": 0.12210970263223303,
            "tooltip": null
        }
    ],
    "row_ids": ["fpr", "fnr", "overall"],
    "values": {
        "fpr": {
            "alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f": 0.21801750988077712
        },
        "fnr": {
            "alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f": 0.12210970263223303
        },
        "overall": {
            "alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f": 0.7605880393198141
        }
    }
}

visualizations/heatmap.json 

type Heatmap = {
    schema_version: '1.0.0';
    title: LocalizedString;
    subtitle?: LocalizedString;
    metric: 'score' | 'f1' | 'fpr' | 'fnr';
    row_ids: string[];
    column_ids: string[];
    values: Record<string, Record<string, NullableNumber>>;
};

values is indexed as values[row_id][model_id]. In the current snapshot, metric is "fnr".

{
    "schema_version": "1.0.0",
    "title": { "en": "Heatmap FNR", "ru": "Тепловая карта FNR" },
    "subtitle": { "en": "Dataset x model", "ru": "Датасет x модель" },
    "metric": "fnr",
    "row_ids": ["aegis_2_0_prompt"],
    "column_ids": ["qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1"],
    "values": {
        "aegis_2_0_prompt": {
            "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1": 0.05854579792256846
        }
    }
}

visualizations/grouped_bars.json 

type GroupedBarsModel = {
    model_id: string;
    groups: Record<
        string,
        {
            fnr: NullableNumber;
            fpr: NullableNumber;
            f1: NullableNumber;
        }
    >;
};

type GroupedBars = {
    schema_version: '1.0.0';
    title: LocalizedString;
    subtitle?: LocalizedString;
    group_ids: string[];
    models: GroupedBarsModel[];
};

{
    "schema_version": "1.0.0",
    "title": { "en": "Grouped bars", "ru": "Групповые столбцы" },
    "subtitle": {
        "en": "FNR / FPR / F1 by benchmark group",
        "ru": "FNR / FPR / F1 по группам"
    },
    "group_ids": ["s_eval", "aegis_2_0"],
    "models": [
        {
            "model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
            "groups": {
                "s_eval": { "fnr": 0.34645000000000004, "fpr": null, "f1": null },
                "aegis_2_0": {
                    "fnr": 0.08384142688006596,
                    "fpr": 0.2825050061521388,
                    "f1": 0.8346634240315878
                }
            }
        }
    ]
}

visualizations/pareto.json 

type ParetoModel = {
    latency_ms: LatencyMs;
    integral: NullableNumber;
    fpr: NullableNumber;
};

type Pareto = {
    schema_version: '1.0.0';
    title: LocalizedString;
    subtitle?: LocalizedString;
    x_axis: LocalizedString;
    y_axis: LocalizedString;
    by_model_id: Record<string, ParetoModel>;
};

{
    "schema_version": "1.0.0",
    "title": { "en": "Pareto: latency vs quality", "ru": "Парето: задержка vs качество" },
    "subtitle": {
        "en": "p95 latency vs integral score",
        "ru": "p95 задержка vs интегральный скор"
    },
    "x_axis": { "en": "Latency p95, ms", "ru": "Задержка p95, мс" },
    "y_axis": { "en": "Integral score", "ru": "Интегральный скор" },
    "by_model_id": {
        "alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f": {
            "latency_ms": { "p50": 71.79, "p95": 102.57, "p99": 144.35 },
            "integral": 0.7605880393198141,
            "fpr": 0.21801750988077712
        }
    }
}

visualizations/performance.json 

type PerformanceRow = {
    model_id: string;
    rank: number;
    latency_ms: LatencyMs;
    integral: NullableNumber;
};

type Performance = {
    schema_version: '1.0.0';
    title: LocalizedString;
    rows: PerformanceRow[];
};

{
    "schema_version": "1.0.0",
    "title": { "en": "Latency / performance", "ru": "Задержка / производительность" },
    "rows": [
        {
            "model_id": "alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f",
            "rank": 1,
            "latency_ms": { "p50": 71.79, "p95": 102.57, "p99": 144.35 },
            "integral": 0.7605880393198141
        }
    ]
}

visualizations/robustness.json 

type RobustnessRow = {
    model_id: string;
    score_real: NullableNumber;
    score_robust: NullableNumber;
    delta_score: NullableNumber;
    fnr_real: NullableNumber;
    fnr_robust: NullableNumber;
    delta_fnr: NullableNumber;
    category_label: LocalizedString;
};

type Robustness = {
    schema_version: '1.0.0';
    title: LocalizedString;
    rows: RobustnessRow[];
};

{
    "schema_version": "1.0.0",
    "title": { "en": "Robustness comparison", "ru": "Сравнение robustness" },
    "rows": [
        {
            "model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
            "score_real": 0.6313559322033898,
            "score_robust": 0.8121144139091419,
            "delta_score": 0.18075848170575204,
            "fnr_real": 0.13623188405797101,
            "fnr_robust": 0.21190130624092887,
            "delta_fnr": 0.07566942218295786,
            "category_label": {
                "en": "Qwen3Guard-Gen-0.6B",
                "ru": "Qwen3Guard-Gen-0.6B"
            }
        }
    ]
}

Validation invariants

  • catalog.models.length === manifest.model_count.
  • catalog.groups.length === manifest.group_count.
  • catalog.datasets.length === manifest.dataset_count.
  • leaderboard.rows[].model_id must reference catalog.models[].model_id.
  • details_matrix.group_cells[].model_id and details_matrix.dataset_cells[].model_id must reference the catalog.
  • details_matrix.group_cells[].group_id must reference catalog.groups[].group_id.
  • details_matrix.dataset_cells[].dataset_id must reference catalog.datasets[].dataset_id.
  • The current snapshot is expected to contain 48 unique model_id values in catalog and leaderboard.
  • The current snapshot is expected to contain 912 unique model_id + group_id pairs in details_matrix.group_cells.
  • The current snapshot is expected to contain 2112 unique model_id + group_id + dataset_id triples in details_matrix.dataset_cells.