Hugging Face's logo

Spaces:
hivetrace
/
GuardRateLeaderboard
Running

App Files Community
GuardRateLeaderboard / docs /backend-data-contract_ru.md
Anton Malykhin
feat: stabilize HF deploy, locale routing, and bucket caching
c85bc2e
preview code
|
Raw
History Blame Contribute Delete
19.4 kB

Backend bucket data contract

Документ фиксирует текущий формат данных в приватном HF bucket, который фронт может интегрировать. Это контракт по форме JSON, ссылкам между сущностями и nullable-полям. Корректность формул, ранжирования и бизнес-расчётов остаётся на стороне backend.

Проверенный snapshot:

{
    "schema_version": "1.0.0",
    "snapshot_id": "snapshot_2026-07-10",
    "snapshot_date": "2026-07-10",
    "generated_at": "2026-07-10T19:17:38+00:00",
    "model_count": 48,
    "group_count": 19,
    "dataset_count": 44
}

Общие правила

  • Все файлы используют schema_version: "1.0.0".
  • Все отсутствующие числовые метрики передаются как null, не пропускаются.
  • model_id, group_id и dataset_id являются строковыми идентификаторами.
  • model_id в текущем bucket включает суффикс run, например qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1.
  • Порядок отображения берётся из массивов rows, model_ids, group_ids, row_ids, column_ids.
  • title, subtitle, x_axis, y_axis, label, description и похожие поля с переводами имеют форму { "en": "...", "ru": "..." }.
  • subtitle в визуализациях может присутствовать в bucket; фронт может его использовать или игнорировать.
  • manifest.files является источником путей к payload-файлам.
  • manifest.hashes содержит SHA-256 payload-файлов и должен обновляться при изменении соответствующего JSON.

Список файлов 

manifest.json
catalog.json
leaderboard.json
details_matrix.json
drilldown/index.json
visualizations/radar.json
visualizations/scatter.json
visualizations/heatmap.json
visualizations/grouped_bars.json
visualizations/pareto.json
visualizations/performance.json
visualizations/robustness.json

Common types 

type LocalizedString = {
    en: string;
    ru: string;
};

type NullableNumber = number | null;
type MetricDirection = 'higher' | 'lower';
type MetricKey =
    'integral' | 'score' | 'f1' | 'recall' | 'precision' | 'accuracy' | 'fpr' | 'fnr' | 'latency_ms';

type LatencyMs = {
    p50: NullableNumber;
    p95: NullableNumber;
    p99: NullableNumber;
};

type GroupMetrics = {
    score: NullableNumber;
    f1: NullableNumber;
    fpr: NullableNumber;
    fnr: NullableNumber;
};

type FullMetrics = GroupMetrics & {
    recall: NullableNumber;
    precision: NullableNumber;
    accuracy: NullableNumber;
};

manifest.json 

type BucketFileKey =
    | 'catalog'
    | 'leaderboard'
    | 'details_matrix'
    | 'drilldown_index'
    | 'radar'
    | 'scatter'
    | 'heatmap'
    | 'grouped_bars'
    | 'pareto'
    | 'performance'
    | 'robustness';

type Manifest = {
    schema_version: '1.0.0';
    generated_at: string;
    snapshot_id: string;
    snapshot_date: string;
    is_ok: boolean;
    model_count: number;
    group_count: number;
    dataset_count: number;
    files: Record<BucketFileKey, string>;
    hashes: Record<BucketFileKey, string>;
};

{
    "schema_version": "1.0.0",
    "generated_at": "2026-07-10T19:17:38+00:00",
    "snapshot_id": "snapshot_2026-07-10",
    "snapshot_date": "2026-07-10",
    "is_ok": true,
    "model_count": 48,
    "group_count": 19,
    "dataset_count": 44,
    "files": {
        "catalog": "catalog.json",
        "leaderboard": "leaderboard.json",
        "details_matrix": "details_matrix.json",
        "drilldown_index": "drilldown/index.json",
        "radar": "visualizations/radar.json",
        "scatter": "visualizations/scatter.json",
        "heatmap": "visualizations/heatmap.json",
        "grouped_bars": "visualizations/grouped_bars.json",
        "pareto": "visualizations/pareto.json",
        "performance": "visualizations/performance.json",
        "robustness": "visualizations/robustness.json"
    },
    "hashes": {
        "catalog": "6cde39ef39563ec1db551620cdcbd6a8db4751b05a3d886c97453a382f0309fc",
        "leaderboard": "3e1f957fd2e3d994b200b7d3b35f6606c3fc56fad6c1fe8facac72a513b88cc0",
        "details_matrix": "ea9e89936cf59075e78c8f907b8a1b922f8ce1105a56669a829131e1ce56b4ba",
        "drilldown_index": "5cc2ceca5eeb38514a872b91fe301b1481109404547bbcc9a30622eb19b1a4da",
        "radar": "5ce3a62644f768feb3f52bd787993f6281eec935879d9fd04c10c3835028f249",
        "scatter": "f8c5b6cc91a2f586fbfb9a69bcc9b1472363c4cdda61c6501aff9abcc14b0756",
        "heatmap": "d2665f3f4df58ca2d05a138cee5186bf3a16d287d61c459e9b2ecc4e69914b1a",
        "grouped_bars": "678a531e2b1593de9e8bdd45f17d116aa4f5051d4e246f2abdcf634d5d86d7f4",
        "pareto": "0a1b2bad8addd2df35f86a90d4fce59a76f846d031b56278ef4fb029ff425535",
        "performance": "5cecdcb010454e7918250c526eb69b20199ba8317017d9b30938104b97fb77bf",
        "robustness": "362664c44b3acf2b5927de59fac7f8c3ed4492c6321351c21e68c139a250953f"
    }
}

catalog.json 

type BucketModel = {
    model_id: string;
    display_name: string;
    short_name: string;
    org: string;
    family: string;
    guardrail_type: 'llm-judge' | 'classifier' | 'unknown';
    license: string | null;
    params_b: NullableNumber;
    size_label: LocalizedString | null;
    languages: string[];
    note: LocalizedString | null;
    run_date: string;
    hf_url: string | null;
    eval_leak: boolean | null;
};

type BucketGroup = {
    group_id: string;
    label: LocalizedString;
    description: LocalizedString | null;
    what_it_tests: LocalizedString | null;
    languages: string[];
    dataset_ids: string[];
};

type BucketDataset = {
    dataset_id: string;
    group_id: string;
    label: LocalizedString;
    source_name: LocalizedString | null;
    split: LocalizedString | null;
    subset: LocalizedString | null;
    language: string | null;
    is_combined: boolean;
};

type Catalog = {
    schema_version: '1.0.0';
    models: BucketModel[];
    groups: BucketGroup[];
    datasets: BucketDataset[];
    metric_directions: Record<MetricKey, MetricDirection>;
    metric_labels: Partial<Record<MetricKey, LocalizedString>>;
};

{
    "schema_version": "1.0.0",
    "models": [
        {
            "model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
            "display_name": "Qwen3Guard-Gen-0.6B",
            "short_name": "Qwen3Guard-Gen-0.6B",
            "org": "Qwen",
            "family": "Qwen3",
            "guardrail_type": "llm-judge",
            "license": "Apache-2.0",
            "params_b": 0.6,
            "size_label": { "en": "0.6B", "ru": "0.6B" },
            "languages": ["multilingual", "en", "ru", "zh"],
            "note": {
                "en": "Small Qwen3Guard for low-latency; model_size marked as 0.8B in the run, actual checkpoint ~0.6B.",
                "ru": "Малый Qwen3Guard для low-latency; model_size в прогоне помечен 0.8B, фактический чекпойнт ~0.6B."
            },
            "run_date": "2026-06-06",
            "hf_url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-0.6B",
            "eval_leak": null
        }
    ],
    "groups": [
        {
            "group_id": "s_eval",
            "label": { "en": "S-Eval", "ru": "S-Eval" },
            "description": {
                "en": "S-Eval benchmark family.",
                "ru": "Семейство бенчмарков S-Eval."
            },
            "what_it_tests": {
                "en": "Recognition of unsafe intent and robustness to adversarial attacks.",
                "ru": "Распознавание небезопасного намерения и устойчивость к adversarial-атакам."
            },
            "languages": ["en", "ru"],
            "dataset_ids": ["s_eval_base_risk", "s_eval_attack_set"]
        }
    ],
    "datasets": [
        {
            "dataset_id": "s_eval_base_risk",
            "group_id": "s_eval",
            "label": { "en": "S-Eval (base risk)", "ru": "S-Eval (base risk)" },
            "source_name": null,
            "split": null,
            "subset": null,
            "language": "en",
            "is_combined": false
        }
    ],
    "metric_directions": {
        "integral": "higher",
        "score": "higher",
        "f1": "higher",
        "recall": "higher",
        "precision": "higher",
        "accuracy": "higher",
        "fpr": "lower",
        "fnr": "lower",
        "latency_ms": "lower"
    },
    "metric_labels": {
        "score": { "en": "Score", "ru": "Скор" },
        "f1": { "en": "F1", "ru": "F1" },
        "fpr": { "en": "FPR", "ru": "FPR" },
        "fnr": { "en": "FNR", "ru": "FNR" },
        "integral": { "en": "Integral score", "ru": "Интегральный скор" },
        "latency_ms": { "en": "Latency, ms", "ru": "Задержка, мс" }
    }
}

leaderboard.json 

type LeaderboardRow = {
    model_id: string;
    rank: number;
    run_id: string;
    run_date: string;
    integral: NullableNumber;
    min_group: NullableNumber;
    overall_fpr: NullableNumber;
    overall_fnr: NullableNumber;
    overall_f1: NullableNumber;
    latency_ms: LatencyMs;
    groups: Record<string, GroupMetrics>;
};

type Leaderboard = {
    schema_version: '1.0.0';
    rows: LeaderboardRow[];
};

{
    "schema_version": "1.0.0",
    "rows": [
        {
            "model_id": "alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f",
            "rank": 1,
            "run_id": "run_20260701_155019_4a9b279f",
            "run_date": "2026-07-01",
            "integral": 0.7605880393198141,
            "min_group": 0.15654499428022556,
            "overall_fpr": 0.21801750988077712,
            "overall_fnr": 0.12210970263223303,
            "overall_f1": 0.8224670573864103,
            "latency_ms": { "p50": 71.79, "p95": 102.57, "p99": 144.35 },
            "groups": {
                "s_eval": {
                    "score": 0.8761476877122706,
                    "f1": null,
                    "fpr": null,
                    "fnr": 0.11835
                }
            }
        }
    ]
}

details_matrix.json 

type GroupCell = FullMetrics & {
    model_id: string;
    group_id: string;
};

type DatasetCell = GroupCell & {
    dataset_id: string;
    sample_count: number | null;
};

type DetailsMatrix = {
    schema_version: '1.0.0';
    group_cells: GroupCell[];
    dataset_cells: DatasetCell[];
};

{
    "schema_version": "1.0.0",
    "group_cells": [
        {
            "model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
            "group_id": "s_eval",
            "score": 0.6505268251330913,
            "f1": null,
            "fpr": null,
            "fnr": 0.34645000000000004,
            "recall": null,
            "precision": null,
            "accuracy": null
        }
    ],
    "dataset_cells": [
        {
            "model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
            "group_id": "s_eval",
            "dataset_id": "s_eval_base_risk",
            "score": 0.698,
            "f1": null,
            "fpr": null,
            "fnr": 0.302,
            "recall": 0.698,
            "precision": null,
            "accuracy": null,
            "sample_count": 1000
        }
    ]
}

drilldown/index.json 

type DrilldownMetric = 'score' | 'f1' | 'fpr' | 'fnr';

type DrilldownHeatmapRow = {
    model_id: string;
    group_id: string;
    dataset_ids: string[];
    selected_metric: DrilldownMetric;
    metrics: GroupMetrics;
    description: LocalizedString | null;
    prompt_viewer_available: boolean;
    prompt_viewer_path: string | null;
};

type DrilldownIndex = {
    schema_version: '1.0.0';
    heatmap: DrilldownHeatmapRow[];
};

{
    "schema_version": "1.0.0",
    "heatmap": [
        {
            "model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
            "group_id": "s_eval",
            "dataset_ids": ["s_eval_base_risk", "s_eval_attack_set"],
            "selected_metric": "fnr",
            "metrics": {
                "score": 0.6505268251330913,
                "f1": null,
                "fpr": null,
                "fnr": 0.34645000000000004
            },
            "description": {
                "en": "Small Qwen3Guard for low-latency; model_size marked as 0.8B in the run, actual checkpoint ~0.6B.",
                "ru": "Малый Qwen3Guard для low-latency; model_size в прогоне помечен 0.8B, фактический чекпойнт ~0.6B."
            },
            "prompt_viewer_available": false,
            "prompt_viewer_path": null
        }
    ]
}

visualizations/radar.json 

type Radar = {
    schema_version: '1.0.0';
    title: LocalizedString;
    subtitle?: LocalizedString;
    model_ids: string[];
    group_ids: string[];
    default_model_ids: string[];
    values: Record<string, Record<string, NullableNumber>>;
    f1_values: Record<string, Record<string, NullableNumber>>;
};

values и f1_values индексируются как values[group_id][model_id].

{
    "schema_version": "1.0.0",
    "title": { "en": "Model Safety Profile", "ru": "Профиль безопасности модели" },
    "subtitle": { "en": "Group scores", "ru": "Групповые скоры" },
    "model_ids": ["qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1"],
    "group_ids": ["s_eval"],
    "default_model_ids": ["qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1"],
    "values": {
        "s_eval": {
            "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1": 0.6505268251330913
        }
    },
    "f1_values": {
        "s_eval": {
            "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1": null
        }
    }
}

visualizations/scatter.json 

type ScatterRowId = 'fpr' | 'fnr' | 'overall';

type ScatterPoint = {
    model_id: string;
    fpr: NullableNumber;
    fnr: NullableNumber;
    tooltip: string | null;
};

type Scatter = {
    schema_version: '1.0.0';
    title: LocalizedString;
    x_axis: LocalizedString;
    y_axis: LocalizedString;
    points: ScatterPoint[];
    row_ids: ScatterRowId[];
    values: Record<ScatterRowId, Record<string, NullableNumber>>;
};

values индексируется как values[row_id][model_id].

{
    "schema_version": "1.0.0",
    "title": { "en": "FPR vs FNR (overall)", "ru": "FPR vs FNR (overall)" },
    "x_axis": { "en": "FPR", "ru": "FPR" },
    "y_axis": { "en": "FNR", "ru": "FNR" },
    "points": [
        {
            "model_id": "alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f",
            "fpr": 0.21801750988077712,
            "fnr": 0.12210970263223303,
            "tooltip": null
        }
    ],
    "row_ids": ["fpr", "fnr", "overall"],
    "values": {
        "fpr": {
            "alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f": 0.21801750988077712
        },
        "fnr": {
            "alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f": 0.12210970263223303
        },
        "overall": {
            "alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f": 0.7605880393198141
        }
    }
}

visualizations/heatmap.json 

type Heatmap = {
    schema_version: '1.0.0';
    title: LocalizedString;
    subtitle?: LocalizedString;
    metric: 'score' | 'f1' | 'fpr' | 'fnr';
    row_ids: string[];
    column_ids: string[];
    values: Record<string, Record<string, NullableNumber>>;
};

values индексируется как values[row_id][model_id]. В текущем snapshot metric равен "fnr".

{
    "schema_version": "1.0.0",
    "title": { "en": "Heatmap FNR", "ru": "Тепловая карта FNR" },
    "subtitle": { "en": "Dataset x model", "ru": "Датасет x модель" },
    "metric": "fnr",
    "row_ids": ["aegis_2_0_prompt"],
    "column_ids": ["qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1"],
    "values": {
        "aegis_2_0_prompt": {
            "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1": 0.05854579792256846
        }
    }
}

visualizations/grouped_bars.json 

type GroupedBarsModel = {
    model_id: string;
    groups: Record<
        string,
        {
            fnr: NullableNumber;
            fpr: NullableNumber;
            f1: NullableNumber;
        }
    >;
};

type GroupedBars = {
    schema_version: '1.0.0';
    title: LocalizedString;
    subtitle?: LocalizedString;
    group_ids: string[];
    models: GroupedBarsModel[];
};

{
    "schema_version": "1.0.0",
    "title": { "en": "Grouped bars", "ru": "Групповые столбцы" },
    "subtitle": {
        "en": "FNR / FPR / F1 by benchmark group",
        "ru": "FNR / FPR / F1 по группам"
    },
    "group_ids": ["s_eval", "aegis_2_0"],
    "models": [
        {
            "model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
            "groups": {
                "s_eval": { "fnr": 0.34645000000000004, "fpr": null, "f1": null },
                "aegis_2_0": {
                    "fnr": 0.08384142688006596,
                    "fpr": 0.2825050061521388,
                    "f1": 0.8346634240315878
                }
            }
        }
    ]
}

visualizations/pareto.json 

type ParetoModel = {
    latency_ms: LatencyMs;
    integral: NullableNumber;
    fpr: NullableNumber;
};

type Pareto = {
    schema_version: '1.0.0';
    title: LocalizedString;
    subtitle?: LocalizedString;
    x_axis: LocalizedString;
    y_axis: LocalizedString;
    by_model_id: Record<string, ParetoModel>;
};

{
    "schema_version": "1.0.0",
    "title": { "en": "Pareto: latency vs quality", "ru": "Парето: задержка vs качество" },
    "subtitle": {
        "en": "p95 latency vs integral score",
        "ru": "p95 задержка vs интегральный скор"
    },
    "x_axis": { "en": "Latency p95, ms", "ru": "Задержка p95, мс" },
    "y_axis": { "en": "Integral score", "ru": "Интегральный скор" },
    "by_model_id": {
        "alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f": {
            "latency_ms": { "p50": 71.79, "p95": 102.57, "p99": 144.35 },
            "integral": 0.7605880393198141,
            "fpr": 0.21801750988077712
        }
    }
}

visualizations/performance.json 

type PerformanceRow = {
    model_id: string;
    rank: number;
    latency_ms: LatencyMs;
    integral: NullableNumber;
};

type Performance = {
    schema_version: '1.0.0';
    title: LocalizedString;
    rows: PerformanceRow[];
};

{
    "schema_version": "1.0.0",
    "title": { "en": "Latency / performance", "ru": "Задержка / производительность" },
    "rows": [
        {
            "model_id": "alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f",
            "rank": 1,
            "latency_ms": { "p50": 71.79, "p95": 102.57, "p99": 144.35 },
            "integral": 0.7605880393198141
        }
    ]
}

visualizations/robustness.json 

type RobustnessRow = {
    model_id: string;
    score_real: NullableNumber;
    score_robust: NullableNumber;
    delta_score: NullableNumber;
    fnr_real: NullableNumber;
    fnr_robust: NullableNumber;
    delta_fnr: NullableNumber;
    category_label: LocalizedString;
};

type Robustness = {
    schema_version: '1.0.0';
    title: LocalizedString;
    rows: RobustnessRow[];
};

{
    "schema_version": "1.0.0",
    "title": { "en": "Robustness comparison", "ru": "Сравнение robustness" },
    "rows": [
        {
            "model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
            "score_real": 0.6313559322033898,
            "score_robust": 0.8121144139091419,
            "delta_score": 0.18075848170575204,
            "fnr_real": 0.13623188405797101,
            "fnr_robust": 0.21190130624092887,
            "delta_fnr": 0.07566942218295786,
            "category_label": {
                "en": "Qwen3Guard-Gen-0.6B",
                "ru": "Qwen3Guard-Gen-0.6B"
            }
        }
    ]
}

Проверочные инварианты

  • catalog.models.length === manifest.model_count.
  • catalog.groups.length === manifest.group_count.
  • catalog.datasets.length === manifest.dataset_count.
  • leaderboard.rows[].model_id должен ссылаться на catalog.models[].model_id.
  • details_matrix.group_cells[].model_id и details_matrix.dataset_cells[].model_id должны ссылаться на catalog.
  • details_matrix.group_cells[].group_id должен ссылаться на catalog.groups[].group_id.
  • details_matrix.dataset_cells[].dataset_id должен ссылаться на catalog.datasets[].dataset_id.
  • Для текущего snapshot ожидается 48 уникальных model_id в catalog и leaderboard.
  • Для текущего snapshot ожидается 912 уникальных пар model_id + group_id в details_matrix.group_cells.
  • Для текущего snapshot ожидается 2112 уникальных троек model_id + group_id + dataset_id в details_matrix.dataset_cells.