docs/backend-data-contract_en.md

# Backend bucket data contract

This document defines the current data format in the private HF bucket that the frontend can
integrate with. It is a contract for JSON shapes, relationships between entities, and nullable
fields. Formula correctness, ranking logic, and business calculations remain the backend's
responsibility.

Verified snapshot:

```json
{
	"schema_version": "1.0.0",
	"snapshot_id": "snapshot_2026-07-10",
	"snapshot_date": "2026-07-10",
	"generated_at": "2026-07-10T19:17:38+00:00",
	"model_count": 48,
	"group_count": 19,
	"dataset_count": 44
}
```

## General rules

- All files use `schema_version: "1.0.0"`.
- All missing numeric metrics are represented as `null`; they are not omitted.
- `model_id`, `group_id`, and `dataset_id` are string identifiers.
- In the current bucket, `model_id` includes the run suffix, for example
  `qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1`.
- Display order comes from the `rows`, `model_ids`, `group_ids`, `row_ids`, and `column_ids`
  arrays.
- `title`, `subtitle`, `x_axis`, `y_axis`, `label`, `description`, and similar translated
  fields have the shape `{ "en": "...", "ru": "..." }`.
- `subtitle` may be present in visualization payloads; the frontend may use or ignore it.
- `manifest.files` is the source of paths to payload files.
- `manifest.hashes` contains SHA-256 hashes of payload files and must be updated whenever the
  corresponding JSON changes.

## File list

```text
manifest.json
catalog.json
leaderboard.json
details_matrix.json
drilldown/index.json
visualizations/radar.json
visualizations/scatter.json
visualizations/heatmap.json
visualizations/grouped_bars.json
visualizations/pareto.json
visualizations/performance.json
visualizations/robustness.json
```

## Common types

```ts
type LocalizedString = {
	en: string;
	ru: string;
};

type NullableNumber = number | null;
type MetricDirection = 'higher' | 'lower';
type MetricKey =
	'integral' | 'score' | 'f1' | 'recall' | 'precision' | 'accuracy' | 'fpr' | 'fnr' | 'latency_ms';

type LatencyMs = {
	p50: NullableNumber;
	p95: NullableNumber;
	p99: NullableNumber;
};

type GroupMetrics = {
	score: NullableNumber;
	f1: NullableNumber;
	fpr: NullableNumber;
	fnr: NullableNumber;
};

type FullMetrics = GroupMetrics & {
	recall: NullableNumber;
	precision: NullableNumber;
	accuracy: NullableNumber;
};
```

## manifest.json

```ts
type BucketFileKey =
	| 'catalog'
	| 'leaderboard'
	| 'details_matrix'
	| 'drilldown_index'
	| 'radar'
	| 'scatter'
	| 'heatmap'
	| 'grouped_bars'
	| 'pareto'
	| 'performance'
	| 'robustness';

type Manifest = {
	schema_version: '1.0.0';
	generated_at: string;
	snapshot_id: string;
	snapshot_date: string;
	is_ok: boolean;
	model_count: number;
	group_count: number;
	dataset_count: number;
	files: Record<BucketFileKey, string>;
	hashes: Record<BucketFileKey, string>;
};
```

```json
{
	"schema_version": "1.0.0",
	"generated_at": "2026-07-10T19:17:38+00:00",
	"snapshot_id": "snapshot_2026-07-10",
	"snapshot_date": "2026-07-10",
	"is_ok": true,
	"model_count": 48,
	"group_count": 19,
	"dataset_count": 44,
	"files": {
		"catalog": "catalog.json",
		"leaderboard": "leaderboard.json",
		"details_matrix": "details_matrix.json",
		"drilldown_index": "drilldown/index.json",
		"radar": "visualizations/radar.json",
		"scatter": "visualizations/scatter.json",
		"heatmap": "visualizations/heatmap.json",
		"grouped_bars": "visualizations/grouped_bars.json",
		"pareto": "visualizations/pareto.json",
		"performance": "visualizations/performance.json",
		"robustness": "visualizations/robustness.json"
	},
	"hashes": {
		"catalog": "6cde39ef39563ec1db551620cdcbd6a8db4751b05a3d886c97453a382f0309fc",
		"leaderboard": "3e1f957fd2e3d994b200b7d3b35f6606c3fc56fad6c1fe8facac72a513b88cc0",
		"details_matrix": "ea9e89936cf59075e78c8f907b8a1b922f8ce1105a56669a829131e1ce56b4ba",
		"drilldown_index": "5cc2ceca5eeb38514a872b91fe301b1481109404547bbcc9a30622eb19b1a4da",
		"radar": "5ce3a62644f768feb3f52bd787993f6281eec935879d9fd04c10c3835028f249",
		"scatter": "f8c5b6cc91a2f586fbfb9a69bcc9b1472363c4cdda61c6501aff9abcc14b0756",
		"heatmap": "d2665f3f4df58ca2d05a138cee5186bf3a16d287d61c459e9b2ecc4e69914b1a",
		"grouped_bars": "678a531e2b1593de9e8bdd45f17d116aa4f5051d4e246f2abdcf634d5d86d7f4",
		"pareto": "0a1b2bad8addd2df35f86a90d4fce59a76f846d031b56278ef4fb029ff425535",
		"performance": "5cecdcb010454e7918250c526eb69b20199ba8317017d9b30938104b97fb77bf",
		"robustness": "362664c44b3acf2b5927de59fac7f8c3ed4492c6321351c21e68c139a250953f"
	}
}
```

## catalog.json

```ts
type BucketModel = {
	model_id: string;
	display_name: string;
	short_name: string;
	org: string;
	family: string;
	guardrail_type: 'llm-judge' | 'classifier' | 'unknown';
	license: string | null;
	params_b: NullableNumber;
	size_label: LocalizedString | null;
	languages: string[];
	note: LocalizedString | null;
	run_date: string;
	hf_url: string | null;
	eval_leak: boolean | null;
};

type BucketGroup = {
	group_id: string;
	label: LocalizedString;
	description: LocalizedString | null;
	what_it_tests: LocalizedString | null;
	languages: string[];
	dataset_ids: string[];
};

type BucketDataset = {
	dataset_id: string;
	group_id: string;
	label: LocalizedString;
	source_name: LocalizedString | null;
	split: LocalizedString | null;
	subset: LocalizedString | null;
	language: string | null;
	is_combined: boolean;
};

type Catalog = {
	schema_version: '1.0.0';
	models: BucketModel[];
	groups: BucketGroup[];
	datasets: BucketDataset[];
	metric_directions: Record<MetricKey, MetricDirection>;
	metric_labels: Partial<Record<MetricKey, LocalizedString>>;
};
```

```json
{
	"schema_version": "1.0.0",
	"models": [
		{
			"model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
			"display_name": "Qwen3Guard-Gen-0.6B",
			"short_name": "Qwen3Guard-Gen-0.6B",
			"org": "Qwen",
			"family": "Qwen3",
			"guardrail_type": "llm-judge",
			"license": "Apache-2.0",
			"params_b": 0.6,
			"size_label": { "en": "0.6B", "ru": "0.6B" },
			"languages": ["multilingual", "en", "ru", "zh"],
			"note": {
				"en": "Small Qwen3Guard for low-latency; model_size marked as 0.8B in the run, actual checkpoint ~0.6B.",
				"ru": "Малый Qwen3Guard для low-latency; model_size в прогоне помечен 0.8B, фактический чекпойнт ~0.6B."
			},
			"run_date": "2026-06-06",
			"hf_url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-0.6B",
			"eval_leak": null
		}
	],
	"groups": [
		{
			"group_id": "s_eval",
			"label": { "en": "S-Eval", "ru": "S-Eval" },
			"description": {
				"en": "S-Eval benchmark family.",
				"ru": "Семейство бенчмарков S-Eval."
			},
			"what_it_tests": {
				"en": "Recognition of unsafe intent and robustness to adversarial attacks.",
				"ru": "Распознавание небезопасного намерения и устойчивость к adversarial-атакам."
			},
			"languages": ["en", "ru"],
			"dataset_ids": ["s_eval_base_risk", "s_eval_attack_set"]
		}
	],
	"datasets": [
		{
			"dataset_id": "s_eval_base_risk",
			"group_id": "s_eval",
			"label": { "en": "S-Eval (base risk)", "ru": "S-Eval (base risk)" },
			"source_name": null,
			"split": null,
			"subset": null,
			"language": "en",
			"is_combined": false
		}
	],
	"metric_directions": {
		"integral": "higher",
		"score": "higher",
		"f1": "higher",
		"recall": "higher",
		"precision": "higher",
		"accuracy": "higher",
		"fpr": "lower",
		"fnr": "lower",
		"latency_ms": "lower"
	},
	"metric_labels": {
		"score": { "en": "Score", "ru": "Скор" },
		"f1": { "en": "F1", "ru": "F1" },
		"fpr": { "en": "FPR", "ru": "FPR" },
		"fnr": { "en": "FNR", "ru": "FNR" },
		"integral": { "en": "Integral score", "ru": "Интегральный скор" },
		"latency_ms": { "en": "Latency, ms", "ru": "Задержка, мс" }
	}
}
```

## leaderboard.json

```ts
type LeaderboardRow = {
	model_id: string;
	rank: number;
	run_id: string;
	run_date: string;
	integral: NullableNumber;
	min_group: NullableNumber;
	overall_fpr: NullableNumber;
	overall_fnr: NullableNumber;
	overall_f1: NullableNumber;
	latency_ms: LatencyMs;
	groups: Record<string, GroupMetrics>;
};

type Leaderboard = {
	schema_version: '1.0.0';
	rows: LeaderboardRow[];
};
```

```json
{
	"schema_version": "1.0.0",
	"rows": [
		{
			"model_id": "alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f",
			"rank": 1,
			"run_id": "run_20260701_155019_4a9b279f",
			"run_date": "2026-07-01",
			"integral": 0.7605880393198141,
			"min_group": 0.15654499428022556,
			"overall_fpr": 0.21801750988077712,
			"overall_fnr": 0.12210970263223303,
			"overall_f1": 0.8224670573864103,
			"latency_ms": { "p50": 71.79, "p95": 102.57, "p99": 144.35 },
			"groups": {
				"s_eval": {
					"score": 0.8761476877122706,
					"f1": null,
					"fpr": null,
					"fnr": 0.11835
				}
			}
		}
	]
}
```

## details_matrix.json

```ts
type GroupCell = FullMetrics & {
	model_id: string;
	group_id: string;
};

type DatasetCell = GroupCell & {
	dataset_id: string;
	sample_count: number | null;
};

type DetailsMatrix = {
	schema_version: '1.0.0';
	group_cells: GroupCell[];
	dataset_cells: DatasetCell[];
};
```

```json
{
	"schema_version": "1.0.0",
	"group_cells": [
		{
			"model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
			"group_id": "s_eval",
			"score": 0.6505268251330913,
			"f1": null,
			"fpr": null,
			"fnr": 0.34645000000000004,
			"recall": null,
			"precision": null,
			"accuracy": null
		}
	],
	"dataset_cells": [
		{
			"model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
			"group_id": "s_eval",
			"dataset_id": "s_eval_base_risk",
			"score": 0.698,
			"f1": null,
			"fpr": null,
			"fnr": 0.302,
			"recall": 0.698,
			"precision": null,
			"accuracy": null,
			"sample_count": 1000
		}
	]
}
```

## drilldown/index.json

```ts
type DrilldownMetric = 'score' | 'f1' | 'fpr' | 'fnr';

type DrilldownHeatmapRow = {
	model_id: string;
	group_id: string;
	dataset_ids: string[];
	selected_metric: DrilldownMetric;
	metrics: GroupMetrics;
	description: LocalizedString | null;
	prompt_viewer_available: boolean;
	prompt_viewer_path: string | null;
};

type DrilldownIndex = {
	schema_version: '1.0.0';
	heatmap: DrilldownHeatmapRow[];
};
```

```json
{
	"schema_version": "1.0.0",
	"heatmap": [
		{
			"model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
			"group_id": "s_eval",
			"dataset_ids": ["s_eval_base_risk", "s_eval_attack_set"],
			"selected_metric": "fnr",
			"metrics": {
				"score": 0.6505268251330913,
				"f1": null,
				"fpr": null,
				"fnr": 0.34645000000000004
			},
			"description": {
				"en": "Small Qwen3Guard for low-latency; model_size marked as 0.8B in the run, actual checkpoint ~0.6B.",
				"ru": "Малый Qwen3Guard для low-latency; model_size в прогоне помечен 0.8B, фактический чекпойнт ~0.6B."
			},
			"prompt_viewer_available": false,
			"prompt_viewer_path": null
		}
	]
}
```

## visualizations/radar.json

```ts
type Radar = {
	schema_version: '1.0.0';
	title: LocalizedString;
	subtitle?: LocalizedString;
	model_ids: string[];
	group_ids: string[];
	default_model_ids: string[];
	values: Record<string, Record<string, NullableNumber>>;
	f1_values: Record<string, Record<string, NullableNumber>>;
};
```

`values` and `f1_values` are indexed as `values[group_id][model_id]`.

```json
{
	"schema_version": "1.0.0",
	"title": { "en": "Model Safety Profile", "ru": "Профиль безопасности модели" },
	"subtitle": { "en": "Group scores", "ru": "Групповые скоры" },
	"model_ids": ["qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1"],
	"group_ids": ["s_eval"],
	"default_model_ids": ["qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1"],
	"values": {
		"s_eval": {
			"qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1": 0.6505268251330913
		}
	},
	"f1_values": {
		"s_eval": {
			"qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1": null
		}
	}
}
```

## visualizations/scatter.json

```ts
type ScatterRowId = 'fpr' | 'fnr' | 'overall';

type ScatterPoint = {
	model_id: string;
	fpr: NullableNumber;
	fnr: NullableNumber;
	tooltip: string | null;
};

type Scatter = {
	schema_version: '1.0.0';
	title: LocalizedString;
	x_axis: LocalizedString;
	y_axis: LocalizedString;
	points: ScatterPoint[];
	row_ids: ScatterRowId[];
	values: Record<ScatterRowId, Record<string, NullableNumber>>;
};
```

`values` is indexed as `values[row_id][model_id]`.

```json
{
	"schema_version": "1.0.0",
	"title": { "en": "FPR vs FNR (overall)", "ru": "FPR vs FNR (overall)" },
	"x_axis": { "en": "FPR", "ru": "FPR" },
	"y_axis": { "en": "FNR", "ru": "FNR" },
	"points": [
		{
			"model_id": "alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f",
			"fpr": 0.21801750988077712,
			"fnr": 0.12210970263223303,
			"tooltip": null
		}
	],
	"row_ids": ["fpr", "fnr", "overall"],
	"values": {
		"fpr": {
			"alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f": 0.21801750988077712
		},
		"fnr": {
			"alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f": 0.12210970263223303
		},
		"overall": {
			"alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f": 0.7605880393198141
		}
	}
}
```

## visualizations/heatmap.json

```ts
type Heatmap = {
	schema_version: '1.0.0';
	title: LocalizedString;
	subtitle?: LocalizedString;
	metric: 'score' | 'f1' | 'fpr' | 'fnr';
	row_ids: string[];
	column_ids: string[];
	values: Record<string, Record<string, NullableNumber>>;
};
```

`values` is indexed as `values[row_id][model_id]`. In the current snapshot, `metric` is
`"fnr"`.

```json
{
	"schema_version": "1.0.0",
	"title": { "en": "Heatmap FNR", "ru": "Тепловая карта FNR" },
	"subtitle": { "en": "Dataset x model", "ru": "Датасет x модель" },
	"metric": "fnr",
	"row_ids": ["aegis_2_0_prompt"],
	"column_ids": ["qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1"],
	"values": {
		"aegis_2_0_prompt": {
			"qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1": 0.05854579792256846
		}
	}
}
```

## visualizations/grouped_bars.json

```ts
type GroupedBarsModel = {
	model_id: string;
	groups: Record<
		string,
		{
			fnr: NullableNumber;
			fpr: NullableNumber;
			f1: NullableNumber;
		}
	>;
};

type GroupedBars = {
	schema_version: '1.0.0';
	title: LocalizedString;
	subtitle?: LocalizedString;
	group_ids: string[];
	models: GroupedBarsModel[];
};
```

```json
{
	"schema_version": "1.0.0",
	"title": { "en": "Grouped bars", "ru": "Групповые столбцы" },
	"subtitle": {
		"en": "FNR / FPR / F1 by benchmark group",
		"ru": "FNR / FPR / F1 по группам"
	},
	"group_ids": ["s_eval", "aegis_2_0"],
	"models": [
		{
			"model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
			"groups": {
				"s_eval": { "fnr": 0.34645000000000004, "fpr": null, "f1": null },
				"aegis_2_0": {
					"fnr": 0.08384142688006596,
					"fpr": 0.2825050061521388,
					"f1": 0.8346634240315878
				}
			}
		}
	]
}
```

## visualizations/pareto.json

```ts
type ParetoModel = {
	latency_ms: LatencyMs;
	integral: NullableNumber;
	fpr: NullableNumber;
};

type Pareto = {
	schema_version: '1.0.0';
	title: LocalizedString;
	subtitle?: LocalizedString;
	x_axis: LocalizedString;
	y_axis: LocalizedString;
	by_model_id: Record<string, ParetoModel>;
};
```

```json
{
	"schema_version": "1.0.0",
	"title": { "en": "Pareto: latency vs quality", "ru": "Парето: задержка vs качество" },
	"subtitle": {
		"en": "p95 latency vs integral score",
		"ru": "p95 задержка vs интегральный скор"
	},
	"x_axis": { "en": "Latency p95, ms", "ru": "Задержка p95, мс" },
	"y_axis": { "en": "Integral score", "ru": "Интегральный скор" },
	"by_model_id": {
		"alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f": {
			"latency_ms": { "p50": 71.79, "p95": 102.57, "p99": 144.35 },
			"integral": 0.7605880393198141,
			"fpr": 0.21801750988077712
		}
	}
}
```

## visualizations/performance.json

```ts
type PerformanceRow = {
	model_id: string;
	rank: number;
	latency_ms: LatencyMs;
	integral: NullableNumber;
};

type Performance = {
	schema_version: '1.0.0';
	title: LocalizedString;
	rows: PerformanceRow[];
};
```

```json
{
	"schema_version": "1.0.0",
	"title": { "en": "Latency / performance", "ru": "Задержка / производительность" },
	"rows": [
		{
			"model_id": "alibaba_aaig_yufeng_xguard_reason_8b__run_20260701_155019_4a9b279f",
			"rank": 1,
			"latency_ms": { "p50": 71.79, "p95": 102.57, "p99": 144.35 },
			"integral": 0.7605880393198141
		}
	]
}
```

## visualizations/robustness.json

```ts
type RobustnessRow = {
	model_id: string;
	score_real: NullableNumber;
	score_robust: NullableNumber;
	delta_score: NullableNumber;
	fnr_real: NullableNumber;
	fnr_robust: NullableNumber;
	delta_fnr: NullableNumber;
	category_label: LocalizedString;
};

type Robustness = {
	schema_version: '1.0.0';
	title: LocalizedString;
	rows: RobustnessRow[];
};
```

```json
{
	"schema_version": "1.0.0",
	"title": { "en": "Robustness comparison", "ru": "Сравнение robustness" },
	"rows": [
		{
			"model_id": "qwen_qwen3guard_gen_0_6b__run_20260606_005401_a4794dd1",
			"score_real": 0.6313559322033898,
			"score_robust": 0.8121144139091419,
			"delta_score": 0.18075848170575204,
			"fnr_real": 0.13623188405797101,
			"fnr_robust": 0.21190130624092887,
			"delta_fnr": 0.07566942218295786,
			"category_label": {
				"en": "Qwen3Guard-Gen-0.6B",
				"ru": "Qwen3Guard-Gen-0.6B"
			}
		}
	]
}
```

## Validation invariants

- `catalog.models.length === manifest.model_count`.
- `catalog.groups.length === manifest.group_count`.
- `catalog.datasets.length === manifest.dataset_count`.
- `leaderboard.rows[].model_id` must reference `catalog.models[].model_id`.
- `details_matrix.group_cells[].model_id` and `details_matrix.dataset_cells[].model_id` must
  reference the catalog.
- `details_matrix.group_cells[].group_id` must reference `catalog.groups[].group_id`.
- `details_matrix.dataset_cells[].dataset_id` must reference `catalog.datasets[].dataset_id`.
- The current snapshot is expected to contain 48 unique `model_id` values in `catalog` and
  `leaderboard`.
- The current snapshot is expected to contain 912 unique `model_id + group_id` pairs in
  `details_matrix.group_cells`.
- The current snapshot is expected to contain 2112 unique `model_id + group_id + dataset_id`
  triples in `details_matrix.dataset_cells`.