Slayer1B-Benchmarks / results.schema.json
Maggio33's picture
Benchmarks: first results bench50-v0.3.1 (results 1cf2b8bb, page b71d0c71)
2d3d73f verified
Raw History Blame Contribute Delete
26.5 kB
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "urn:fabryka-ai:benchmark-results:2:protocol-0.3.1",
"type": "object",
"properties": {
"schema_version": {
"const": 2
},
"protocol_id": {
"enum": [
"bench50-v0.3.1-panel200",
"bench50-v0.3.1-fullB"
]
},
"run_id": {
"type": "string",
"minLength": 1
},
"models": {
"type": "array",
"items": {
"type": "object",
"properties": {
"model_id": {
"type": "string",
"minLength": 1
},
"repo": {
"type": "string",
"minLength": 1
},
"revision": {
"type": [
"string",
"null"
]
},
"weights_sha256": {
"anyOf": [
{
"type": "string",
"pattern": "^[0-9a-f]{64}$"
},
{
"type": "null"
}
]
}
},
"required": [
"model_id",
"repo",
"revision",
"weights_sha256"
],
"additionalProperties": false
}
},
"tasks": {
"type": "array",
"items": {
"type": "object",
"properties": {
"task_id": {
"type": "string",
"pattern": "^T(0[1-9]|[1-4][0-9]|50)$"
},
"task": {
"type": "string",
"minLength": 1
},
"family": {
"type": "string",
"minLength": 1
},
"language": {
"enum": [
"PL",
"EN"
]
},
"split": {
"type": "string",
"minLength": 1
}
},
"required": [
"task_id",
"task",
"family",
"language",
"split"
],
"additionalProperties": false
}
},
"observations": {
"type": "array",
"items": {
"$ref": "#/$defs/observation"
}
},
"contrasts": {
"type": "array",
"items": {
"$ref": "#/$defs/contrast"
}
},
"provenance": {
"type": "object",
"properties": {
"inventory_commit": {
"type": "string",
"pattern": "^[0-9a-f]{40}$"
},
"kind": {
"enum": [
"measurement",
"contract_example"
]
},
"sources": {
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"properties": {
"run_id": {
"type": "string",
"minLength": 1
},
"results_sha256": {
"type": "string",
"pattern": "^[0-9a-f]{64}$"
}
},
"required": [
"run_id",
"results_sha256"
],
"additionalProperties": false
}
},
"qualification_sha256": {
"type": "string",
"pattern": "^[0-9a-f]{64}$"
},
"contamination_sha256": {
"type": "string",
"pattern": "^[0-9a-f]{64}$"
},
"references_sha256": {
"type": "string",
"pattern": "^[0-9a-f]{64}$"
},
"superseded": {
"type": "array",
"items": {
"type": "object",
"properties": {
"model_id": {
"type": "string",
"minLength": 1
},
"task_id": {
"type": "string",
"pattern": "^T(0[1-9]|[1-4][0-9]|50)$"
},
"by_results_sha256": {
"type": "string",
"pattern": "^[0-9a-f]{64}$"
},
"replaced": {
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"properties": {
"state": {
"enum": [
"not_run",
"unsupported",
"blocked_access",
"blocked_runtime",
"blocked_security",
"blocked_judge"
]
},
"results_sha256": {
"type": "string",
"pattern": "^[0-9a-f]{64}$"
}
},
"required": [
"state",
"results_sha256"
],
"additionalProperties": false
}
}
},
"required": [
"model_id",
"task_id",
"by_results_sha256",
"replaced"
],
"additionalProperties": false
}
},
"contrasts_sha256": {
"type": "string",
"pattern": "^[0-9a-f]{64}$"
},
"amendments": {
"type": "array",
"items": {
"type": "object",
"properties": {
"model_id": {
"type": "string",
"minLength": 1
},
"task_id": {
"type": "string",
"pattern": "^T(0[1-9]|[1-4][0-9]|50)$"
},
"code": {
"type": "string",
"minLength": 1
},
"base_items_sha256": {
"type": "string",
"pattern": "^[0-9a-f]{64}$"
},
"artifact_sha256": {
"type": "string",
"pattern": "^[0-9a-f]{64}$"
}
},
"required": [
"model_id",
"task_id",
"code",
"base_items_sha256",
"artifact_sha256"
],
"additionalProperties": false
}
}
},
"required": [
"inventory_commit",
"kind"
],
"additionalProperties": false,
"allOf": [
{
"if": {
"properties": {
"kind": {
"const": "measurement"
}
}
},
"then": {
"required": [
"sources",
"qualification_sha256",
"contamination_sha256",
"references_sha256"
]
}
}
]
}
},
"required": [
"schema_version",
"protocol_id",
"run_id",
"models",
"tasks",
"observations",
"contrasts",
"provenance"
],
"additionalProperties": false,
"$defs": {
"ci95": {
"type": "object",
"properties": {
"lower": {
"type": "number"
},
"upper": {
"type": "number"
},
"level": {
"const": 0.95
},
"method": {
"enum": [
"paired_bootstrap",
"paired_cluster_bootstrap"
]
},
"replicates": {
"const": 10000
},
"seed": {
"const": 20261028
}
},
"required": [
"lower",
"upper",
"level",
"method",
"replicates",
"seed"
],
"additionalProperties": false
},
"observation": {
"type": "object",
"properties": {
"task_id": {
"type": "string",
"pattern": "^T(0[1-9]|[1-4][0-9]|50)$"
},
"model_id": {
"type": "string",
"minLength": 1
},
"protocol_id": {
"enum": [
"bench50-v0.3.1-panel200",
"bench50-v0.3.1-fullB"
]
},
"metric": {
"type": "string",
"minLength": 1
},
"unit": {
"enum": [
"percent",
"score",
"byte_ppl"
]
},
"higher_is_better": {
"type": "boolean"
},
"n_population": {
"type": [
"integer",
"null"
],
"minimum": 0
},
"n_selected": {
"type": [
"integer",
"null"
],
"minimum": 0
},
"n_scored": {
"type": "integer",
"minimum": 0
},
"state": {
"enum": [
"ok",
"not_run",
"blocked_access",
"blocked_runtime",
"blocked_security",
"blocked_judge",
"unsupported",
"failed",
"skipped"
]
},
"value": {
"type": [
"number",
"null"
]
},
"ci95": {
"anyOf": [
{
"type": "object",
"properties": {
"lower": {
"type": "number"
},
"upper": {
"type": "number"
},
"level": {
"const": 0.95
},
"method": {
"enum": [
"paired_bootstrap",
"paired_cluster_bootstrap"
]
},
"replicates": {
"const": 10000
},
"seed": {
"const": 20261028
}
},
"required": [
"lower",
"upper",
"level",
"method",
"replicates",
"seed"
],
"additionalProperties": false
},
{
"type": "null"
}
]
},
"source_id_digest": {
"anyOf": [
{
"type": "string",
"pattern": "^[0-9a-f]{64}$"
},
{
"type": "null"
}
]
},
"results_sha256": {
"anyOf": [
{
"type": "string",
"pattern": "^[0-9a-f]{64}$"
},
{
"type": "null"
}
]
},
"runner_sha256": {
"anyOf": [
{
"type": "string",
"pattern": "^[0-9a-f]{64}$"
},
{
"type": "null"
}
]
},
"reason": {
"type": [
"string",
"null"
],
"maxLength": 200
},
"contamination": {
"enum": [
"clean",
"flagged",
"partially_scanned",
"unscanned",
"unknown"
]
},
"contamination_receipt_sha256": {
"type": [
"string",
"null"
],
"pattern": "^[0-9a-f]{64}$"
},
"value_hit_free": {
"type": [
"number",
"null"
]
},
"n_hit_free": {
"type": [
"integer",
"null"
],
"minimum": 0
},
"hit_rate": {
"type": [
"number",
"null"
],
"minimum": 0,
"maximum": 1
},
"reference": {
"type": [
"object",
"null"
],
"properties": {
"chance_value": {
"type": [
"number",
"null"
],
"description": "expected metric of uniform random choice on the frozen sample (mean 1/n_options for accuracy); null if not defined"
},
"majority_value": {
"type": [
"number",
"null"
],
"description": "same metric for always predicting the majority gold class of the frozen sample; null if classes are not fixed"
},
"majority_prediction_share": {
"type": [
"number",
"null"
],
"minimum": 0,
"maximum": 1,
"description": "share of the model predictions falling on its most frequent predicted class"
},
"constant_prediction": {
"type": [
"boolean",
"null"
],
"description": "true when the model predicts one class for every scored item"
}
},
"required": [
"chance_value",
"majority_value",
"majority_prediction_share",
"constant_prediction"
],
"additionalProperties": false
},
"value_hit_free_answer_fields": {
"type": [
"number",
"null"
]
},
"n_hit_free_answer_fields": {
"type": [
"integer",
"null"
],
"minimum": 0
},
"hit_rate_answer_fields": {
"type": [
"number",
"null"
],
"minimum": 0,
"maximum": 1
},
"ci95_hit_free": {
"anyOf": [
{
"type": "object",
"properties": {
"lower": {
"type": "number"
},
"upper": {
"type": "number"
},
"level": {
"const": 0.95
},
"method": {
"enum": [
"paired_bootstrap",
"paired_cluster_bootstrap"
]
},
"replicates": {
"const": 10000
},
"seed": {
"const": 20261028
}
},
"required": [
"lower",
"upper",
"level",
"method",
"replicates",
"seed"
],
"additionalProperties": false
},
{
"type": "null"
}
]
},
"ci95_hit_free_answer_fields": {
"anyOf": [
{
"type": "object",
"properties": {
"lower": {
"type": "number"
},
"upper": {
"type": "number"
},
"level": {
"const": 0.95
},
"method": {
"enum": [
"paired_bootstrap",
"paired_cluster_bootstrap"
]
},
"replicates": {
"const": 10000
},
"seed": {
"const": 20261028
}
},
"required": [
"lower",
"upper",
"level",
"method",
"replicates",
"seed"
],
"additionalProperties": false
},
{
"type": "null"
}
]
}
},
"required": [
"task_id",
"model_id",
"protocol_id",
"metric",
"unit",
"higher_is_better",
"n_population",
"n_selected",
"n_scored",
"state",
"value",
"ci95",
"source_id_digest",
"results_sha256",
"runner_sha256",
"contamination",
"contamination_receipt_sha256"
],
"additionalProperties": false,
"allOf": [
{
"if": {
"properties": {
"state": {
"const": "ok"
}
}
},
"then": {
"properties": {
"value": {
"type": "number"
},
"n_scored": {
"type": "integer",
"minimum": 1
},
"source_id_digest": {
"type": "string",
"pattern": "^[0-9a-f]{64}$"
},
"results_sha256": {
"type": "string",
"pattern": "^[0-9a-f]{64}$"
},
"runner_sha256": {
"type": "string",
"pattern": "^[0-9a-f]{64}$"
}
}
},
"else": {
"properties": {
"value": {
"type": "null"
},
"ci95": {
"type": "null"
}
}
}
},
{
"if": {
"properties": {
"contamination": {
"const": "flagged"
},
"state": {
"const": "ok"
}
},
"required": [
"contamination",
"state"
]
},
"then": {
"required": [
"value_hit_free",
"n_hit_free",
"hit_rate",
"value_hit_free_answer_fields",
"n_hit_free_answer_fields",
"hit_rate_answer_fields"
],
"properties": {
"n_hit_free": {
"type": "integer"
},
"hit_rate": {
"type": "number"
},
"n_hit_free_answer_fields": {
"type": "integer"
},
"hit_rate_answer_fields": {
"type": "number"
}
}
}
},
{
"if": {
"properties": {
"contamination": {
"enum": [
"clean",
"flagged",
"partially_scanned"
]
}
}
},
"then": {
"properties": {
"contamination_receipt_sha256": {
"type": "string"
}
}
}
},
{
"if": {
"properties": {
"n_hit_free": {
"const": 0
}
},
"required": [
"n_hit_free"
]
},
"then": {
"properties": {
"value_hit_free": {
"type": "null"
}
}
},
"else": {
"if": {
"properties": {
"contamination": {
"const": "flagged"
},
"state": {
"const": "ok"
}
},
"required": [
"contamination",
"state"
]
},
"then": {
"properties": {
"value_hit_free": {
"type": "number"
}
}
}
}
},
{
"if": {
"properties": {
"n_hit_free_answer_fields": {
"const": 0
}
},
"required": [
"n_hit_free_answer_fields"
]
},
"then": {
"properties": {
"value_hit_free_answer_fields": {
"type": "null"
}
}
},
"else": {
"if": {
"properties": {
"contamination": {
"const": "flagged"
},
"state": {
"const": "ok"
}
},
"required": [
"contamination",
"state"
]
},
"then": {
"properties": {
"value_hit_free_answer_fields": {
"type": "number"
}
}
}
}
}
]
},
"contrast": {
"type": "object",
"properties": {
"task_id": {
"type": "string",
"pattern": "^T(0[1-9]|[1-4][0-9]|50)$"
},
"model_id": {
"type": "string",
"minLength": 1
},
"protocol_id": {
"enum": [
"bench50-v0.3.1-panel200",
"bench50-v0.3.1-fullB"
]
},
"model_ref": {
"type": "string",
"minLength": 1
},
"delta": {
"type": [
"number",
"null"
]
},
"ci95": {
"anyOf": [
{
"type": "object",
"properties": {
"lower": {
"type": "number"
},
"upper": {
"type": "number"
},
"level": {
"const": 0.95
},
"method": {
"enum": [
"paired_bootstrap",
"paired_cluster_bootstrap"
]
},
"replicates": {
"const": 10000
},
"seed": {
"const": 20261028
}
},
"required": [
"lower",
"upper",
"level",
"method",
"replicates",
"seed"
],
"additionalProperties": false
},
{
"type": "null"
}
]
},
"p_raw": {
"type": [
"number",
"null"
],
"minimum": 0,
"maximum": 1
},
"p_holm": {
"type": [
"number",
"null"
],
"minimum": 0,
"maximum": 1
},
"family_size": {
"type": "integer",
"minimum": 1
},
"verdict": {
"enum": [
"withheld",
"better",
"worse",
"inconclusive"
]
},
"review": {
"type": "object",
"properties": {
"status": {
"enum": [
"pending",
"accepted",
"rejected"
]
},
"actor": {
"type": [
"string",
"null"
]
},
"artifact_sha256": {
"anyOf": [
{
"type": "string",
"pattern": "^[0-9a-f]{64}$"
},
{
"type": "null"
}
]
}
},
"required": [
"status",
"actor",
"artifact_sha256"
],
"additionalProperties": false,
"allOf": [
{
"if": {
"properties": {
"status": {
"const": "accepted"
}
}
},
"then": {
"properties": {
"actor": {
"type": "string",
"minLength": 1
},
"artifact_sha256": {
"type": "string",
"pattern": "^[0-9a-f]{64}$"
}
}
}
}
]
},
"subset": {
"enum": [
"full",
"hit_free",
"hit_free_answer_fields"
],
"description": "items on which the paired contrast was computed; hit-free subsets are defined by the training corpus of model_id and are identical for both models"
},
"n": {
"type": [
"integer",
"null"
],
"minimum": 0
},
"metric": {
"type": "string",
"minLength": 1
},
"unit": {
"type": "string",
"minLength": 1
}
},
"required": [
"task_id",
"model_id",
"protocol_id",
"model_ref",
"delta",
"ci95",
"p_raw",
"p_holm",
"family_size",
"verdict",
"review",
"subset",
"n",
"metric",
"unit"
],
"additionalProperties": false,
"allOf": [
{
"if": {
"properties": {
"verdict": {
"enum": [
"better",
"worse",
"inconclusive"
]
}
}
},
"then": {
"properties": {
"review": {
"properties": {
"status": {
"const": "accepted"
}
}
},
"delta": {
"type": "number"
},
"p_raw": {
"type": "number"
},
"p_holm": {
"type": "number"
}
}
}
},
{
"if": {
"properties": {
"review": {
"properties": {
"status": {
"enum": [
"pending",
"rejected"
]
}
}
}
}
},
"then": {
"properties": {
"verdict": {
"const": "withheld"
}
}
}
}
]
}
}
}