{ "$schema": "http://json-schema.org/draft-07/schema#", "version": "0.3.0", "type": "object", "description": "Schema for storing and validating LLMs evaluation data, including model configuration, prompts, instances, Output, and evaluation metrics", "required": [ "schema_version", "evaluation_id", "retrieved_timestamp", "source_metadata", "model_info", "eval_library", "evaluation_results" ], "additionalProperties": false, "properties": { "schema_version": { "type": "string", "description": "Version of the schema used for this evaluation data" }, "evaluation_id": { "type": "string", "description": "Unique identifier for this specific evaluation run. Use eval_name/model_id/retrieved_timestamp format" }, "evaluation_timestamp": { "type": "string", "description": "Timestamp for when the evaluation was run" }, "retrieved_timestamp": { "type": "string", "description": "Timestamp for when this record was created - using Unix Epoch time format" }, "source_metadata": { "type": "object", "description": "Metadata about the source of the leaderboard data", "required": [ "source_type", "source_organization_name", "evaluator_relationship" ], "properties": { "source_name": { "type": "string", "description": "Name of the source (e.g. title of the source leaderboard or name of the platform used for the evaluation)." }, "source_type": { "type": "string", "enum": [ "documentation", "evaluation_run" ], "description": "Whether the data comes from a direct evaluation run or from documentation" }, "source_organization_name": { "type": "string", "description": "Name of the organization that provides the data" }, "source_organization_url": { "type": "string", "description": "URL for the organization that provides the data" }, "source_organization_logo_url": { "type": "string", "description": "URL for the Logo for the organization that provides the data" }, "evaluator_relationship": { "type": "string", "description": "Relationship between the evaluator and the model", "enum": [ "first_party", "third_party", "collaborative", "other" ] }, "additional_details": { "type": "object", "description": "Additional parameters (key-value pairs, all values must be strings)", "additionalProperties": { "type": "string" } } } }, "eval_library": { "type": "object", "description": "Evaluation library/framework used to run the evaluation", "properties": { "name": { "type": "string", "description": "Name of the evaluation library (e.g. inspect_ai, lm_eval, helm)" }, "version": { "type": "string", "description": "Version of the evaluation library. Use 'unknown' if the version is not available." }, "additional_details": { "type": "object", "description": "Additional parameters (key-value pairs, all values must be strings)", "additionalProperties": { "type": "string" } } }, "required": [ "name", "version" ] }, "model_info": { "$ref": "#/$defs/model_info" }, "evaluation_results": { "type": "array", "description": "Array of evaluation results", "items": { "type": "object", "required": [ "evaluation_name", "source_data", "metric_config", "score_details" ], "properties": { "evaluation_result_id": { "type": "string", "description": "Stable identifier for this metric result inside an evaluation run. Recommended deterministic join key for instance-level records." }, "evaluation_name": { "type": "string", "description": "Name of the evaluation" }, "source_data": { "description": "Source of dataset for this evaluation: URL, HuggingFace dataset, or private/custom dataset.", "oneOf": [ { "$ref": "#/$defs/source_data_url" }, { "$ref": "#/$defs/source_data_hf" }, { "$ref": "#/$defs/source_data_private" } ] }, "evaluation_timestamp": { "type": "string", "description": "Timestamp for when the evaluations were run" }, "metric_config": { "type": "object", "description": "Details about the metric", "required": [ "lower_is_better" ], "properties": { "evaluation_description": { "type": "string", "description": "Description of the evaluation" }, "metric_id": { "type": "string", "description": "Stable metric identifier for joining/deduping/querying. Use a canonical global id when applicable (e.g. accuracy, f1_macro, auroc, rmse, pass_at_k). For benchmark/leaderboard-specific metrics, use a namespaced id (e.g. rewardbench.overall, lmarena.elo)." }, "metric_name": { "type": "string", "description": "Display name for the metric (e.g. Accuracy, F1-macro, pass@1)." }, "metric_kind": { "type": "string", "description": "Normalized metric family/type used for safe aggregation (e.g. accuracy, f1, auroc, rmse, mae, pass_rate, elo)." }, "metric_unit": { "type": "string", "description": "Unit of the metric values (e.g. proportion, percent, points, ms, tokens)." }, "metric_parameters": { "type": "object", "description": "Metric-specific parameters (e.g. {\"k\": 1} for pass@k).", "additionalProperties": { "type": [ "string", "number", "boolean", "null" ] } }, "lower_is_better": { "type": "boolean", "description": "Whether a lower score is better" }, "score_type": { "type": "string", "description": "Type of score", "enum": [ "binary", "continuous", "levels" ] }, "level_names": { "type": "array", "description": "Names of the score levels", "items": { "type": "string" } }, "level_metadata": { "type": "array", "description": "Additional Description for each Score Level", "items": { "type": "string" } }, "has_unknown_level": { "type": "boolean", "description": "Indicates whether there is an Unknown Level - if True, then a score of -1 will be treated as Unknown" }, "min_score": { "type": "number", "description": "Minimum possible score for continuous metric" }, "max_score": { "type": "number", "description": "Maximum possible score for continuous metric" }, "llm_scoring": { "type": "object", "description": "Configuration when LLM is used as scorer/judge", "required": [ "judges", "input_prompt" ], "properties": { "judges": { "type": "array", "description": "LLM judge(s) - single item for judge, multiple for jury", "items": { "$ref": "#/$defs/judge_config" }, "minItems": 1 }, "input_prompt": { "type": "string", "description": "Prompt template used for judging" }, "aggregation_method": { "type": "string", "enum": [ "majority_vote", "average", "weighted_average", "median" ], "description": "How to aggregate scores when multiple judges" }, "expert_baseline": { "type": "number", "description": "Expert/human baseline score for comparison" }, "additional_details": { "type": "object", "description": "Additional parameters (key-value pairs, all values must be strings)", "additionalProperties": { "type": "string" } } } }, "additional_details": { "type": "object", "description": "Additional parameters (key-value pairs, all values must be strings)", "additionalProperties": { "type": "string" } } }, "if": { "properties": { "score_type": { "const": "levels" } } }, "then": { "required": [ "level_names", "has_unknown_level" ] }, "else": { "if": { "properties": { "score_type": { "const": "continuous" } } }, "then": { "required": [ "min_score", "max_score" ] } } }, "score_details": { "type": "object", "description": "The score for the evaluation and related details", "required": [ "score" ], "properties": { "score": { "type": "number", "description": "The score for the evaluation" }, "details": { "type": "object", "description": "Additional parameters (key-value pairs, all values must be strings)", "additionalProperties": { "type": "string" } }, "uncertainty": { "type": "object", "description": "Quantification of uncertainty around the reported score", "properties": { "standard_error": { "type": "object", "description": "Standard error of the score estimate (SE_mean = standard_deviation / sqrt(num_samples))", "properties": { "value": { "type": "number", "description": "The standard error value" }, "method": { "type": "string", "description": "How the standard error was computed (e.g. 'analytic', 'bootstrap', 'jackknife')" } }, "required": [ "value" ] }, "confidence_interval": { "type": "object", "description": "Lower and upper bounds for the metric at a given confidence level.", "properties": { "lower": { "type": "number", "description": "Lower bound of the confidence interval" }, "upper": { "type": "number", "description": "Upper bound of the confidence interval" }, "confidence_level": { "type": "number", "description": "Confidence level (e.g. 0.95 for a 95% confidence interval)", "minimum": 0, "maximum": 1 }, "method": { "type": "string", "description": "How the confidence interval was computed" } }, "required": [ "lower", "upper" ] }, "standard_deviation": { "type": "number", "description": "Standard deviation of the per-sample scores" }, "num_samples": { "type": "integer", "description": "Number of samples used to compute the uncertainty estimates" }, "num_bootstrap_samples": { "type": "integer", "description": "Number of bootstrap resamples used, if bootstrap method was applied" } } } } }, "generation_config": { "type": "object", "properties": { "generation_args": { "type": "object", "description": "Parameters used to generate results - properties may vary by model type", "properties": { "temperature": { "type": [ "null", "number" ], "description": "Sampling temperature" }, "top_p": { "type": [ "null", "number" ], "description": "Nucleus sampling parameter" }, "top_k": { "type": [ "null", "number" ], "description": "Top-k sampling parameter" }, "max_tokens": { "type": "integer", "minimum": 1, "description": "Maximum number of tokens to generate" }, "execution_command": { "type": "string", "description": "Command used to run the model to generate results" }, "reasoning": { "type": "boolean", "description": "Whether reasoning orchain-of-thought was used to generate results" }, "prompt_template": { "type": "string", "description": "Input prompt template for task (should contain agentic info if needed)." }, "agentic_eval_config": { "type": "object", "description": "General configuration for agentic evaluations.", "properties": { "available_tools": { "type": "array", "description": "List of all available tools with their configurations", "items": { "type": "object", "properties": { "name": { "type": "string", "description": "e.g. bash, calculator, ..." }, "description": { "type": "string" }, "parameters": { "type": "object", "description": "Additional parameters (key-value pairs, all values must be strings)", "additionalProperties": { "type": "string" } } } } }, "additional_details": { "type": "object", "description": "Additional parameters (key-value pairs, all values must be strings)", "additionalProperties": { "type": "string" } } } }, "eval_plan": { "type": "object", "description": "Plan (solvers) used in evaluation. Solvers are crucial parts of Inspect evaluations which can serve a wide variety of purposes like providing system prompts, prompt engineering, model generation or multi-turn dialog.", "properties": { "name": { "type": "string" }, "steps": { "type": "array", "description": "Array of evaluation plan steps (each step is a JSON-serialized string)", "items": { "type": "string" } }, "config": { "type": "object", "description": "Additional parameters (key-value pairs, all values must be strings)", "additionalProperties": { "type": "string" } } } }, "eval_limits": { "type": "object", "description": "Listed evaluation limits like time limit, message limit, token limit.", "properties": { "time_limit": { "type": "integer", "description": "Time limit for evaluation." }, "message_limit": { "type": "integer", "description": "Message limit for evaluation." }, "token_limit": { "type": "integer", "description": "Token limit for evaluation." } } }, "sandbox": { "type": "object", "properties": { "type": { "type": "string", "description": "Type of sandbox e.g. docker" }, "config": { "type": "string", "description": "Config file name/path e.g. compose.yaml. TODO or full config? Not sure based on the Inspect docs" } } }, "max_attempts": { "type": "integer", "description": "Maximum number of submission attempts (default 1).", "default": 1 }, "incorrect_attempt_feedback": { "type": "string", "description": "Feedback from the model after incorrect attempt." } }, "additionalProperties": false }, "additional_details": { "type": "object", "description": "Additional parameters (key-value pairs, all values must be strings)", "additionalProperties": { "type": "string" } } } } } } }, "detailed_evaluation_results": { "type": "object", "description": "Reference to the evaluation results for all individual samples in the evaluation", "properties": { "format": { "type": "string", "description": "Format of the detailed evaluation results", "enum": [ "jsonl", "json" ] }, "file_path": { "type": "string", "description": "Path to the detailed evaluation results file" }, "hash_algorithm": { "type": "string", "description": "Hash algorithm used for checksum and sample_hash in instance-level data", "enum": [ "sha256", "md5" ] }, "checksum": { "type": "string", "description": "Checksum value of the file" }, "total_rows": { "type": "integer", "description": "Total number of rows in the detailed evaluation results file" }, "additional_details": { "type": "object", "description": "Additional parameters (key-value pairs, all values must be strings)", "additionalProperties": { "type": "string" } } } } }, "$defs": { "judge_config": { "type": "object", "description": "Configuration for a single LLM judge/juror", "required": [ "model_info" ], "properties": { "model_info": { "$ref": "#/$defs/model_info" }, "temperature": { "type": "number" }, "weight": { "type": "number", "description": "Weight of this judge's score in aggregation (used in jury)" }, "additional_details": { "type": "object", "description": "Additional parameters (key-value pairs, all values must be strings)", "additionalProperties": { "type": "string" } } } }, "model_info": { "type": "object", "description": "Complete model specification including basic information, technical configuration and inference settings", "required": [ "name", "id" ], "properties": { "name": { "type": "string", "description": "Model name provided by evaluation source" }, "id": { "type": "string", "description": "Model name in HuggingFace format (e.g. meta-llama/Llama-3.1-8B-Instruct for models available on HuggingFace or openai/azure/gpt-4o-mini-2024-07-18 for closed API models)" }, "developer": { "type": "string", "description": "Name of organization that provides the model (e.g. 'OpenAI')" }, "inference_platform": { "type": "string", "description": "Name of inference platform which provides an access to models by API to run the evaluations or provides models weights to run them locally (e.g. HuggingFace, Bedrock, Together AI)" }, "inference_engine": { "type": "object", "description": "Name of inference engine which provides an access to optimized models to use them for local evaluations (e.g. vLLM, Ollama).", "properties": { "name": { "type": "string", "description": "Name of the inference engine" }, "version": { "type": "string", "description": "Version of the inference engine" } } }, "additional_details": { "type": "object", "description": "Additional parameters (key-value pairs, all values must be strings)", "additionalProperties": { "type": "string" } } } }, "source_data_url": { "type": "object", "description": "URL source for the evaluation data", "required": [ "dataset_name", "source_type", "url" ], "properties": { "dataset_name": { "type": "string", "description": "Name of the source dataset" }, "source_type": { "const": "url" }, "url": { "type": "array", "items": { "type": "string" }, "minItems": 1, "description": "URL(s) for the source of the evaluation data" }, "additional_details": { "type": "object", "description": "Additional parameters (key-value pairs, all values must be strings)", "additionalProperties": { "type": "string" } } } }, "source_data_hf": { "type": "object", "description": "Details about HuggingFace dataset used for evaluation", "required": [ "dataset_name", "source_type" ], "properties": { "dataset_name": { "type": "string", "description": "Name of the source dataset" }, "source_type": { "const": "hf_dataset" }, "hf_repo": { "type": "string", "description": "HuggingFace repository identifier" }, "hf_split": { "type": "string", "description": "One of train, val or test." }, "samples_number": { "type": "integer", "description": "Number of samples in the dataset" }, "sample_ids": { "type": "array", "description": "Array of sample ids used for evaluation", "items": { "type": "string" } }, "additional_details": { "type": "object", "description": "Additional parameters (key-value pairs, all values must be strings)", "additionalProperties": { "type": "string" } } } }, "source_data_private": { "type": "object", "description": "Generic source data when neither URL array nor HuggingFace dataset applies", "required": [ "dataset_name", "source_type" ], "properties": { "dataset_name": { "type": "string", "description": "Name of the source dataset" }, "source_type": { "const": "other" }, "additional_details": { "type": "object", "description": "Additional parameters (key-value pairs, all values must be strings)", "additionalProperties": { "type": "string" } } } } } }