Spaces:
Running
Running
| { "$schema": "http://json-schema.org/draft-07/schema#", | |
| "version": "instance_level_eval_0.3.0", | |
| "type": "object", | |
| "description": "Schema for storing instance-level evaluation data for LLM evaluations, supporting single-turn, multi-turn, and agentic interactions", | |
| "required": [ | |
| "schema_version", | |
| "evaluation_id", | |
| "model_id", | |
| "evaluation_name", | |
| "sample_id", | |
| "interaction_type", | |
| "input", | |
| "answer_attribution", | |
| "evaluation" | |
| ], | |
| "additionalProperties": false, | |
| "properties": { | |
| "schema_version": { | |
| "type": "string", | |
| "description": "Version of the schema used for this instance data" | |
| }, | |
| "evaluation_id": { | |
| "type": "string", | |
| "description": "Foreign key linking to the aggregate evaluation JSON. Must match the evaluation_id in the aggregate file." | |
| }, | |
| "model_id": { | |
| "type": "string", | |
| "description": "Identifier of the model in HuggingFace format (e.g. meta-llama/Llama-3.2-1B-Instruct)" | |
| }, | |
| "evaluation_name": { | |
| "type": "string", | |
| "description": "The specific eval name, ideally unique (e.g. GSM8K, mmlu_physics). Primarily for display/filtering when evaluation_result_id is unavailable. If an evaluation run reports multiple aggregate results/metrics for the same sample, emit multiple instance records (one per aggregate result) rather than trying to attach a single instance record to many aggregate results." | |
| }, | |
| "evaluation_result_id": { | |
| "type": "string", | |
| "description": "Preferred deterministic foreign key to aggregate evaluation_results[].evaluation_result_id. This is intended to point to exactly one aggregate evaluation result; if a single underlying interaction/sample contributes to multiple aggregate results, emit multiple instance records with different evaluation_result_id values." | |
| }, | |
| "sample_id": { | |
| "type": "string", | |
| "description": "Question/sample identifier from the original dataset (e.g. gsm8k_0001)" | |
| }, | |
| "sample_hash": { | |
| "type": ["string", "null"], | |
| "description": "Hash of (input.raw + input.reference) to ensure comparison is between the same sample across models, in case sample_id is not consistent" | |
| }, | |
| "interaction_type": { | |
| "type": "string", | |
| "enum": ["single_turn", "multi_turn", "agentic"], | |
| "description": "Type of interaction: single_turn for simple Q&A, multi_turn for conversations, agentic for tool-using agents" | |
| }, | |
| "trajectory": { | |
| "description": "Agent trajectory embedded in this instance or referenced as a module/file. Event IDs are scoped to the referenced trajectory document.", | |
| "oneOf": [ | |
| { "$ref": "trajectory.schema.json" }, | |
| { "$ref": "#/definitions/module_reference" } | |
| ] | |
| }, | |
| "input": { | |
| "type": "object", | |
| "description": "Input data for the evaluation sample", | |
| "required": ["raw", "reference"], | |
| "properties": { | |
| "raw": { | |
| "type": "string", | |
| "description": "The raw input as defined in the eval" | |
| }, | |
| "formatted": { | |
| "type": ["string", "null"], | |
| "description": "Includes chat template, CoT and all relevant modifications - basically the un-tokenized version of what the model sees" | |
| }, | |
| "reference": { | |
| "type": "array", | |
| "description": "Ground truths or reference answers for comparison/scoring", | |
| "items": { "type": "string" } | |
| }, | |
| "choices": { | |
| "type": ["array", "null"], | |
| "description": "Optional list of choices for multiple-choice questions", | |
| "items": { | |
| "type": "string" | |
| } | |
| } | |
| } | |
| }, | |
| "output": { | |
| "type": ["object", "null"], | |
| "description": "Output data - only used for single_turn interactions, null for multi_turn/agentic", | |
| "required": ["raw"], | |
| "properties": { | |
| "raw": { | |
| "type": "array", | |
| "description": "Complete model responses", | |
| "items": { "type": "string" } | |
| }, | |
| "reasoning_trace": { | |
| "type": ["array", "null"], | |
| "description": "Reasoning traces of the model if applicable (e.g. chain-of-thought tokens)", | |
| "items": { "type": "string" } | |
| } | |
| } | |
| }, | |
| "messages": { | |
| "type": ["array", "null"], | |
| "description": "Full message transcript - used for multi_turn and agentic, null for single_turn. Contains all system, user, assistant, and tool messages in order.", | |
| "items": { | |
| "type": "object", | |
| "required": ["turn_idx", "role"], | |
| "properties": { | |
| "turn_idx": { | |
| "type": "integer", | |
| "minimum": 0, | |
| "description": "Index starting from 0 indicating the position in the conversation" | |
| }, | |
| "role": { | |
| "type": "string", | |
| "description": "Role of the speaker (e.g. user, assistant, system, tool)" | |
| }, | |
| "content": { | |
| "type": ["string", "null"], | |
| "description": "The actual raw text for that particular turn (can be null if empty)" | |
| }, | |
| "reasoning_trace": { | |
| "type": ["string", "null"], | |
| "description": "Reasoning trace for that particular turn if applicable" | |
| }, | |
| "tool_calls": { | |
| "type": ["array", "null"], | |
| "description": "List of tool invocations for this turn, if applicable", | |
| "items": { | |
| "type": "object", | |
| "required": ["id", "name"], | |
| "properties": { | |
| "id": { | |
| "type": "string", | |
| "description": "Unique identifier for the tool call" | |
| }, | |
| "name": { | |
| "type": "string", | |
| "description": "Name of tool/function" | |
| }, | |
| "arguments": { | |
| "type": ["object", "null"], | |
| "description": "Arguments used to call the tool (all values must be strings)", | |
| "additionalProperties": { | |
| "type": "string" | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "tool_call_id": { | |
| "type": ["array", "null"], | |
| "description": "Reference to the tool call ID(s) this message has the content payload for.", | |
| "items": { | |
| "type": "string" | |
| } | |
| } | |
| } | |
| } | |
| }, | |
| "answer_attribution": { | |
| "type": "array", | |
| "description": "Information about how the answer was extracted from the model output", | |
| "items": { | |
| "type": "object", | |
| "required": ["source", "extracted_value", "extraction_method", "is_terminal"], | |
| "anyOf": [ | |
| { "required": ["turn_idx"] }, | |
| { "required": ["event_id"] } | |
| ], | |
| "properties": { | |
| "turn_idx": { | |
| "type": "integer", | |
| "minimum": 0, | |
| "description": "Legacy turn index in messages. Retained for compatibility; event_id is the stable trajectory pointer." | |
| }, | |
| "event_id": { | |
| "type": "string", | |
| "description": "Stable ID of the trajectory event that supports this answer. Resolve within the linked trajectory document." | |
| }, | |
| "source": { | |
| "type": "string", | |
| "description": "Source of the extracted value (e.g. 'output.raw' or 'messages[turn_idx].content')" | |
| }, | |
| "extracted_value": { | |
| "type": "string", | |
| "description": "Value that was extracted" | |
| }, | |
| "extraction_method": { | |
| "type": "string", | |
| "description": "Method used to extract the value (e.g. regex, exact_match, llm_judge, custom)" | |
| }, | |
| "is_terminal": { | |
| "type": "boolean", | |
| "description": "Whether this is the final answer (false if intermediate outputs are used to build up to a final answer)" | |
| } | |
| } | |
| } | |
| }, | |
| "evaluation": { | |
| "type": "object", | |
| "description": "Evaluation results and scoring data", | |
| "required": ["score", "is_correct"], | |
| "properties": { | |
| "score": { | |
| "type": "number", | |
| "description": "Instance-level score" | |
| }, | |
| "is_correct": { | |
| "type": "boolean", | |
| "description": "Whether the final answer is correct" | |
| }, | |
| "num_turns": { | |
| "type": ["integer", "null"], | |
| "minimum": 1, | |
| "description": "Number of turns in the interaction" | |
| }, | |
| "tool_calls_count": { | |
| "type": ["integer", "null"], | |
| "minimum": 0, | |
| "description": "Count of tool calls across all turns in messages" | |
| } | |
| } | |
| }, | |
| "token_usage": { | |
| "type": ["object", "null"], | |
| "description": "Token usage for the model completion", | |
| "required": ["input_tokens", "output_tokens", "total_tokens"], | |
| "properties": { | |
| "input_tokens": { | |
| "type": "integer", | |
| "minimum": 0, | |
| "description": "Total input tokens used" | |
| }, | |
| "output_tokens": { | |
| "type": "integer", | |
| "minimum": 0, | |
| "description": "Total output tokens used" | |
| }, | |
| "total_tokens": { | |
| "type": "integer", | |
| "minimum": 0, | |
| "description": "Total tokens used" | |
| }, | |
| "input_tokens_cache_write": { | |
| "type": ["integer", "null"], | |
| "minimum": 0, | |
| "description": "Number of tokens written to the cache" | |
| }, | |
| "input_tokens_cache_read": { | |
| "type": ["integer", "null"], | |
| "minimum": 0, | |
| "description": "Number of tokens retrieved from the cache" | |
| }, | |
| "reasoning_tokens": { | |
| "type": ["integer", "null"], | |
| "minimum": 0, | |
| "description": "Number of tokens used for reasoning" | |
| } | |
| } | |
| }, | |
| "performance": { | |
| "type": ["object", "null"], | |
| "description": "Performance and latency metrics", | |
| "properties": { | |
| "latency_ms": { | |
| "type": ["number", "null"], | |
| "minimum": 0, | |
| "description": "Total latency in milliseconds" | |
| }, | |
| "time_to_first_token_ms": { | |
| "type": ["number", "null"], | |
| "minimum": 0, | |
| "description": "Time to first token in milliseconds" | |
| }, | |
| "generation_time_ms": { | |
| "type": ["number", "null"], | |
| "minimum": 0, | |
| "description": "Time for generation in milliseconds" | |
| }, | |
| "additional_details": { | |
| "type": ["object", "null"], | |
| "description": "Additional performance metrics (key-value pairs, all values must be strings)", | |
| "additionalProperties": { | |
| "type": "string" | |
| } | |
| } | |
| } | |
| }, | |
| "error": { | |
| "type": ["string", "null"], | |
| "description": "Information about any error that occurred (e.g. timeout, refusal, API error)" | |
| }, | |
| "metadata": { | |
| "type": ["object", "null"], | |
| "description": "Optional metadata about the sample (e.g. subject, difficulty, tags)", | |
| "additionalProperties": { | |
| "type": "string" | |
| } | |
| } | |
| }, | |
| "definitions": { | |
| "module_reference": { | |
| "type": "object", | |
| "description": "Reference to a trajectory stored as a separate file or module.", | |
| "required": ["type", "uri"], | |
| "properties": { | |
| "type": { "const": "module_reference" }, | |
| "uri": { | |
| "type": "string", | |
| "format": "uri-reference", | |
| "description": "URI or relative path to the referenced trajectory." | |
| }, | |
| "media_type": { "type": "string" }, | |
| "format": { "type": "string", "description": "Trajectory encoding or interchange format, e.g. ATIF or JSONL." }, | |
| "schema_uri": { "type": "string", "format": "uri-reference" }, | |
| "schema_version": { "type": "string" }, | |
| "sha256": { | |
| "type": "string", | |
| "pattern": "^[A-Fa-f0-9]{64}$", | |
| "description": "Optional SHA-256 digest of the referenced file." | |
| } | |
| }, | |
| "additionalProperties": false | |
| } | |
| }, | |
| "allOf": [ | |
| { | |
| "if": { | |
| "properties": { | |
| "interaction_type": { | |
| "const": "single_turn" | |
| } | |
| } | |
| }, | |
| "then": { | |
| "required": ["output"], | |
| "properties": { | |
| "output": { | |
| "type": "object", | |
| "not": { | |
| "type": "null" | |
| } | |
| }, | |
| "messages": { | |
| "type": "null" | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "if": { | |
| "properties": { | |
| "interaction_type": { | |
| "enum": ["multi_turn", "agentic"] | |
| } | |
| } | |
| }, | |
| "then": { | |
| "required": ["messages"], | |
| "properties": { | |
| "output": { | |
| "type": "null" | |
| }, | |
| "messages": { | |
| "type": "array", | |
| "not": { | |
| "type": "null" | |
| } | |
| }, | |
| "metrics": { | |
| "required": ["num_turns"] | |
| } | |
| } | |
| } | |
| } | |
| ] | |
| } | |