{   "$schema": "http://json-schema.org/draft-07/schema#",
    "version": "instance_level_eval_0.3.0",
    "type": "object",
    "description": "Schema for storing instance-level evaluation data for LLM evaluations, supporting single-turn, multi-turn, and agentic interactions",
    "required": [
        "schema_version",
        "evaluation_id",
        "model_id",
        "evaluation_name",
        "sample_id",
        "interaction_type",
        "input",
        "answer_attribution",
        "evaluation"
    ],
    "additionalProperties": false,
    "properties": {
        "schema_version": {
            "type": "string",
            "description": "Version of the schema used for this instance data"
        },
        "evaluation_id": {
            "type": "string",
            "description": "Foreign key linking to the aggregate evaluation JSON. Must match the evaluation_id in the aggregate file."
        },
        "model_id": {
            "type": "string",
            "description": "Identifier of the model in HuggingFace format (e.g. meta-llama/Llama-3.2-1B-Instruct)"
        },
        "evaluation_name": {
            "type": "string",
            "description": "The specific eval name, ideally unique (e.g. GSM8K, mmlu_physics). Primarily for display/filtering when evaluation_result_id is unavailable. If an evaluation run reports multiple aggregate results/metrics for the same sample, emit multiple instance records (one per aggregate result) rather than trying to attach a single instance record to many aggregate results."
        },
        "evaluation_result_id": {
            "type": "string",
            "description": "Preferred deterministic foreign key to aggregate evaluation_results[].evaluation_result_id. This is intended to point to exactly one aggregate evaluation result; if a single underlying interaction/sample contributes to multiple aggregate results, emit multiple instance records with different evaluation_result_id values."
        },
        "sample_id": {
            "type": "string",
            "description": "Question/sample identifier from the original dataset (e.g. gsm8k_0001)"
        },
        "sample_hash": {
            "type": ["string", "null"],
            "description": "Hash of (input.raw + input.reference) to ensure comparison is between the same sample across models, in case sample_id is not consistent"
        },
        "interaction_type": {
            "type": "string",
            "enum": ["single_turn", "multi_turn", "agentic"],
            "description": "Type of interaction: single_turn for simple Q&A, multi_turn for conversations, agentic for tool-using agents"
        },
        "trajectory": {
            "description": "Agent trajectory embedded in this instance or referenced as a module/file. Event IDs are scoped to the referenced trajectory document.",
            "oneOf": [
                { "$ref": "trajectory.schema.json" },
                { "$ref": "#/definitions/module_reference" }
            ]
        },
        "input": {
            "type": "object",
            "description": "Input data for the evaluation sample",
            "required": ["raw", "reference"],
            "properties": {
                "raw": {
                    "type": "string",
                    "description": "The raw input as defined in the eval"
                },
                "formatted": {
                    "type": ["string", "null"],
                    "description": "Includes chat template, CoT and all relevant modifications - basically the un-tokenized version of what the model sees"
                },
                "reference": {
                    "type": "array",
                    "description": "Ground truths or reference answers for comparison/scoring",
                    "items": { "type": "string" }
                },
                "choices": {
                    "type": ["array", "null"],
                    "description": "Optional list of choices for multiple-choice questions",
                    "items": {
                        "type": "string"
                    }
                }
            }
        },
        "output": {
            "type": ["object", "null"],
            "description": "Output data - only used for single_turn interactions, null for multi_turn/agentic",
            "required": ["raw"],
            "properties": {
                "raw": {
                    "type": "array",
                    "description": "Complete model responses",
                    "items": { "type": "string" }
                },
                "reasoning_trace": {
                    "type": ["array", "null"],
                    "description": "Reasoning traces of the model if applicable (e.g. chain-of-thought tokens)",
                    "items": { "type": "string" }
                }
            }
        },
        "messages": {
            "type": ["array", "null"],
            "description": "Full message transcript - used for multi_turn and agentic, null for single_turn. Contains all system, user, assistant, and tool messages in order.",
            "items": {
                "type": "object",
                "required": ["turn_idx", "role"],
                "properties": {
                    "turn_idx": {
                        "type": "integer",
                        "minimum": 0,
                        "description": "Index starting from 0 indicating the position in the conversation"
                    },
                    "role": {
                        "type": "string",
                        "description": "Role of the speaker (e.g. user, assistant, system, tool)"
                    },
                    "content": {
                        "type": ["string", "null"],
                        "description": "The actual raw text for that particular turn (can be null if empty)"
                    },
                    "reasoning_trace": {
                        "type": ["string", "null"],
                        "description": "Reasoning trace for that particular turn if applicable"
                    },
                    "tool_calls": {
                        "type": ["array", "null"],
                        "description": "List of tool invocations for this turn, if applicable",
                        "items": {
                            "type": "object",
                            "required": ["id", "name"],
                            "properties": {
                                "id": {
                                    "type": "string",
                                    "description": "Unique identifier for the tool call"
                                },
                                "name": {
                                    "type": "string",
                                    "description": "Name of tool/function"
                                },
                                "arguments": {
                                    "type": ["object", "null"],
                                    "description": "Arguments used to call the tool (all values must be strings)",
                                    "additionalProperties": {
                                        "type": "string"
                                    }
                                }
                            }
                        }
                    },
                    "tool_call_id": {
                        "type": ["array", "null"],
                        "description": "Reference to the tool call ID(s) this message has the content payload for.",
                        "items": {
                            "type": "string"
                        }
                    }
                }
            }
        },
        "answer_attribution": {
            "type": "array",
            "description": "Information about how the answer was extracted from the model output",
            "items": {
                "type": "object",
                "required": ["source", "extracted_value", "extraction_method", "is_terminal"],
                "anyOf": [
                    { "required": ["turn_idx"] },
                    { "required": ["event_id"] }
                ],
                "properties": {
                    "turn_idx": {
                        "type": "integer",
                        "minimum": 0,
                        "description": "Legacy turn index in messages. Retained for compatibility; event_id is the stable trajectory pointer."
                    },
                    "event_id": {
                        "type": "string",
                        "description": "Stable ID of the trajectory event that supports this answer. Resolve within the linked trajectory document."
                    },
                    "source": {
                        "type": "string",
                        "description": "Source of the extracted value (e.g. 'output.raw' or 'messages[turn_idx].content')"
                    },
                    "extracted_value": {
                        "type": "string",
                        "description": "Value that was extracted"
                    },
                    "extraction_method": {
                        "type": "string",
                        "description": "Method used to extract the value (e.g. regex, exact_match, llm_judge, custom)"
                    },
                    "is_terminal": {
                        "type": "boolean",
                        "description": "Whether this is the final answer (false if intermediate outputs are used to build up to a final answer)"
                    }
                }
            }
        },
        "evaluation": {
            "type": "object",
            "description": "Evaluation results and scoring data",
            "required": ["score", "is_correct"],
            "properties": {
                "score": {
                    "type": "number",
                    "description": "Instance-level score"
                },
                "is_correct": {
                    "type": "boolean",
                    "description": "Whether the final answer is correct"
                },
                "num_turns": {
                    "type": ["integer", "null"],
                    "minimum": 1,
                    "description": "Number of turns in the interaction"
                },
                "tool_calls_count": {
                    "type": ["integer", "null"],
                    "minimum": 0,
                    "description": "Count of tool calls across all turns in messages"
                }
            }
        },
        "token_usage": {
            "type": ["object", "null"],
            "description": "Token usage for the model completion",
            "required": ["input_tokens", "output_tokens", "total_tokens"],
            "properties": {
                "input_tokens": {
                    "type": "integer",
                    "minimum": 0,
                    "description": "Total input tokens used"
                },
                "output_tokens": {
                    "type": "integer",
                    "minimum": 0,
                    "description": "Total output tokens used"
                },
                "total_tokens": {
                    "type": "integer",
                    "minimum": 0,
                    "description": "Total tokens used"
                },
                "input_tokens_cache_write": {
                    "type": ["integer", "null"],
                    "minimum": 0,
                    "description": "Number of tokens written to the cache"
                },
                "input_tokens_cache_read": {
                    "type": ["integer", "null"],
                    "minimum": 0,
                    "description": "Number of tokens retrieved from the cache"
                },
                "reasoning_tokens": {
                    "type": ["integer", "null"],
                    "minimum": 0,
                    "description": "Number of tokens used for reasoning"
                }
            }
        },
        "performance": {
            "type": ["object", "null"],
            "description": "Performance and latency metrics",
            "properties": {
                "latency_ms": {
                    "type": ["number", "null"],
                    "minimum": 0,
                    "description": "Total latency in milliseconds"
                },
                "time_to_first_token_ms": {
                    "type": ["number", "null"],
                    "minimum": 0,
                    "description": "Time to first token in milliseconds"
                },
                "generation_time_ms": {
                    "type": ["number", "null"],
                    "minimum": 0,
                    "description": "Time for generation in milliseconds"
                },
                "additional_details": {
                    "type": ["object", "null"],
                    "description": "Additional performance metrics (key-value pairs, all values must be strings)",
                    "additionalProperties": {
                        "type": "string"
                    }
                }
            }
        },
        "error": {
            "type": ["string", "null"],
            "description": "Information about any error that occurred (e.g. timeout, refusal, API error)"
        },
        "metadata": {
            "type": ["object", "null"],
            "description": "Optional metadata about the sample (e.g. subject, difficulty, tags)",
            "additionalProperties": {
                "type": "string"
            }
        }
    },
    "definitions": {
        "module_reference": {
            "type": "object",
            "description": "Reference to a trajectory stored as a separate file or module.",
            "required": ["type", "uri"],
            "properties": {
                "type": { "const": "module_reference" },
                "uri": {
                    "type": "string",
                    "format": "uri-reference",
                    "description": "URI or relative path to the referenced trajectory."
                },
                "media_type": { "type": "string" },
                "format": { "type": "string", "description": "Trajectory encoding or interchange format, e.g. ATIF or JSONL." },
                "schema_uri": { "type": "string", "format": "uri-reference" },
                "schema_version": { "type": "string" },
                "sha256": {
                    "type": "string",
                    "pattern": "^[A-Fa-f0-9]{64}$",
                    "description": "Optional SHA-256 digest of the referenced file."
                }
            },
            "additionalProperties": false
        }
    },
    "allOf": [
        {
            "if": {
                "properties": {
                    "interaction_type": {
                        "const": "single_turn"
                    }
                }
            },
            "then": {
                "required": ["output"],
                "properties": {
                    "output": {
                        "type": "object",
                        "not": {
                            "type": "null"
                        }
                    },
                    "messages": {
                        "type": "null"
                    }
                }
            }
        },
        {
            "if": {
                "properties": {
                    "interaction_type": {
                        "enum": ["multi_turn", "agentic"]
                    }
                }
            },
            "then": {
                "required": ["messages"],
                "properties": {
                    "output": {
                        "type": "null"
                    },
                    "messages": {
                        "type": "array",
                        "not": {
                            "type": "null"
                        }
                    },
                    "metrics": {
                        "required": ["num_turns"]
                    }
                }
            }
        }
    ]
}