| # Models used by the evaluation. Replace every <...> value that the selected methods need before running. | |
| # base_url: null uses the client's default endpoint. api_key_env names the environment | |
| # variable that holds the key for that endpoint (OPENAI_API_KEY by default); each key is sent only to its endpoint. | |
| answer: # answers every question | |
| model: <ANSWER_MODEL> | |
| base_url: null | |
| api_key_env: OPENAI_API_KEY | |
| reasoning: <true|false> # true for a reasoning model: the methods' internal calls use max_completion_tokens | |
| tokenizer: <TOKENIZER> # tiktoken encoding that counts the context budgets and chunk sizes | |
| judge: # grades each answer against the gold | |
| model: <JUDGE_MODEL> | |
| base_url: null | |
| api_key_env: OPENAI_API_KEY | |
| reasoning: <true|false> | |
| embedding: # embeddings of every method that retrieves by vector similarity | |
| model: <EMBEDDING_MODEL> | |
| base_url: null | |
| api_key_env: OPENAI_API_KEY | |
| dim: <EMBEDDING_DIM> | |
| batch_size: <EMBEDDING_BATCH_SIZE> # texts per embedding request | |
| tokenizer: <EMBEDDING_TOKENIZER> # tiktoken encoding of the embedding model (input truncation) | |
| memory: # optional: the model of every LLM call inside a method (hipporag, remem, | |
| # letta and mirix also answer with it) | |
| model: null # null = the answer model, at answer.base_url | |
| base_url: null | |
| api_key_env: null # null = OPENAI_API_KEY | |
| reasoning: null # null = the answer model's setting; required when memory.model is another model | |