# Method parameters; the models are set in backbone.yaml. Values written as <...> must be set before a run. # Run from code/: python -m memebench.run_eval --method --patient # Temperature and output-token limit of the runner's answering call (answer_temperature: null sends no temperature, # for models that accept only their default). hipporag, remem, letta and mirix answer inside the method, # with their max_tokens; lightmem also holds answer_max_tokens back from its context budget. answer_temperature: 0.3 answer_max_tokens: 10000 # Shared values: every method below that has the setting refers to it by name (*NAME), so one value applies to # all methods. top_k is the number of items a method retrieves for an answer (run_eval --top-k overrides it). top_k: &TOP_K 5 # fact-linking and passage-retrieval depths of HippoRAG 2 and REMem linking_top_k: &LINKING_TOP_K retrieval_top_k: &RETRIEVAL_TOP_K # token budget of the history or retrieved text in the answer prompt (long_context fills it with the most # recent visits) max_context_tokens: &MAX_CONTEXT_TOKENS # context window that Letta and MIRIX give their agents answer_model_max_context_tokens: &ANSWER_MODEL_MAX_CONTEXT_TOKENS # chunk size of a visit before Letta, MIRIX, A-MEM and Mem0 store it, and the chunk overlap for Letta and MIRIX chunk_tokens: &CHUNK_TOKENS chunk_overlap_tokens: &CHUNK_OVERLAP_TOKENS # Letta's archival passage size passage_chunk_tokens: &PASSAGE_CHUNK_TOKENS # MIRIX's per-visit input limit max_input_tokens: &MAX_INPUT_TOKENS # longest embedding input, in embedding.tokenizer tokens: A-MEM and Mem0 truncate longer texts to it; it is the # embedding_max_seq_len of HippoRAG 2 and REMem and the embedding_chunk_size of MIRIX embedding_max_input_tokens: &EMBEDDING_MAX_INPUT_TOKENS # tokens of max_context_tokens held back for the answer prompt by HippoRAG 2 and LightMem context_reserve_tokens: &CONTEXT_RESERVE_TOKENS methods: long_context: max_context_tokens: *MAX_CONTEXT_TOKENS # tokens held back from max_context_tokens for the prompt and the question margin: 0 bm25_rag: top_k: *TOP_K k1: 1.5 b: 0.75 chunk_size: 512 chunk_overlap: 50 max_context_tokens: *MAX_CONTEXT_TOKENS dense_rag: top_k: *TOP_K chunk_size: 4096 chunk_overlap: 50 max_context_tokens: *MAX_CONTEXT_TOKENS hipporag: temperature: 0.3 max_tokens: 10000 openie_mode: online is_directed_graph: false synonymy_edge_sim_threshold: 0.8 synonymy_edge_topk: 2047 linking_top_k: *LINKING_TOP_K retrieval_top_k: *RETRIEVAL_TOP_K qa_top_k: *TOP_K damping: 0.5 passage_node_weight: 0.05 force_index_from_scratch: false force_openie_from_scratch: false save_openie: true embedding_provider: openai embedding_max_seq_len: *EMBEDDING_MAX_INPUT_TOKENS max_context_tokens: *MAX_CONTEXT_TOKENS context_reserve_tokens: *CONTEXT_RESERVE_TOKENS # a passage that exceeds the context budget is cut to the remaining tokens only when more than this many remain min_truncated_passage_tokens: remem: agent_max_steps: 5 agent_fixed_tools: false temperature: 0.3 max_tokens: 10000 dataset: memebench extract_method: episodic_gist is_directed_graph: false synonymy_edge_sim_threshold: 0.8 synonymy_edge_topk: 10 linking_top_k: *LINKING_TOP_K retrieval_top_k: *RETRIEVAL_TOP_K qa_top_k: *TOP_K damping: 0.5 passage_node_weight: 0.05 save_openie: true embedding_provider: openai embedding_max_seq_len: *EMBEDDING_MAX_INPUT_TOKENS max_context_tokens: *MAX_CONTEXT_TOKENS letta: context_window: *ANSWER_MODEL_MAX_CONTEXT_TOKENS message_buffer_autoclear: true max_steps: 50 temperature: 0.3 max_tokens: 10000 retrieve_num: *TOP_K embedding_chunk_size: *PASSAGE_CHUNK_TOKENS memorize_chunk_tokens: *CHUNK_TOKENS memorize_chunk_overlap_tokens: *CHUNK_OVERLAP_TOKENS mirix: answer_agent: true answer_max_steps: 4 memory_types: [core, episodic, semantic, resource, knowledge_vault] temperature: 0.3 max_tokens: 10000 retrieve_num: *TOP_K memorize_chunk_tokens: *CHUNK_TOKENS memorize_chunk_overlap_tokens: *CHUNK_OVERLAP_TOKENS query_memory_item_tokens: 4096 query_memory_context_tokens: 10240 max_input_tokens: *MAX_INPUT_TOKENS context_window: *ANSWER_MODEL_MAX_CONTEXT_TOKENS embedding_max_input_tokens: *EMBEDDING_MAX_INPUT_TOKENS amem: retrieve_num: *TOP_K amem_evo_threshold: 100 temperature: 0.3 max_tokens: 10000 memorize_chunk_tokens: *CHUNK_TOKENS embedding_max_input_tokens: *EMBEDDING_MAX_INPUT_TOKENS lightmem: retrieve_num: *TOP_K pre_compress: false compress_rate: 0.6 compress_device: cpu topic_segment: true # Hugging Face revision of LightMem's topic segmenter (llmlingua-2-bert-base-multilingual-cased-meetingbank) segmenter_revision: 5f0c82792b7ea14c6484e015b6a072009496b7f2 index_strategy: embedding retrieve_strategy: embedding update_mode: offline extraction_mode: flat messages_use: hybrid # longer patient turns are split at sentence boundaries max_user_tokens: # short-term buffer size shortmem_max_tokens: temperature: 0.3 max_tokens: 10000 max_context_tokens: *MAX_CONTEXT_TOKENS context_reserve_tokens: *CONTEXT_RESERVE_TOKENS # factor on the character cut when the retrieved text exceeds the context budget truncation_ratio: offline_consolidate: true consolidate_top_k: 20 consolidate_keep_top_n: 10 consolidate_score_threshold: 0.9 consolidate_max_calls: 20000 consolidate_max_workers: 20 mem0: retrieve_num: *TOP_K memorize_chunk_tokens: *CHUNK_TOKENS max_context_tokens: *MAX_CONTEXT_TOKENS temperature: 0.3 max_tokens: 10000 embedding_max_input_tokens: *EMBEDDING_MAX_INPUT_TOKENS