code_release/code/memebench/configs/methods.yaml

# Method parameters; the models are set in backbone.yaml. Values written as <...> must be set before a run.
# Run from code/: python -m memebench.run_eval --method <method> --patient <P01..P23>

# Temperature and output-token limit of the runner's answering call (answer_temperature: null sends no temperature,
# for models that accept only their default). hipporag, remem, letta and mirix answer inside the method,
# with their max_tokens; lightmem also holds answer_max_tokens back from its context budget.
answer_temperature: 0.3
answer_max_tokens: 10000

# Shared values: every method below that has the setting refers to it by name (*NAME), so one value applies to
# all methods. top_k is the number of items a method retrieves for an answer (run_eval --top-k overrides it).
top_k: &TOP_K 5
# fact-linking and passage-retrieval depths of HippoRAG 2 and REMem
linking_top_k: &LINKING_TOP_K <LINKING_TOP_K>
retrieval_top_k: &RETRIEVAL_TOP_K <RETRIEVAL_TOP_K>
# token budget of the history or retrieved text in the answer prompt (long_context fills it with the most
# recent visits)
max_context_tokens: &MAX_CONTEXT_TOKENS <MAX_CONTEXT_TOKENS>
# context window that Letta and MIRIX give their agents
answer_model_max_context_tokens: &ANSWER_MODEL_MAX_CONTEXT_TOKENS <ANSWER_MODEL_MAX_CONTEXT_TOKENS>
# chunk size of a visit before Letta, MIRIX, A-MEM and Mem0 store it, and the chunk overlap for Letta and MIRIX
chunk_tokens: &CHUNK_TOKENS <CHUNK_TOKENS>
chunk_overlap_tokens: &CHUNK_OVERLAP_TOKENS <CHUNK_OVERLAP_TOKENS>
# Letta's archival passage size
passage_chunk_tokens: &PASSAGE_CHUNK_TOKENS <PASSAGE_CHUNK_TOKENS>
# MIRIX's per-visit input limit
max_input_tokens: &MAX_INPUT_TOKENS <MAX_INPUT_TOKENS>
# longest embedding input, in embedding.tokenizer tokens: A-MEM and Mem0 truncate longer texts to it; it is the
# embedding_max_seq_len of HippoRAG 2 and REMem and the embedding_chunk_size of MIRIX
embedding_max_input_tokens: &EMBEDDING_MAX_INPUT_TOKENS <EMBEDDING_MAX_INPUT_TOKENS>
# tokens of max_context_tokens held back for the answer prompt by HippoRAG 2 and LightMem
context_reserve_tokens: &CONTEXT_RESERVE_TOKENS <CONTEXT_RESERVE_TOKENS>

methods:
  long_context:
    max_context_tokens: *MAX_CONTEXT_TOKENS
    # tokens held back from max_context_tokens for the prompt and the question
    margin: 0

  bm25_rag:
    top_k: *TOP_K
    k1: 1.5
    b: 0.75
    chunk_size: 512
    chunk_overlap: 50
    max_context_tokens: *MAX_CONTEXT_TOKENS

  dense_rag:
    top_k: *TOP_K
    chunk_size: 4096
    chunk_overlap: 50
    max_context_tokens: *MAX_CONTEXT_TOKENS

  hipporag:
    temperature: 0.3
    max_tokens: 10000
    openie_mode: online
    is_directed_graph: false
    synonymy_edge_sim_threshold: 0.8
    synonymy_edge_topk: 2047
    linking_top_k: *LINKING_TOP_K
    retrieval_top_k: *RETRIEVAL_TOP_K
    qa_top_k: *TOP_K
    damping: 0.5
    passage_node_weight: 0.05
    force_index_from_scratch: false
    force_openie_from_scratch: false
    save_openie: true
    embedding_provider: openai
    embedding_max_seq_len: *EMBEDDING_MAX_INPUT_TOKENS
    max_context_tokens: *MAX_CONTEXT_TOKENS
    context_reserve_tokens: *CONTEXT_RESERVE_TOKENS
    # a passage that exceeds the context budget is cut to the remaining tokens only when more than this many remain
    min_truncated_passage_tokens: <HIPPORAG_MIN_TRUNCATED_PASSAGE_TOKENS>

  remem:
    agent_max_steps: 5
    agent_fixed_tools: false
    temperature: 0.3
    max_tokens: 10000
    dataset: memebench
    extract_method: episodic_gist
    is_directed_graph: false
    synonymy_edge_sim_threshold: 0.8
    synonymy_edge_topk: 10
    linking_top_k: *LINKING_TOP_K
    retrieval_top_k: *RETRIEVAL_TOP_K
    qa_top_k: *TOP_K
    damping: 0.5
    passage_node_weight: 0.05
    save_openie: true
    embedding_provider: openai
    embedding_max_seq_len: *EMBEDDING_MAX_INPUT_TOKENS
    max_context_tokens: *MAX_CONTEXT_TOKENS

  letta:
    context_window: *ANSWER_MODEL_MAX_CONTEXT_TOKENS
    message_buffer_autoclear: true
    max_steps: 50
    temperature: 0.3
    max_tokens: 10000
    retrieve_num: *TOP_K
    embedding_chunk_size: *PASSAGE_CHUNK_TOKENS
    memorize_chunk_tokens: *CHUNK_TOKENS
    memorize_chunk_overlap_tokens: *CHUNK_OVERLAP_TOKENS

  mirix:
    answer_agent: true
    answer_max_steps: 4
    memory_types: [core, episodic, semantic, resource, knowledge_vault]
    temperature: 0.3
    max_tokens: 10000
    retrieve_num: *TOP_K
    memorize_chunk_tokens: *CHUNK_TOKENS
    memorize_chunk_overlap_tokens: *CHUNK_OVERLAP_TOKENS
    query_memory_item_tokens: 4096
    query_memory_context_tokens: 10240
    max_input_tokens: *MAX_INPUT_TOKENS
    context_window: *ANSWER_MODEL_MAX_CONTEXT_TOKENS
    embedding_max_input_tokens: *EMBEDDING_MAX_INPUT_TOKENS

  amem:
    retrieve_num: *TOP_K
    amem_evo_threshold: 100
    temperature: 0.3
    max_tokens: 10000
    memorize_chunk_tokens: *CHUNK_TOKENS
    embedding_max_input_tokens: *EMBEDDING_MAX_INPUT_TOKENS

  lightmem:
    retrieve_num: *TOP_K
    pre_compress: false
    compress_rate: 0.6
    compress_device: cpu
    topic_segment: true
    # Hugging Face revision of LightMem's topic segmenter (llmlingua-2-bert-base-multilingual-cased-meetingbank)
    segmenter_revision: 5f0c82792b7ea14c6484e015b6a072009496b7f2
    index_strategy: embedding
    retrieve_strategy: embedding
    update_mode: offline
    extraction_mode: flat
    messages_use: hybrid
    # longer patient turns are split at sentence boundaries
    max_user_tokens: <LIGHTMEM_MAX_USER_TOKENS>
    # short-term buffer size
    shortmem_max_tokens: <LIGHTMEM_SHORTMEM_MAX_TOKENS>
    temperature: 0.3
    max_tokens: 10000
    max_context_tokens: *MAX_CONTEXT_TOKENS
    context_reserve_tokens: *CONTEXT_RESERVE_TOKENS
    # factor on the character cut when the retrieved text exceeds the context budget
    truncation_ratio: <LIGHTMEM_TRUNCATION_RATIO>
    offline_consolidate: true
    consolidate_top_k: 20
    consolidate_keep_top_n: 10
    consolidate_score_threshold: 0.9
    consolidate_max_calls: 20000
    consolidate_max_workers: 20

  mem0:
    retrieve_num: *TOP_K
    memorize_chunk_tokens: *CHUNK_TOKENS
    max_context_tokens: *MAX_CONTEXT_TOKENS
    temperature: 0.3
    max_tokens: 10000
    embedding_max_input_tokens: *EMBEDDING_MAX_INPUT_TOKENS