Spaces:

clinical-agent-memory-bench
/

meme-bench

Running

meme-bench / code_release /code /memebench /configs /methods.yaml

Upload 99 files

750b88d verified 13 days ago

6.3 kB

# Method parameters; the models are set in backbone.yaml. Values written as <...> must be set before a run.
# Run from code/: python -m memebench.run_eval --method <method> --patient <P01..P23>

# Temperature and output-token limit of the runner's answering call (answer_temperature: null sends no temperature,
# for models that accept only their default). hipporag, remem, letta and mirix answer inside the method,
# with their max_tokens; lightmem also holds answer_max_tokens back from its context budget.
answer_temperature: 0.3
answer_max_tokens: 10000

# Shared values: every method below that has the setting refers to it by name (*NAME), so one value applies to
# all methods. top_k is the number of items a method retrieves for an answer (run_eval --top-k overrides it).
top_k: &TOP_K 5
# fact-linking and passage-retrieval depths of HippoRAG 2 and REMem
linking_top_k: &LINKING_TOP_K <LINKING_TOP_K>
retrieval_top_k: &RETRIEVAL_TOP_K <RETRIEVAL_TOP_K>
# token budget of the history or retrieved text in the answer prompt (long_context fills it with the most
# recent visits)
max_context_tokens: &MAX_CONTEXT_TOKENS <MAX_CONTEXT_TOKENS>
# context window that Letta and MIRIX give their agents
answer_model_max_context_tokens: &ANSWER_MODEL_MAX_CONTEXT_TOKENS <ANSWER_MODEL_MAX_CONTEXT_TOKENS>
# chunk size of a visit before Letta, MIRIX, A-MEM and Mem0 store it, and the chunk overlap for Letta and MIRIX
chunk_tokens: &CHUNK_TOKENS <CHUNK_TOKENS>
chunk_overlap_tokens: &CHUNK_OVERLAP_TOKENS <CHUNK_OVERLAP_TOKENS>
# Letta's archival passage size
passage_chunk_tokens: &PASSAGE_CHUNK_TOKENS <PASSAGE_CHUNK_TOKENS>
# MIRIX's per-visit input limit
max_input_tokens: &MAX_INPUT_TOKENS <MAX_INPUT_TOKENS>
# longest embedding input, in embedding.tokenizer tokens: A-MEM and Mem0 truncate longer texts to it; it is the
# embedding_max_seq_len of HippoRAG 2 and REMem and the embedding_chunk_size of MIRIX
embedding_max_input_tokens: &EMBEDDING_MAX_INPUT_TOKENS <EMBEDDING_MAX_INPUT_TOKENS>
# tokens of max_context_tokens held back for the answer prompt by HippoRAG 2 and LightMem
context_reserve_tokens: &CONTEXT_RESERVE_TOKENS <CONTEXT_RESERVE_TOKENS>

methods:
  long_context:
    max_context_tokens: *MAX_CONTEXT_TOKENS
    # tokens held back from max_context_tokens for the prompt and the question
    margin: 0

  bm25_rag:
    top_k: *TOP_K
    k1: 1.5
    b: 0.75
    chunk_size: 512
    chunk_overlap: 50
    max_context_tokens: *MAX_CONTEXT_TOKENS

  dense_rag:
    top_k: *TOP_K
    chunk_size: 4096
    chunk_overlap: 50
    max_context_tokens: *MAX_CONTEXT_TOKENS

  hipporag:
    temperature: 0.3
    max_tokens: 10000
    openie_mode: online
    is_directed_graph: false
    synonymy_edge_sim_threshold: 0.8
    synonymy_edge_topk: 2047
    linking_top_k: *LINKING_TOP_K
    retrieval_top_k: *RETRIEVAL_TOP_K
    qa_top_k: *TOP_K
    damping: 0.5
    passage_node_weight: 0.05
    force_index_from_scratch: false
    force_openie_from_scratch: false
    save_openie: true
    embedding_provider: openai
    embedding_max_seq_len: *EMBEDDING_MAX_INPUT_TOKENS
    max_context_tokens: *MAX_CONTEXT_TOKENS
    context_reserve_tokens: *CONTEXT_RESERVE_TOKENS
    # a passage that exceeds the context budget is cut to the remaining tokens only when more than this many remain
    min_truncated_passage_tokens: <HIPPORAG_MIN_TRUNCATED_PASSAGE_TOKENS>

  remem:
    agent_max_steps: 5
    agent_fixed_tools: false
    temperature: 0.3
    max_tokens: 10000
    dataset: memebench
    extract_method: episodic_gist
    is_directed_graph: false
    synonymy_edge_sim_threshold: 0.8
    synonymy_edge_topk: 10
    linking_top_k: *LINKING_TOP_K
    retrieval_top_k: *RETRIEVAL_TOP_K
    qa_top_k: *TOP_K
    damping: 0.5
    passage_node_weight: 0.05
    save_openie: true
    embedding_provider: openai
    embedding_max_seq_len: *EMBEDDING_MAX_INPUT_TOKENS
    max_context_tokens: *MAX_CONTEXT_TOKENS

  letta:
    context_window: *ANSWER_MODEL_MAX_CONTEXT_TOKENS
    message_buffer_autoclear: true
    max_steps: 50
    temperature: 0.3
    max_tokens: 10000
    retrieve_num: *TOP_K
    embedding_chunk_size: *PASSAGE_CHUNK_TOKENS
    memorize_chunk_tokens: *CHUNK_TOKENS
    memorize_chunk_overlap_tokens: *CHUNK_OVERLAP_TOKENS

  mirix:
    answer_agent: true
    answer_max_steps: 4
    memory_types: [core, episodic, semantic, resource, knowledge_vault]
    temperature: 0.3
    max_tokens: 10000
    retrieve_num: *TOP_K
    memorize_chunk_tokens: *CHUNK_TOKENS
    memorize_chunk_overlap_tokens: *CHUNK_OVERLAP_TOKENS
    query_memory_item_tokens: 4096
    query_memory_context_tokens: 10240
    max_input_tokens: *MAX_INPUT_TOKENS
    context_window: *ANSWER_MODEL_MAX_CONTEXT_TOKENS
    embedding_max_input_tokens: *EMBEDDING_MAX_INPUT_TOKENS

  amem:
    retrieve_num: *TOP_K
    amem_evo_threshold: 100
    temperature: 0.3
    max_tokens: 10000
    memorize_chunk_tokens: *CHUNK_TOKENS
    embedding_max_input_tokens: *EMBEDDING_MAX_INPUT_TOKENS

  lightmem:
    retrieve_num: *TOP_K
    pre_compress: false
    compress_rate: 0.6
    compress_device: cpu
    topic_segment: true
    # Hugging Face revision of LightMem's topic segmenter (llmlingua-2-bert-base-multilingual-cased-meetingbank)
    segmenter_revision: 5f0c82792b7ea14c6484e015b6a072009496b7f2
    index_strategy: embedding
    retrieve_strategy: embedding
    update_mode: offline
    extraction_mode: flat
    messages_use: hybrid
    # longer patient turns are split at sentence boundaries
    max_user_tokens: <LIGHTMEM_MAX_USER_TOKENS>
    # short-term buffer size
    shortmem_max_tokens: <LIGHTMEM_SHORTMEM_MAX_TOKENS>
    temperature: 0.3
    max_tokens: 10000
    max_context_tokens: *MAX_CONTEXT_TOKENS
    context_reserve_tokens: *CONTEXT_RESERVE_TOKENS
    # factor on the character cut when the retrieved text exceeds the context budget
    truncation_ratio: <LIGHTMEM_TRUNCATION_RATIO>
    offline_consolidate: true
    consolidate_top_k: 20
    consolidate_keep_top_n: 10
    consolidate_score_threshold: 0.9
    consolidate_max_calls: 20000
    consolidate_max_workers: 20

  mem0:
    retrieve_num: *TOP_K
    memorize_chunk_tokens: *CHUNK_TOKENS
    max_context_tokens: *MAX_CONTEXT_TOKENS
    temperature: 0.3
    max_tokens: 10000
    embedding_max_input_tokens: *EMBEDDING_MAX_INPUT_TOKENS

	# Method parameters; the models are set in backbone.yaml. Values written as <...> must be set before a run.
	# Run from code/: python -m memebench.run_eval --method <method> --patient <P01..P23>

	# Temperature and output-token limit of the runner's answering call (answer_temperature: null sends no temperature,
	# for models that accept only their default). hipporag, remem, letta and mirix answer inside the method,
	# with their max_tokens; lightmem also holds answer_max_tokens back from its context budget.
	answer_temperature: 0.3
	answer_max_tokens: 10000

	# Shared values: every method below that has the setting refers to it by name (*NAME), so one value applies to
	# all methods. top_k is the number of items a method retrieves for an answer (run_eval --top-k overrides it).
	top_k: &TOP_K 5
	# fact-linking and passage-retrieval depths of HippoRAG 2 and REMem
	linking_top_k: &LINKING_TOP_K <LINKING_TOP_K>
	retrieval_top_k: &RETRIEVAL_TOP_K <RETRIEVAL_TOP_K>
	# token budget of the history or retrieved text in the answer prompt (long_context fills it with the most
	# recent visits)
	max_context_tokens: &MAX_CONTEXT_TOKENS <MAX_CONTEXT_TOKENS>
	# context window that Letta and MIRIX give their agents
	answer_model_max_context_tokens: &ANSWER_MODEL_MAX_CONTEXT_TOKENS <ANSWER_MODEL_MAX_CONTEXT_TOKENS>
	# chunk size of a visit before Letta, MIRIX, A-MEM and Mem0 store it, and the chunk overlap for Letta and MIRIX
	chunk_tokens: &CHUNK_TOKENS <CHUNK_TOKENS>
	chunk_overlap_tokens: &CHUNK_OVERLAP_TOKENS <CHUNK_OVERLAP_TOKENS>
	# Letta's archival passage size
	passage_chunk_tokens: &PASSAGE_CHUNK_TOKENS <PASSAGE_CHUNK_TOKENS>
	# MIRIX's per-visit input limit
	max_input_tokens: &MAX_INPUT_TOKENS <MAX_INPUT_TOKENS>
	# longest embedding input, in embedding.tokenizer tokens: A-MEM and Mem0 truncate longer texts to it; it is the
	# embedding_max_seq_len of HippoRAG 2 and REMem and the embedding_chunk_size of MIRIX
	embedding_max_input_tokens: &EMBEDDING_MAX_INPUT_TOKENS <EMBEDDING_MAX_INPUT_TOKENS>
	# tokens of max_context_tokens held back for the answer prompt by HippoRAG 2 and LightMem
	context_reserve_tokens: &CONTEXT_RESERVE_TOKENS <CONTEXT_RESERVE_TOKENS>

	methods:
	long_context:
	max_context_tokens: *MAX_CONTEXT_TOKENS
	# tokens held back from max_context_tokens for the prompt and the question
	margin: 0

	bm25_rag:
	top_k: *TOP_K
	k1: 1.5
	b: 0.75
	chunk_size: 512
	chunk_overlap: 50
	max_context_tokens: *MAX_CONTEXT_TOKENS

	dense_rag:
	top_k: *TOP_K
	chunk_size: 4096
	chunk_overlap: 50
	max_context_tokens: *MAX_CONTEXT_TOKENS

	hipporag:
	temperature: 0.3
	max_tokens: 10000
	openie_mode: online
	is_directed_graph: false
	synonymy_edge_sim_threshold: 0.8
	synonymy_edge_topk: 2047
	linking_top_k: *LINKING_TOP_K
	retrieval_top_k: *RETRIEVAL_TOP_K
	qa_top_k: *TOP_K
	damping: 0.5
	passage_node_weight: 0.05
	force_index_from_scratch: false
	force_openie_from_scratch: false
	save_openie: true
	embedding_provider: openai
	embedding_max_seq_len: *EMBEDDING_MAX_INPUT_TOKENS
	max_context_tokens: *MAX_CONTEXT_TOKENS
	context_reserve_tokens: *CONTEXT_RESERVE_TOKENS
	# a passage that exceeds the context budget is cut to the remaining tokens only when more than this many remain
	min_truncated_passage_tokens: <HIPPORAG_MIN_TRUNCATED_PASSAGE_TOKENS>

	remem:
	agent_max_steps: 5
	agent_fixed_tools: false
	temperature: 0.3
	max_tokens: 10000
	dataset: memebench
	extract_method: episodic_gist
	is_directed_graph: false
	synonymy_edge_sim_threshold: 0.8
	synonymy_edge_topk: 10
	linking_top_k: *LINKING_TOP_K
	retrieval_top_k: *RETRIEVAL_TOP_K
	qa_top_k: *TOP_K
	damping: 0.5
	passage_node_weight: 0.05
	save_openie: true
	embedding_provider: openai
	embedding_max_seq_len: *EMBEDDING_MAX_INPUT_TOKENS
	max_context_tokens: *MAX_CONTEXT_TOKENS

	letta:
	context_window: *ANSWER_MODEL_MAX_CONTEXT_TOKENS
	message_buffer_autoclear: true
	max_steps: 50
	temperature: 0.3
	max_tokens: 10000
	retrieve_num: *TOP_K
	embedding_chunk_size: *PASSAGE_CHUNK_TOKENS
	memorize_chunk_tokens: *CHUNK_TOKENS
	memorize_chunk_overlap_tokens: *CHUNK_OVERLAP_TOKENS

	mirix:
	answer_agent: true
	answer_max_steps: 4
	memory_types: [core, episodic, semantic, resource, knowledge_vault]
	temperature: 0.3
	max_tokens: 10000
	retrieve_num: *TOP_K
	memorize_chunk_tokens: *CHUNK_TOKENS
	memorize_chunk_overlap_tokens: *CHUNK_OVERLAP_TOKENS
	query_memory_item_tokens: 4096
	query_memory_context_tokens: 10240
	max_input_tokens: *MAX_INPUT_TOKENS
	context_window: *ANSWER_MODEL_MAX_CONTEXT_TOKENS
	embedding_max_input_tokens: *EMBEDDING_MAX_INPUT_TOKENS

	amem:
	retrieve_num: *TOP_K
	amem_evo_threshold: 100
	temperature: 0.3
	max_tokens: 10000
	memorize_chunk_tokens: *CHUNK_TOKENS
	embedding_max_input_tokens: *EMBEDDING_MAX_INPUT_TOKENS

	lightmem:
	retrieve_num: *TOP_K
	pre_compress: false
	compress_rate: 0.6
	compress_device: cpu
	topic_segment: true
	# Hugging Face revision of LightMem's topic segmenter (llmlingua-2-bert-base-multilingual-cased-meetingbank)
	segmenter_revision: 5f0c82792b7ea14c6484e015b6a072009496b7f2
	index_strategy: embedding
	retrieve_strategy: embedding
	update_mode: offline
	extraction_mode: flat
	messages_use: hybrid
	# longer patient turns are split at sentence boundaries
	max_user_tokens: <LIGHTMEM_MAX_USER_TOKENS>
	# short-term buffer size
	shortmem_max_tokens: <LIGHTMEM_SHORTMEM_MAX_TOKENS>
	temperature: 0.3
	max_tokens: 10000
	max_context_tokens: *MAX_CONTEXT_TOKENS
	context_reserve_tokens: *CONTEXT_RESERVE_TOKENS
	# factor on the character cut when the retrieved text exceeds the context budget
	truncation_ratio: <LIGHTMEM_TRUNCATION_RATIO>
	offline_consolidate: true
	consolidate_top_k: 20
	consolidate_keep_top_n: 10
	consolidate_score_threshold: 0.9
	consolidate_max_calls: 20000
	consolidate_max_workers: 20

	mem0:
	retrieve_num: *TOP_K
	memorize_chunk_tokens: *CHUNK_TOKENS
	max_context_tokens: *MAX_CONTEXT_TOKENS
	temperature: 0.3
	max_tokens: 10000
	embedding_max_input_tokens: *EMBEDDING_MAX_INPUT_TOKENS