File size: 1,261 Bytes
ad90e7a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 | from nzfc_gram_runtime import NZFCGramLongMemoryChat
from nzfc_gram_runtime.nonquant import attach_nonquant_gemma
from nzfc_gram_runtime.cache_profiles import attach_adaptive_kv_cache_generation
from nzfc_gram_runtime.quality import attach_answer_quality_governor
MODEL_ID = 'google/gemma-4-E2B-it'
bot = NZFCGramLongMemoryChat(
repo_dir='.',
model_id=MODEL_ID,
memory_db_path='./user_memory_adaptive_cache.sqlite3',
load_model=False,
require_model=False,
preload_static_memory=True,
)
attach_nonquant_gemma(
bot,
model_id=MODEL_ID,
device_map='balanced_low_0',
gpu_max_memory_gib=11,
gpu_max_memory_gib_candidates=[11, 10, 9, 8],
cpu_max_memory_gib=48,
prefer_bf16=True,
use_fp32=False,
)
attach_adaptive_kv_cache_generation(bot, default_cache_policy='adaptive', default_prefer_cache=True)
attach_answer_quality_governor(bot)
out = bot.generate_answer(
system_prompt='You are a concise assistant.',
user_prompt='Explain memory as evidence in one sentence.',
max_new_tokens=80,
cache_policy='adaptive',
prefer_cache=True,
)
print(out['answer'])
print('use_cache:', out.get('use_cache'))
print('fallback_used:', out.get('fallback_used'))
print('latency_s:', out.get('latency_s'))
|