File size: 1,368 Bytes
c8cdc68 c679ff7 c8cdc68 c679ff7 c8cdc68 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | # examples/nonquant_bf16_final_usage.py
from nzfc_gram_runtime import NZFCGramLongMemoryChat
from nzfc_gram_runtime.nonquant import attach_nonquant_gemma, patch_generation_use_cache_false
bot = NZFCGramLongMemoryChat(
repo_dir='.',
model_id='google/gemma-4-E2B-it',
load_model=False,
require_model=False,
preload_static_memory=True,
)
meta = attach_nonquant_gemma(
bot,
model_id='google/gemma-4-E2B-it',
device_map='balanced_low_0',
gpu_max_memory_gib=11,
cpu_max_memory_gib=48,
prefer_bf16=True,
use_fp32=False,
)
patch_generation_use_cache_false(bot)
print('MODEL META:', meta)
out1 = bot.chat(
'\uc55e\uc73c\ub85c \ub0b4 \uc7a5\uae30 \ubcc4\uba85\uc740 AlphaFox_Final \uc774\ub77c\uace0 \uae30\uc5b5\ud574\uc918.',
user_id='user001',
project_id='demo',
session_id='s1',
save_turn=True,
save_scope='project',
response_language='ko',
max_new_tokens=48,
)
print('\nTURN 1:')
print(out1['answer'])
out2 = bot.chat(
'\ub0b4\uac00 \uc804\uc5d0 \ub9d0\ud55c \uc7a5\uae30 \ubcc4\uba85\uc774 \ubb50\uc600\uc9c0?',
user_id='user001',
project_id='demo',
session_id='s2',
save_turn=False,
response_language='ko',
max_new_tokens=64,
)
print('\nTURN 2:')
print(out2['answer'])
print('\nSTATUS:', out2['status'])
print('\nMEMORY BOUNDARY:', out2['memory_boundary'])
|