hypernet_sp/evals/fluency_chat.py

"""Conversational FLUENCY check (not task-accuracy). A natural multi-turn chat with NO 'just the number'
constraints — print the FULL assistant response each turn so we can read whether the dialogue is fluent /
coherent / chat-like (vs reasoning-leak, boxed-math style, abrupt salvaged answers). Uses the app's normal
routing (intent -> retrieve/compute/chitchat). SP+pins (shipping, rw=1024)."""
import sys, os, time
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import tiered_rag_mlx as T
T.ANSCAP = 1200

TURNS = [
 "Hey! I'm thinking about a weekend trip to Kyoto. Got any suggestions?",
 "My budget is around 800 dollars.",
 "What are a couple of must-see temples there?",
 "Remind me — what did I say my budget was?",
 "If I spend 120 dollars a day on food for 3 days, am I still within budget?",
 "Thanks. Can you give me a quick summary of my plan so far?",
]

chat = T.ChatSession(T.TieredMemory("/tmp/fluency.jsonl"), rw=1024)
for i, u in enumerate(TURNS):
    t0 = time.time()
    store = "session" if T.intent_of(u) == "fact" else "none"
    ack = T.intent_of(u) == "fact"
    a, src, _ = chat.turn(u, store=store, ack_only=ack)
    print("=" * 78, flush=True)
    print(f"USER: {u}", flush=True)
    print(f"  [intent={T.intent_of(u)} · src={src or '—'} · {time.time()-t0:.0f}s]", flush=True)
    print(f"ASSISTANT: {a}", flush=True)
print("\nFLUENCY_DONE", flush=True)