hypernet_sp/debug_followup.py

"""Diagnose composite FAIL S2.math-followup (answered 24 instead of 26): rerun the muffin
pair with full think-stream dumps + DecodePolicy firing telemetry, so we can see WHETHER
the convergence trigger fired on the intermediate value (24) and force-closed the think
before the change computation reached 26.   python3 debug_followup.py [seed]"""
import os, sys
import torch

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "runtime"))
import memory_core as mc
import decode_policy
from app_session_torch import AppSession


def main():
    torch.set_num_threads(os.cpu_count())
    import joblib
    from transformers import AutoModelForCausalLM, AutoTokenizer
    from rag import BGERetriever
    sys.path.pop(1)
    from attn_export3_torch import load_pooler
    seed = int(sys.argv[1]) if len(sys.argv) > 1 else 1

    fired_log = []
    orig = decode_policy.DecodePolicy.note_text

    def spy(self, text):
        r = orig(self, text)
        if r:
            fired_log.append({"counts": dict(self.counts), "at_chars": len(text),
                              "converged": self.converged_answer()})
        return r
    decode_policy.DecodePolicy.note_text = spy

    tok = AutoTokenizer.from_pretrained("fft_hf")
    llm = AutoModelForCausalLM.from_pretrained("fft_hf", dtype=torch.float32).eval()
    pooler, bge = load_pooler(), BGERetriever()
    mem = mc.TieredMemory("/dev/null", bge=bge)
    mem.session.append("My hotel room number for tonight is 1408.")   # state as in S2
    mem.pin("My hotel room number for tonight is 1408.")
    s = AppSession(llm, tok, pooler, bge, joblib.load("evals/intent_clf.joblib"),
                   joblib.load("evals/specificity_clf.joblib"), mem, seed=seed)

    for msg, want in [
        ("A bakery sells muffins for $4 each. Maria buys 6 muffins. How much does she spend in total?", "24"),
        ("I pay with a $50 bill. How much change do I get back?", "26"),
    ]:
        start = len(s.gen)
        ans, src, chunks = s.turn(msg, store="none")
        body = tok.decode(s.gen[start:])
        print("=" * 70)
        print(f"Q: {msg}\nsrc={src}\nchunks={chunks}\nANSWER: {ans!r}  (want {want}: "
              f"{'HIT' if want in ans else 'MISS'})")
        print(f"policy fired: {fired_log[-1] if fired_log else 'no'}")
        fired_log.clear()
        print(f"--- turn body ({len(s.gen) - start} tok) ---\n{body[:3000]}")
    print("DEBUG_FOLLOWUP_DONE")


if __name__ == "__main__":
    main()