Update app.py

app.py

@@ -10,10 +10,6 @@ from typing import List, Dict, Any, Optional, Tuple
 
 import gradio as gr
 
-# =========================
-# RFT Memory Receipt Engine
-# =========================
-
 BASE_DIR = os.environ.get("RFT_MEM_BASE", "var/rftmem")
 os.makedirs(BASE_DIR, exist_ok=True)
 
@@ -36,10 +32,7 @@ def atomic_write(path: str, data: bytes) -> None:
 
 
 def safe_fts_match(user_query: str) -> str:
-    """
-    FTS5 MATCH can error on punctuation/special syntax.
-    Convert input into a conservative OR query: word1 OR word2 OR ...
-    """
+    # Conservative FTS query: token1 OR token2 OR ...
     words = re.findall(r"[A-Za-z0-9_]+", (user_query or "").lower())
     if not words:
         return "___NO_HITS___"
@@ -66,9 +59,7 @@ class RetrievalHit:
 
 class RFTMemoryStore:
     """
-    Source of truth: append-only JSONL per session + hash-chained ledger.
-    Index: SQLite (events table + FTS5) for lexical retrieval.
-    Receipts: JSON files saved per assistant turn, downloadable and verifiable.
+    Append-only session ledger + SQLite FTS retrieval + receipt verification.
     """
 
     def __init__(self, base_dir: str):
@@ -76,9 +67,12 @@ class RFTMemoryStore:
         self.db_path = os.path.join(base_dir, "index.sqlite")
         self._init_db()
 
+    def _connect(self) -> sqlite3.Connection:
+        return sqlite3.connect(self.db_path)
+
     def _init_db(self):
         os.makedirs(self.base_dir, exist_ok=True)
-        con = sqlite3.connect(self.db_path)
+        con = self._connect()
         cur = con.cursor()
 
         cur.execute("""
@@ -96,15 +90,6 @@ class RFTMemoryStore:
         )
         """)
 
-        cur.execute("""
-        CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
-            event_id,
-            session_id,
-            text,
-            content=''
-        )
-        """)
-
         cur.execute("""
         CREATE TABLE IF NOT EXISTS receipts (
             receipt_id TEXT PRIMARY KEY,
@@ -116,9 +101,43 @@ class RFTMemoryStore:
         )
         """)
 
-        con.commit()
+        # --- Ensure FTS exists in a join-safe form ---
+        cur.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name='events_fts'")
+        row = cur.fetchone()
+        needs_rebuild = False
+
+        if row is None:
+            needs_rebuild = True
+        else:
+            sql = (row[0] or "").lower()
+            # If it contains content='' or content="" then it's contentless and join-on-columns is unreliable
+            if "content=''" in sql or 'content=""' in sql:
+                needs_rebuild = True
+
+        if needs_rebuild:
+            cur.execute("DROP TABLE IF EXISTS events_fts")
+            # Stored-content FTS table (simple and reliable)
+            cur.execute("""
+            CREATE VIRTUAL TABLE events_fts USING fts5(
+                event_id,
+                session_id,
+                text
+            )
+            """)
+            con.commit()
+            # Reindex from events
+            cur.execute("DELETE FROM events_fts")
+            cur.execute("""
+                INSERT INTO events_fts(event_id, session_id, text)
+                SELECT event_id, session_id, text FROM events
+            """)
+            con.commit()
+
         con.close()
 
+    # ----------------
+    # Filesystem layout
+    # ----------------
     def session_dir(self, session_id: str) -> str:
         d = os.path.join(self.base_dir, "sessions", session_id)
         os.makedirs(d, exist_ok=True)
@@ -132,8 +151,11 @@ class RFTMemoryStore:
         os.makedirs(d, exist_ok=True)
         return d
 
+    # -------------------
+    # RFT collapse scoring
+    # -------------------
     def get_events(self, session_id: str, limit: int = 400) -> List[Dict[str, Any]]:
-        con = sqlite3.connect(self.db_path)
+        con = self._connect()
         cur = con.cursor()
         cur.execute("""
             SELECT event_id, seq, ts_ms, role, text, digest, prev_hash, chain_hash, collapse
@@ -178,8 +200,11 @@ class RFTMemoryStore:
         score = role_w * (0.65 * novelty + 0.35 * length_factor)
         return float(max(0.0, min(1.0, score)))
 
+    # ----------------
+    # Append-only write
+    # ----------------
     def _get_last_seq_and_chain(self, session_id: str) -> Tuple[int, str]:
-        con = sqlite3.connect(self.db_path)
+        con = self._connect()
         cur = con.cursor()
 
         cur.execute("SELECT COALESCE(MAX(seq), 0) FROM events WHERE session_id=?", (session_id,))
@@ -225,7 +250,7 @@ class RFTMemoryStore:
             "collapse": collapse
         }
 
-        # Append-only JSONL
+        # Append-only JSONL source of truth
         log_path = self.session_log_path(session_id)
         line = (json.dumps(rec, ensure_ascii=False) + "\n").encode("utf-8")
         with open(log_path, "ab") as f:
@@ -233,8 +258,8 @@ class RFTMemoryStore:
             f.flush()
             os.fsync(f.fileno())
 
-        # Index update
-        con = sqlite3.connect(self.db_path)
+        # Index
+        con = self._connect()
         cur = con.cursor()
         cur.execute("""
             INSERT INTO events(session_id,event_id,seq,ts_ms,role,text,digest,prev_hash,chain_hash,collapse)
@@ -246,11 +271,17 @@ class RFTMemoryStore:
 
         return rec
 
+    # -------------------------
+    # Retrieval (FTS + fallback)
+    # -------------------------
     def search_lexical(self, session_id: str, query: str, k: int = 8) -> List[RetrievalHit]:
         match = safe_fts_match(query)
+        if match == "___NO_HITS___":
+            return []
 
-        con = sqlite3.connect(self.db_path)
+        con = self._connect()
         cur = con.cursor()
+
         cur.execute("""
             SELECT e.event_id, e.seq, e.role, e.text, e.ts_ms, e.digest, e.chain_hash,
                    bm25(events_fts) as rank
@@ -263,13 +294,36 @@ class RFTMemoryStore:
         rows = cur.fetchall()
         con.close()
 
-        hits = []
+        hits: List[RetrievalHit] = []
         for (eid, seq, role, text, ts, digest, chain_hash, rank) in rows:
             r = float(rank if rank is not None else 0.0)
             score = 1.0 / (1.0 + max(0.0, r))
             hits.append(RetrievalHit(eid, int(seq or 0), role, text, ts, digest, chain_hash, score))
+
+        # Fallback if FTS returns nothing (protects demo UX)
+        if not hits:
+            tokens = re.findall(r"[A-Za-z0-9_]+", (query or "").lower())
+            if tokens:
+                needle = f"%{tokens[-1]}%"
+                con = self._connect()
+                cur = con.cursor()
+                cur.execute("""
+                    SELECT event_id, seq, role, text, ts_ms, digest, chain_hash
+                    FROM events
+                    WHERE session_id=? AND LOWER(text) LIKE ?
+                    ORDER BY seq DESC
+                    LIMIT ?
+                """, (session_id, needle, int(k)))
+                rows2 = cur.fetchall()
+                con.close()
+                for (eid, seq, role, text, ts, digest, chain_hash) in rows2:
+                    hits.append(RetrievalHit(eid, int(seq or 0), role, text, ts, digest, chain_hash, 0.001))
+
         return hits
 
+    # -------------------------
+    # Receipts + verification
+    # -------------------------
     def write_receipt(self, session_id: str, user_text: str, retrieved: List[RetrievalHit], prompt: str, response: str) -> str:
         receipt_id = uuid.uuid4().hex
         ts = now_ms()
@@ -292,7 +346,7 @@ class RFTMemoryStore:
             "response_hash": sha256_str(response),
             "engine": {
                 "name": "RFT Memory Receipt Engine",
-                "version": "0.2",
+                "version": "0.3",
                 "method": "append-only ledger + FTS retrieval + hash-chain receipts"
             }
         }
@@ -300,7 +354,7 @@ class RFTMemoryStore:
         path = os.path.join(self.receipts_dir(session_id), f"{receipt_id}.json")
         atomic_write(path, json.dumps(receipt, indent=2, ensure_ascii=False).encode("utf-8"))
 
-        con = sqlite3.connect(self.db_path)
+        con = self._connect()
         cur = con.cursor()
         cur.execute("""
             INSERT INTO receipts(receipt_id, session_id, ts_ms, prompt_hash, response_hash, receipt_path)
@@ -316,7 +370,7 @@ class RFTMemoryStore:
         if not session_id:
             return False, "Missing session_id."
 
-        con = sqlite3.connect(self.db_path)
+        con = self._connect()
         cur = con.cursor()
 
         for item in receipt_json.get("retrieval", []):
@@ -378,12 +432,6 @@ HOW_TO_MD = """
 - Each turn generates a receipt listing exactly what was retrieved.
 - Receipt verification proves the referenced events exist and hashes match.
 
-## Who this is useful for
-- Agent builders needing restart continuity
-- Teams needing auditability (“show what influenced the output”)
-- Debugging long-lived workflows without replaying full history
-- Anyone controlling token spend via fixed retrieval budgets
-
 ## Why it matters
 Storing logs is easy. Proving influence is the hard part.
 Receipts turn memory into something inspectable and defensible.
@@ -395,12 +443,7 @@ def new_session_id() -> str:
 
 
 def events_to_messages(events: List[Dict[str, Any]]) -> List[Dict[str, str]]:
-    # Gradio expects: [{"role": "...", "content": "..."}, ...]
-    msgs = []
-    for e in events:
-        if e["role"] in ("user", "assistant"):
-            msgs.append({"role": e["role"], "content": e["text"]})
-    return msgs
+    return [{"role": e["role"], "content": e["text"]} for e in events if e["role"] in ("user", "assistant")]
 
 
 def format_ledger(events: List[Dict[str, Any]]) -> str:
@@ -427,7 +470,65 @@ def build_prompt(user_msg: str, hits: List[RetrievalHit]) -> str:
     )
 
 
-def response_from_retrieval(hits: List[RetrievalHit]) -> str:
+def extract_fact_from_hits(hits: List[RetrievalHit], key: str) -> Optional[str]:
+    """
+    Extract simple facts from retrieved memory only.
+    Supports:
+      - "Key = Value"
+      - "Key: Value"
+      - "My name is Value"
+    """
+    key_l = key.lower()
+    patterns = [
+        rf"\b{re.escape(key_l)}\b\s*=\s*([A-Za-z0-9 _\-']+)",
+        rf"\b{re.escape(key_l)}\b\s*:\s*([A-Za-z0-9 _\-']+)",
+    ]
+    for h in hits:
+        t = (h.text or "").strip()
+        tl = t.lower()
+
+        if key_l == "name":
+            m = re.search(r"\bmy name is\b\s+([A-Za-z0-9 _\-']+)", tl)
+            if m:
+                return m.group(1).strip().title()
+
+        for p in patterns:
+            m = re.search(p, tl)
+            if m:
+                return m.group(1).strip().strip(",.")
+    return None
+
+
+def answer_from_memory(user_msg: str, hits: List[RetrievalHit]) -> str:
+    q = (user_msg or "").lower()
+
+    # Search-style queries
+    if q.startswith("search for") or q.startswith("search:"):
+        if not hits:
+            return "No matching memory slices were retrieved for this search."
+        lines = ["Search hits:"]
+        for h in hits:
+            lines.append(f"- {h.score:.4f} | {h.role} | {h.text}")
+        return "\n".join(lines)
+
+    # Simple Q&A based on retrieved memory only
+    if "dog" in q and "name" in q:
+        v = extract_fact_from_hits(hits, "dog")
+        return f"Your dog’s name (from stored memory) is: {v}" if v else "I didn’t retrieve a stored dog name for this query."
+
+    if "city" in q:
+        v = extract_fact_from_hits(hits, "city")
+        return f"Your city (from stored memory) is: {v}" if v else "I didn’t retrieve a stored city for this query."
+
+    if "drink" in q or "favourite drink" in q or "favorite drink" in q:
+        v = extract_fact_from_hits(hits, "drink") or extract_fact_from_hits(hits, "favourite drink") or extract_fact_from_hits(hits, "favorite drink")
+        return f"Your drink (from stored memory) is: {v}" if v else "I didn’t retrieve a stored drink for this query."
+
+    if "my name" in q:
+        v = extract_fact_from_hits(hits, "name")
+        return f"Your name (from stored memory) is: {v}" if v else "I didn’t retrieve a stored name for this query."
+
+    # Default: show what was retrieved (transparent)
     if not hits:
         return "No matching memory slices were retrieved for this query."
     lines = ["Retrieved memory slices:"]
@@ -438,13 +539,13 @@ def response_from_retrieval(hits: List[RetrievalHit]) -> str:
 
 def chat_turn(session_id: str, user_msg: str, retrieval_k: int):
     if not session_id:
-        session_id = new_session_id()
+        session_id = uuid.uuid4().hex
 
     store.append_event(session_id, "user", user_msg)
 
     hits = store.search_lexical(session_id, user_msg, k=int(retrieval_k))
     prompt = build_prompt(user_msg, hits)
-    response = response_from_retrieval(hits)
+    response = answer_from_memory(user_msg, hits)
 
     store.append_event(session_id, "assistant", response)
 
@@ -455,13 +556,13 @@ def chat_turn(session_id: str, user_msg: str, retrieval_k: int):
     retrieved_view = "\n".join([f"{h.score:.4f} | {h.role} | {h.text}" for h in hits]) if hits else "(none)"
     messages = events_to_messages(events)
 
-    # IMPORTANT: return receipt_path twice so File can download it
+    # Return receipt path twice: textbox + downloadable file
     return session_id, messages, retrieved_view, ledger, receipt_path, receipt_path
 
 
 def run_guided_demo(session_id: str, retrieval_k: int):
     if not session_id:
-        session_id = new_session_id()
+        session_id = uuid.uuid4().hex
 
     last = (session_id, [], "", "", "", None)
     for step in GUIDED_DEMO_STEPS:
@@ -499,7 +600,7 @@ def verify_uploaded_receipt(file_obj) -> str:
 
 
 def reset_session():
-    sid = new_session_id()
+    sid = uuid.uuid4().hex
     return sid, [], "", "", "", None
 
 
@@ -508,7 +609,7 @@ def fill_example(selected: str) -> str:
 
 
 with gr.Blocks(title="RFT Memory Receipt Engine") as demo:
-    gr.Markdown("# RFT Memory Receipt Engine\nLocal persistence + lexical retrieval + verifiable receipts.")
+    gr.Markdown("# RFT Memory Receipt Engine\nLocal persistence + retrieval + verifiable receipts.")
 
     with gr.Row():
         session_id = gr.Textbox(label="Session ID", value=new_session_id())
@@ -518,7 +619,6 @@ with gr.Blocks(title="RFT Memory Receipt Engine") as demo:
 
     with gr.Tabs():
         with gr.Tab("Chat"):
-            # DO NOT pass type=... (your Gradio doesn't accept it)
             chatbot = gr.Chatbot(label="Conversation", height=320)
 
             with gr.Row():
@@ -543,7 +643,7 @@ with gr.Blocks(title="RFT Memory Receipt Engine") as demo:
             )
 
         with gr.Tab("Guided Demo"):
-            gr.Markdown("Runs a scripted set of messages to show storage, recall, overrides, search, and receipts.")
+            gr.Markdown("Runs a scripted demo to show storage, recall, overrides, search, and receipts.")
             run_demo_btn = gr.Button("Run Guided Demo", variant="primary")
 
             demo_chatbot = gr.Chatbot(label="Demo conversation", height=320)