Add SQLite persistence — episodes survive Space restarts v1.1.0

- mragent_server.py: all episodes now stored in SQLite (/data/mragent.db)
- /data is the HF Spaces persistent volume (survives restarts)
- Falls back to /app/data if /data is not mounted
- Embeddings stored as JSON in DB alongside episode metadata
- Full CRUD via /ingest, /query, /reconstruct, /stats, /recent
- Dockerfile: creates /data and /app/data directories
- Version bumped to 1.1.0

Dockerfile

@@ -19,6 +19,9 @@ RUN python -c "from sentence_transformers import SentenceTransformer; SentenceTr
 
 COPY mragent_server.py .
 
+# /data is the HF Spaces persistent volume — create a fallback in case it's not mounted
+RUN mkdir -p /data /app/data
+
 EXPOSE 7860
 
 HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \

mragent_server.py

@@ -1,13 +1,17 @@
 """
 evolva-mragent: MRAgent Memory Graph FastAPI Server
 Implements semantic memory storage and retrieval using sentence embeddings.
+Episodes are persisted to SQLite so they survive Space restarts.
+
 Endpoints: /ingest, /query, /reconstruct, /stats, /recent, /health
 """
 
 import os
 import uuid
-import time
+import json
+import sqlite3
 import logging
+import threading
 from datetime import datetime, timezone
 from typing import Optional, List, Dict, Any
 
@@ -18,10 +22,29 @@ from pydantic import BaseModel, Field
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger("mragent")
 
+# ---------------------------------------------------------------------------
+# Configuration
+# ---------------------------------------------------------------------------
+# HF Spaces: /data is a persistent volume that survives restarts.
+# Fall back to /app/data if /data is not available.
+DATA_DIR = "/data" if os.path.isdir("/data") else "/app/data"
+os.makedirs(DATA_DIR, exist_ok=True)
+DB_PATH = os.path.join(DATA_DIR, "mragent.db")
+
+EMBEDDING_MODEL_NAME = "all-MiniLM-L6-v2"
+_model = None
+_model_lock = threading.Lock()
+
+# ---------------------------------------------------------------------------
+# FastAPI app
+# ---------------------------------------------------------------------------
 app = FastAPI(
     title="evolva-mragent Memory Server",
-    description="MRAgent semantic memory graph — stores and retrieves verified claim episodes",
-    version="1.0.0",
+    description=(
+        "MRAgent semantic memory graph — stores and retrieves verified claim episodes. "
+        "Episodes are persisted to SQLite and survive Space restarts."
+    ),
+    version="1.1.0",
 )
 
 app.add_middleware(
@@ -32,25 +55,53 @@ app.add_middleware(
 )
 
 # ---------------------------------------------------------------------------
-# In-memory store (persists for the lifetime of the container)
+# SQLite helpers
 # ---------------------------------------------------------------------------
-EPISODES: List[Dict[str, Any]] = []
-EMBEDDINGS: List[List[float]] = []
-EMBEDDING_MODEL_NAME = "all-MiniLM-L6-v2"
-_model = None  # lazy-loaded
 
+def get_db() -> sqlite3.Connection:
+    conn = sqlite3.connect(DB_PATH, check_same_thread=False)
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def init_db():
+    with get_db() as conn:
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS episodes (
+                episode_id   TEXT PRIMARY KEY,
+                text         TEXT NOT NULL,
+                origin       TEXT,
+                tags         TEXT,          -- JSON array stored as string
+                citation     TEXT,
+                verdict      TEXT,
+                embedding    TEXT,          -- JSON float array stored as string
+                ingested_at  TEXT NOT NULL
+            )
+        """)
+        conn.execute("""
+            CREATE INDEX IF NOT EXISTS idx_ingested_at ON episodes(ingested_at DESC)
+        """)
+        conn.commit()
+    logger.info(f"SQLite DB initialised at {DB_PATH}")
+
+
+# ---------------------------------------------------------------------------
+# Embedding helpers
+# ---------------------------------------------------------------------------
 
 def get_model():
     global _model
     if _model is None:
-        try:
-            from sentence_transformers import SentenceTransformer
-            logger.info("Loading sentence-transformers model …")
-            _model = SentenceTransformer(EMBEDDING_MODEL_NAME)
-            logger.info("Model loaded.")
-        except Exception as e:
-            logger.warning(f"sentence-transformers unavailable ({e}); using zero-vector fallback")
-            _model = None
+        with _model_lock:
+            if _model is None:
+                try:
+                    from sentence_transformers import SentenceTransformer
+                    logger.info("Loading sentence-transformers model …")
+                    _model = SentenceTransformer(EMBEDDING_MODEL_NAME)
+                    logger.info("Model loaded.")
+                except Exception as e:
+                    logger.warning(f"sentence-transformers unavailable ({e}); using zero-vector fallback")
+                    _model = None
     return _model
 
 
@@ -59,8 +110,7 @@ def embed(text: str) -> Optional[List[float]]:
     if model is None:
         return None
     try:
-        vec = model.encode([text])[0].tolist()
-        return vec
+        return model.encode([text])[0].tolist()
     except Exception as e:
         logger.error(f"Embedding error: {e}")
         return None
@@ -99,7 +149,7 @@ class IngestResponse(BaseModel):
 
 class QueryRequest(BaseModel):
     claim: str = Field(..., description="The claim or query string to search for")
-    top_k: int = Field(default=5, ge=1, le=50, description="Number of top episodes to return")
+    top_k: int = Field(default=5, ge=1, le=50)
 
 
 class EpisodeResult(BaseModel):
@@ -135,6 +185,7 @@ class StatsResponse(BaseModel):
     verdict_counts: Dict[str, int]
     has_embedding: bool
     embedding_model: str
+    db_path: str
 
 
 class RecentEpisode(BaseModel):
@@ -157,6 +208,19 @@ class HealthResponse(BaseModel):
     embedding_model: str
     has_embedding: bool
     version: str
+    db_path: str
+
+
+# ---------------------------------------------------------------------------
+# Startup
+# ---------------------------------------------------------------------------
+
+@app.on_event("startup")
+def startup():
+    init_db()
+    # Warm up the embedding model in the background
+    import threading
+    threading.Thread(target=get_model, daemon=True).start()
 
 
 # ---------------------------------------------------------------------------
@@ -169,19 +233,23 @@ def root():
         "message": "evolva-mragent Memory Server",
         "docs": "/docs",
         "health": "/health",
-        "version": "1.0.0",
+        "version": "1.1.0",
+        "persistence": "SQLite",
     }
 
 
 @app.get("/health", response_model=HealthResponse, tags=["system"])
 def health():
+    with get_db() as conn:
+        count = conn.execute("SELECT COUNT(*) FROM episodes").fetchone()[0]
     model_loaded = get_model() is not None
     return HealthResponse(
         status="ok",
-        episode_count=len(EPISODES),
+        episode_count=count,
         embedding_model=EMBEDDING_MODEL_NAME,
         has_embedding=model_loaded,
-        version="1.0.0",
+        version="1.1.0",
+        db_path=DB_PATH,
     )
 
 
@@ -190,24 +258,30 @@ def ingest(req: IngestRequest):
     try:
         ep_id = req.episode_id or str(uuid.uuid4())
         vec = embed(req.text)
-        episode = {
-            "episode_id": ep_id,
-            "text": req.text,
-            "origin": req.origin,
-            "tags": req.tags or [],
-            "citation": req.citation,
-            "verdict": req.verdict,
-            "ingested_at": datetime.now(timezone.utc).isoformat(),
-        }
-        EPISODES.append(episode)
-        EMBEDDINGS.append(vec or [])
+        now = datetime.now(timezone.utc).isoformat()
+
+        with get_db() as conn:
+            conn.execute(
+                """
+                INSERT OR REPLACE INTO episodes
+                  (episode_id, text, origin, tags, citation, verdict, embedding, ingested_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    ep_id,
+                    req.text,
+                    req.origin,
+                    json.dumps(req.tags or []),
+                    req.citation,
+                    req.verdict,
+                    json.dumps(vec) if vec else None,
+                    now,
+                ),
+            )
+            conn.commit()
+
         logger.info(f"Ingested episode {ep_id} (has_embedding={vec is not None})")
-        return IngestResponse(
-            success=True,
-            episode_id=ep_id,
-            has_embedding=vec is not None,
-            error=None,
-        )
+        return IngestResponse(success=True, episode_id=ep_id, has_embedding=vec is not None, error=None)
     except Exception as e:
         logger.error(f"Ingest error: {e}")
         return IngestResponse(success=False, episode_id=None, has_embedding=False, error=str(e))
@@ -216,33 +290,38 @@ def ingest(req: IngestRequest):
 @app.post("/query", response_model=QueryResponse, tags=["memory"])
 def query(req: QueryRequest):
     try:
-        if not EPISODES:
+        with get_db() as conn:
+            rows = conn.execute(
+                "SELECT episode_id, text, origin, citation, verdict, embedding FROM episodes"
+            ).fetchall()
+
+        if not rows:
             return QueryResponse(episodes=[], total_in_memory=0, error=None)
 
         q_vec = embed(req.claim)
-        if q_vec and any(EMBEDDINGS):
+        if q_vec:
             scored = []
-            for ep, ev in zip(EPISODES, EMBEDDINGS):
+            for row in rows:
+                ev = json.loads(row["embedding"]) if row["embedding"] else []
                 score = cosine_similarity(q_vec, ev) if ev else 0.0
-                scored.append((score, ep))
+                scored.append((score, row))
             scored.sort(key=lambda x: x[0], reverse=True)
             top = scored[: req.top_k]
         else:
-            # Fallback: return most recent episodes
-            top = [(0.0, ep) for ep in EPISODES[-req.top_k:]]
+            top = [(0.0, row) for row in rows[-req.top_k:]]
 
         results = [
             EpisodeResult(
-                episode_id=ep.get("episode_id"),
-                text=ep.get("text"),
-                origin=ep.get("origin"),
+                episode_id=row["episode_id"],
+                text=row["text"],
+                origin=row["origin"],
                 score=round(score, 4),
-                citation=ep.get("citation"),
-                verdict=ep.get("verdict"),
+                citation=row["citation"],
+                verdict=row["verdict"],
             )
-            for score, ep in top
+            for score, row in top
         ]
-        return QueryResponse(episodes=results, total_in_memory=len(EPISODES), error=None)
+        return QueryResponse(episodes=results, total_in_memory=len(rows), error=None)
     except Exception as e:
         logger.error(f"Query error: {e}")
         raise HTTPException(status_code=500, detail=str(e))
@@ -274,38 +353,47 @@ def reconstruct(req: ReconstructRequest):
 
 @app.get("/stats", response_model=StatsResponse, tags=["memory"])
 def stats():
+    with get_db() as conn:
+        rows = conn.execute(
+            "SELECT verdict, tags FROM episodes"
+        ).fetchall()
+
     verdict_counts: Dict[str, int] = {}
-    for ep in EPISODES:
-        v = ep.get("verdict") or "Unknown"
+    all_tags = []
+    for row in rows:
+        v = row["verdict"] or "Unknown"
         verdict_counts[v] = verdict_counts.get(v, 0) + 1
+        try:
+            all_tags.extend(json.loads(row["tags"] or "[]"))
+        except Exception:
+            pass
 
-    # Count unique tags as "key nodes", tag co-occurrences as "links"
-    all_tags = [t for ep in EPISODES for t in ep.get("tags", [])]
     unique_tags = len(set(all_tags))
     link_count = max(0, len(all_tags) - unique_tags)
 
     return StatsResponse(
-        episode_count=len(EPISODES),
+        episode_count=len(rows),
         key_node_count=unique_tags,
         link_count=link_count,
         verdict_counts=verdict_counts,
         has_embedding=get_model() is not None,
         embedding_model=EMBEDDING_MODEL_NAME,
+        db_path=DB_PATH,
     )
 
 
 @app.get("/recent", response_model=RecentResponse, tags=["memory"])
 def recent():
-    sorted_eps = sorted(
-        EPISODES,
-        key=lambda e: e.get("ingested_at", ""),
-        reverse=True,
-    )[:20]
+    with get_db() as conn:
+        total = conn.execute("SELECT COUNT(*) FROM episodes").fetchone()[0]
+        rows = conn.execute(
+            "SELECT episode_id, verdict, text, origin, ingested_at, citation "
+            "FROM episodes ORDER BY ingested_at DESC LIMIT 20"
+        ).fetchall()
 
     results = []
-    for ep in sorted_eps:
-        text = ep.get("text", "")
-        # Try to extract claim from text like "VERDICT: X\nCLAIM: ..."
+    for row in rows:
+        text = row["text"] or ""
         claim_str = text
         if "\nCLAIM:" in text:
             claim_str = text.split("\nCLAIM:", 1)[1].strip()[:200]
@@ -314,12 +402,12 @@ def recent():
 
         results.append(
             RecentEpisode(
-                episode_id=ep.get("episode_id"),
-                verdict=ep.get("verdict"),
+                episode_id=row["episode_id"],
+                verdict=row["verdict"],
                 claim=claim_str,
-                origin=ep.get("origin"),
-                ingested_at=ep.get("ingested_at"),
-                citation=ep.get("citation"),
+                origin=row["origin"],
+                ingested_at=row["ingested_at"],
+                citation=row["citation"],
             )
         )
-    return RecentResponse(episodes=results, total=len(EPISODES))
+    return RecentResponse(episodes=results, total=total)