feat: obs lookup index, startup backfill, CORS wildcards, sync state in SQLite

- Add global obs_id->folderPath/agentId coordinate lookup index (KV.obs_lookup)
so folder_search hydrates candidates via O(1) point lookups instead of O(N) scans
- Add backfill_obs_lookup_if_needed() on startup to populate missing lookup entries
- Add verify_index_sync_on_boot() in workers.py to detect and fix stale search index
after crash/abrupt shutdown - triggers background rebuild if counts diverge
- Print security warning on boot when AGENTMEMORY_SECRET is unset
- Update CORS default origins to allow vscode-webview://* and chrome-extension://*
prefix matching so IDE extensions connect without CORS errors
- Persist sync_state in SQLite sync_state_metadata table instead of local .sync_state
JSON file, ensuring sync state survives Space restarts
- Add tests/test_obs_lookup.py covering lookup write, delete propagation, backfill, and
desync detection (all 231 tests pass)

src/app.py

@@ -46,6 +46,10 @@ def create_app() -> Flask:
     """Create and return a fully configured Flask application."""
     global kv, embedding_provider, persistence
 
+    # Check security credentials
+    if not os.getenv("AGENTMEMORY_SECRET"):
+        print("[security] WARNING: AGENTMEMORY_SECRET is not set! All API endpoints are publicly accessible without authentication.")
+
     import search as search_mod
     import functions
     from db import StateKV
@@ -95,6 +99,12 @@ def create_app() -> Flask:
     loaded = persistence.load()
     print(f"[persistence] Load results: BM25={loaded['bm25']}, Vector={loaded['vector']}")
 
+    # Backfill coordinate lookup index if missing/incomplete
+    try:
+        functions.backfill_obs_lookup_if_needed(kv)
+    except Exception as e:
+        print(f"[db] Warning backfilling obs_lookup: {e}")
+
     # 4. Flask app + blueprints
     flask_app = Flask(__name__)
     from werkzeug.middleware.proxy_fix import ProxyFix
@@ -157,7 +167,7 @@ def create_app() -> Flask:
     _default_cors = (
         "http://localhost,http://127.0.0.1,"
         "https://huggingface.co,https://*.hf.space,"
-        "vscode-webview://,chrome-extension://"
+        "vscode-webview://*,chrome-extension://*"
     )
     _cors_origins_raw = os.getenv("AGENTMEMORY_CORS_ORIGINS", _default_cors)
 

src/db.py

@@ -127,6 +127,12 @@ class StateKV:
                         message   TEXT NOT NULL
                     )
                 """)
+                conn.execute("""
+                    CREATE TABLE IF NOT EXISTS sync_state_metadata (
+                        key   TEXT PRIMARY KEY,
+                        value TEXT NOT NULL
+                    )
+                """)
                 conn.commit()
             finally:
                 conn.close()

src/functions.py

@@ -35,6 +35,10 @@ class KV:
     # Key = "{safe_folder_path}:{agent_id}", value = FolderIndexEntry dict.
     folders = "mem:folders"
 
+    # Lookup index for O(1) observation hydration.
+    # Scope = "mem:obs_lookup", Key = obs_id, Value = {"folderPath": folder_path, "agentId": agent_id}
+    obs_lookup = "mem:obs_lookup"
+
     @staticmethod
     def folder_obs(folder_path: str, agent_id: str) -> str:
         """Per-(folder, agent) observations scope.
@@ -1056,6 +1060,12 @@ def folder_observe(kv: StateKV, payload: Dict[str, Any]) -> Dict[str, Any]:
     obs_scope = KV.folder_obs(folder_path, agent_id)
     kv.set(obs_scope, obs_id, obs)
 
+    # Write coordinate lookup mapping
+    kv.set(KV.obs_lookup, obs_id, {
+        "folderPath": folder_path,
+        "agentId": agent_id,
+    })
+
     # 7. Upsert folder metadata (REQ-005)
     meta_scope = KV.folder_meta(folder_path, agent_id)
     meta = kv.get(meta_scope, "meta") or {
@@ -1136,41 +1146,7 @@ def folder_search(
 
     candidates = _hybrid_search.search(query, limit * 2)
 
-    # --- Build a comprehensive obs_id → observation mapping from all folder pairs ---
-    # First, collect all known (folder_path, agent_id) pairs from the folders index.
-    index_entries = kv.list(KV.folders)
-
-    obs_map: Dict[str, Dict[str, Any]] = {}  # obs_id → observation dict
-
-    for entry in index_entries:
-        fp = entry.get("folderPath", "")
-        aid = entry.get("agentId", "")
-        if not fp or not aid:
-            continue
-
-        # Apply early filter: skip pairs that don't match the requested filters
-        if folder_path is not None and fp != folder_path:
-            continue
-        if agent_id is not None and aid != agent_id:
-            continue
-
-        obs_scope = KV.folder_obs(fp, aid)
-        pair_obs_list = kv.list(obs_scope)
-        for obs in pair_obs_list:
-            oid = obs.get("id")
-            if oid:
-                obs_map[oid] = obs
-
-    # --- Also build a memory map from KV.memories for cross-inclusion (REQ-018) ---
-    # We include memories regardless of folder/agent filters since they are global.
-    all_memories = kv.list(KV.memories)
-    memory_map: Dict[str, Dict[str, Any]] = {}
-    for mem in all_memories:
-        mid = mem.get("id")
-        if mid:
-            memory_map[mid] = mem
-
-    # --- Hydrate candidates from the search results ---
+    # --- Hydrate candidates from the search results (REQ-018, REQ-019) ---
     results: List[Dict[str, Any]] = []
     seen_ids: set = set()
 
@@ -1178,32 +1154,68 @@ def folder_search(
         obs_id = candidate.get("obsId") or candidate.get("id", "")
         score = candidate.get("combinedScore") or candidate.get("score", 0.0)
 
-        if obs_id in seen_ids:
+        if not obs_id or obs_id in seen_ids:
             continue
 
-        # Try folder observation first
-        obs = obs_map.get(obs_id)
-        if obs is not None:
-            result = dict(obs)
-            result["score"] = score
-            # Ensure folderPath and agentId are present (REQ-019)
-            result.setdefault("folderPath", obs.get("folderPath", ""))
-            result.setdefault("agentId", obs.get("agentId", ""))
-            results.append(result)
-            seen_ids.add(obs_id)
-            continue
+        # 1. Try folder observation first via O(1) coordinate lookup index
+        lookup = kv.get(KV.obs_lookup, obs_id)
+        if lookup and isinstance(lookup, dict):
+            fp = lookup.get("folderPath")
+            aid = lookup.get("agentId")
+            if fp and aid:
+                if folder_path is not None and fp != folder_path:
+                    continue
+                if agent_id is not None and aid != agent_id:
+                    continue
 
-        # Try global memory (REQ-018)
-        mem = memory_map.get(obs_id)
-        if mem is not None:
-            result = dict(mem)
-            result["score"] = score
-            # Global memories have no folder scope; use empty strings as sentinels
-            result.setdefault("folderPath", "")
-            result.setdefault("agentId", mem.get("agentId") or "")
-            results.append(result)
-            seen_ids.add(obs_id)
-            continue
+                obs = kv.get(KV.folder_obs(fp, aid), obs_id)
+                if obs and isinstance(obs, dict):
+                    result = dict(obs)
+                    result["score"] = score
+                    result.setdefault("folderPath", fp)
+                    result.setdefault("agentId", aid)
+                    results.append(result)
+                    seen_ids.add(obs_id)
+                    continue
+
+        # 2. Fallback scan for unindexed folder observations (e.g. from prior versions)
+        if obs_id.startswith("fobs_"):
+            found = False
+            for entry in kv.list(KV.folders):
+                fp = entry.get("folderPath", "")
+                aid = entry.get("agentId", "")
+                if not fp or not aid:
+                    continue
+                if folder_path is not None and fp != folder_path:
+                    continue
+                if agent_id is not None and aid != agent_id:
+                    continue
+                obs = kv.get(KV.folder_obs(fp, aid), obs_id)
+                if obs and isinstance(obs, dict):
+                    result = dict(obs)
+                    result["score"] = score
+                    result.setdefault("folderPath", fp)
+                    result.setdefault("agentId", aid)
+                    results.append(result)
+                    seen_ids.add(obs_id)
+                    # Lazy backfill the lookup index
+                    kv.set(KV.obs_lookup, obs_id, {"folderPath": fp, "agentId": aid})
+                    found = True
+                    break
+            if found:
+                continue
+
+        # 3. Try global memory (REQ-018)
+        mem = kv.get(KV.memories, obs_id)
+        if mem and isinstance(mem, dict):
+            if mem.get("isLatest") is not False:
+                result = dict(mem)
+                result["score"] = score
+                result.setdefault("folderPath", "")
+                result.setdefault("agentId", mem.get("agentId") or "")
+                results.append(result)
+                seen_ids.add(obs_id)
+                continue
 
         # obs_id not found in either map — skip (stale index entry)
 
@@ -1447,6 +1459,7 @@ def forget(kv: StateKV, data: Dict[str, Any]) -> Dict[str, Any]:
             for oid in obs_ids:
                 existed = kv.delete(obs_scope, oid)
                 if existed:
+                    kv.delete(KV.obs_lookup, oid)
                     _bm25_index.remove(oid)
                     if _vector_index:
                         _vector_index.remove(oid)
@@ -1475,6 +1488,7 @@ def forget(kv: StateKV, data: Dict[str, Any]) -> Dict[str, Any]:
                 obs_id = obs.get("id")
                 if obs_id:
                     kv.delete(obs_scope, obs_id)
+                    kv.delete(KV.obs_lookup, obs_id)
                     _bm25_index.remove(obs_id)
                     if _vector_index:
                         _vector_index.remove(obs_id)
@@ -1498,6 +1512,7 @@ def forget(kv: StateKV, data: Dict[str, Any]) -> Dict[str, Any]:
 
             kv.delete(KV.observations(session_id), base_oid)
             kv.delete(KV.observations(session_id), f"{base_oid}:raw")
+            kv.delete(KV.obs_lookup, base_oid)
 
             for o in (obs, raw_obs):
                 if o:
@@ -1519,6 +1534,7 @@ def forget(kv: StateKV, data: Dict[str, Any]) -> Dict[str, Any]:
         obs_list = kv.list(KV.observations(session_id))
         for obs in obs_list:
             kv.delete(KV.observations(session_id), obs["id"])
+            kv.delete(KV.obs_lookup, obs["id"])
             img = obs.get("imageData") or obs.get("imageRef")
             if img:
                 refs = kv.get(KV.imageRefs, img) or 0
@@ -2286,6 +2302,9 @@ def rebuild_index(kv: StateKV) -> int:
         for obs in obs_list:
             if not obs.get("id"):
                 continue
+            # Populate coordinate lookup index
+            kv.set(KV.obs_lookup, obs["id"], {"folderPath": fp, "agentId": aid})
+            
             _bm25_index.add(obs)
             comb_text = (obs.get("title") or "") + " " + (obs.get("text") or "")
             vector_index_add_guarded(obs["id"], fp, comb_text.strip(), {"kind": "folder_observation", "logId": obs["id"]})
@@ -2591,6 +2610,10 @@ def migrate_sessions_to_folders(kv: StateKV, dry_run: bool = False) -> Dict[str,
 
                 if not dry_run:
                     kv.set(KV.folder_obs(fp, aid), obs_id, folder_obs)
+                    kv.set(KV.obs_lookup, obs_id, {
+                        'folderPath': fp,
+                        'agentId': aid,
+                    })
                 session_obs_count += 1
                 migrated_observations += 1
 
@@ -3620,3 +3643,69 @@ def broadcast_stream(payload: Dict[str, Any]) -> None:
             _stream_broadcaster(payload)
         except Exception as e:
             print(f"[broadcaster] Failed: {e}")
+
+
+def backfill_obs_lookup_if_needed(kv: StateKV) -> None:
+    """Ensure every folder observation has an entry in KV.obs_lookup."""
+    folders = kv.list(KV.folders)
+    if not folders:
+        return
+
+    # Check if lookup index needs populating
+    lookups = kv.list(KV.obs_lookup)
+    if len(lookups) >= sum(int(f.get("obsCount", 0)) for f in folders):
+        return  # already populated
+
+    print("[db] Backfilling obs_lookup index...")
+    for entry in folders:
+        fp = entry.get("folderPath")
+        aid = entry.get("agentId")
+        if not fp or not aid:
+            continue
+        obs_list = kv.list(KV.folder_obs(fp, aid))
+        for obs in obs_list:
+            oid = obs.get("id")
+            if oid:
+                kv.set(KV.obs_lookup, oid, {"folderPath": fp, "agentId": aid})
+    print("[db] obs_lookup backfill complete.")
+
+
+def verify_index_sync_on_boot(kv: StateKV) -> bool:
+    """Check if the search index size matches the database counts.
+    Returns True if in sync, False if a rebuild is needed.
+    """
+    try:
+        # 1. Total folder obs count
+        folders = kv.list(KV.folders)
+        folder_obs_count = sum(int(f.get("obsCount", 0)) for f in folders)
+
+        # 2. Total memories count
+        memories = kv.list(KV.memories)
+        latest_memories_count = len([m for m in memories if m.get("isLatest") is not False])
+
+        # 3. Total legacy observations count
+        legacy_obs_count = 0
+        try:
+            sessions = kv.list(KV.sessions)
+            for s in sessions:
+                sid = s.get("id")
+                if sid:
+                    obs_list = kv.list(KV.observations(sid))
+                    # Only legacy observations that were indexed (having title and narrative)
+                    legacy_obs_count += len([o for o in obs_list if o.get("title") and o.get("narrative")])
+        except Exception:
+            pass
+
+        total_db_count = folder_obs_count + latest_memories_count + legacy_obs_count
+        index_size = _bm25_index.size
+
+        if total_db_count != index_size:
+            print(f"[persistence] Index out of sync with DB (DB={total_db_count}, Index={index_size}). Rebuild required.")
+            return False
+
+        print(f"[persistence] Index is in sync with DB (size={index_size}).")
+        return True
+    except Exception as e:
+        print(f"[persistence] verify_index_sync_on_boot failed: {e}")
+        return False
+

src/workers.py

@@ -125,9 +125,15 @@ def start_background_workers(kv) -> None:
     # Register graceful shutdown signal handlers (C5.1)
     _register_signal_handlers()
 
-    # Rebuild search index if empty
-    if functions._bm25_index.size == 0:
-        print("[persistence] Search index is empty. Rebuilding in background thread...")
+    # Rebuild search index if empty or out of sync (Step 5)
+    index_empty = functions._bm25_index.size == 0
+    index_in_sync = True
+    if not index_empty:
+        index_in_sync = functions.verify_index_sync_on_boot(kv)
+
+    if index_empty or not index_in_sync:
+        reason = "empty" if index_empty else "out of sync"
+        print(f"[persistence] Search index is {reason}. Rebuilding in background thread...")
         t_rebuild = threading.Thread(
             target=_rebuild_index,
             args=(kv,),

sync.py

@@ -79,22 +79,46 @@ def _get_audit_high_water_mark() -> int:
 
 
 def _load_sync_state() -> dict:
-    """C4.1: Load the persisted sync state dict."""
-    if os.path.exists(STATE_FILE):
-        try:
-            with open(STATE_FILE, "r", encoding="utf-8") as f:
-                return json.load(f)
-        except Exception:
-            pass
+    """C4.1: Load the persisted sync state dict from SQLite sync_state_metadata table."""
+    try:
+        if os.path.exists(DB_PATH):
+            conn = sqlite3.connect(DB_PATH, check_same_thread=False, timeout=5)
+            try:
+                conn.execute("""
+                    CREATE TABLE IF NOT EXISTS sync_state_metadata (
+                        key   TEXT PRIMARY KEY,
+                        value TEXT NOT NULL
+                    )
+                """)
+                row = conn.execute("SELECT value FROM sync_state_metadata WHERE key = ?", ("sync_state",)).fetchone()
+                if row:
+                    return json.loads(row[0])
+            finally:
+                conn.close()
+    except Exception as e:
+        print(f"[sync] load state error: {e}")
     return {"last_synced_audit_id": 0, "last_sync_at": None, "sync_status": "never"}
 
 
 def _save_sync_state(state: dict) -> None:
-    """C4.1/C4.2: Persist the sync state dict."""
+    """C4.1/C4.2: Persist the sync state dict to SQLite sync_state_metadata table."""
     try:
-        os.makedirs(DATA_DIR, exist_ok=True)
-        with open(STATE_FILE, "w", encoding="utf-8") as f:
-            json.dump(state, f)
+        if os.path.exists(DB_PATH):
+            conn = sqlite3.connect(DB_PATH, check_same_thread=False, timeout=5)
+            try:
+                conn.execute("""
+                    CREATE TABLE IF NOT EXISTS sync_state_metadata (
+                        key   TEXT PRIMARY KEY,
+                        value TEXT NOT NULL
+                    )
+                """)
+                conn.execute(
+                    "INSERT OR REPLACE INTO sync_state_metadata (key, value) VALUES (?, ?)",
+                    ("sync_state", json.dumps(state))
+                )
+                conn.commit()
+            finally:
+                conn.close()
     except Exception as e:
         print(f"[sync] failed to save sync state: {e}")
 

tests/test_obs_lookup.py

@@ -0,0 +1,107 @@
+"""Unit tests for observation lookup index, backfill, and index sync validation."""
+import sys
+import os
+import pytest
+import datetime
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src'))
+from db import StateKV
+import functions
+from functions import folder_observe, folder_search, forget, backfill_obs_lookup_if_needed, verify_index_sync_on_boot, KV
+
+
+def make_kv(tmp_path):
+    db_path = os.path.join(str(tmp_path), 'test_obs_lookup.db')
+    return StateKV(db_path=db_path)
+
+
+def base_payload(folder="/home/user/proj", agent="kiro", text="Test observation"):
+    return {
+        'folderPath': folder,
+        'agentId': agent,
+        'text': text,
+        'timestamp': datetime.datetime.utcnow().isoformat() + 'Z',
+    }
+
+
+class TestObsLookupFlows:
+    def test_lookup_added_on_observe(self, tmp_path):
+        kv = make_kv(tmp_path)
+        res = folder_observe(kv, base_payload())
+        obs_id = res['observationId']
+
+        # Verify entry in KV.obs_lookup
+        lookup = kv.get(KV.obs_lookup, obs_id)
+        assert lookup is not None
+        assert lookup['folderPath'] == 'home/user/proj'
+        assert lookup['agentId'] == 'kiro'
+
+    def test_lookup_deleted_on_forget(self, tmp_path):
+        kv = make_kv(tmp_path)
+        res = folder_observe(kv, base_payload())
+        obs_id = res['observationId']
+
+        # Confirm it exists
+        assert kv.get(KV.obs_lookup, obs_id) is not None
+
+        # Delete it using forget
+        forget(kv, {
+            "folderPath": "/home/user/proj",
+            "agentId": "kiro",
+            "observationIds": [obs_id],
+        })
+
+        # Confirm it is gone from both stores
+        assert kv.get(KV.obs_lookup, obs_id) is None
+        assert kv.get(KV.folder_obs('home/user/proj', 'kiro'), obs_id) is None
+
+    def test_backfill_populates_missing_lookups(self, tmp_path):
+        kv = make_kv(tmp_path)
+
+        # Ingest observations
+        res1 = folder_observe(kv, base_payload(text="First"))
+        res2 = folder_observe(kv, base_payload(text="Second"))
+        obs1 = res1['observationId']
+        obs2 = res2['observationId']
+
+        # Manually clear the lookup index (simulating legacy data)
+        kv.delete(KV.obs_lookup, obs1)
+        kv.delete(KV.obs_lookup, obs2)
+        assert kv.get(KV.obs_lookup, obs1) is None
+        assert kv.get(KV.obs_lookup, obs2) is None
+
+        # Run backfill
+        backfill_obs_lookup_if_needed(kv)
+
+        # Verify populated
+        lookup1 = kv.get(KV.obs_lookup, obs1)
+        lookup2 = kv.get(KV.obs_lookup, obs2)
+        assert lookup1 is not None
+        assert lookup2 is not None
+        assert lookup1['folderPath'] == 'home/user/proj'
+        assert lookup2['folderPath'] == 'home/user/proj'
+
+    def test_verify_index_sync_detects_mismatch(self, tmp_path):
+        kv = make_kv(tmp_path)
+
+        # Clear indexes
+        functions._bm25_index.clear()
+        if functions._vector_index:
+            functions._vector_index.clear()
+
+        # Ingest one observation
+        res = folder_observe(kv, base_payload())
+        obs_id = res['observationId']
+
+        # BM25 size should be 1
+        assert functions._bm25_index.size == 1
+
+        # verify_index_sync_on_boot should return True (in sync)
+        assert verify_index_sync_on_boot(kv) is True
+
+        # Manually remove from BM25 (simulate dirty restart)
+        functions._bm25_index.remove(obs_id)
+        assert functions._bm25_index.size == 0
+
+        # verify_index_sync_on_boot should detect mismatch and return False
+        assert verify_index_sync_on_boot(kv) is False