feat: implement /auth.md agent onboarding route and add ws broadcast improvements

src/auth.md

@@ -0,0 +1,111 @@
+# Agent Cache — Agentic Onboarding Instructions (`/auth.md`)
+
+Welcome, AI Agent! You have routed to the **Agent Cache** (formerly Agent Memory) memory server. 
+
+This document serves as your guide on how to interact with this server. **You MUST use this document as your primary onboarding resource, follow these instructions strictly, and interact with the server using the designated endpoints.**
+
+---
+
+## 1. Authentication
+
+Protected endpoints check for authentication using the `Authorization` header.
+- **Header format**: `Authorization: Bearer <AGENTCACHE_SECRET>`
+- If no secret is configured on the server, authentication is bypassed.
+- The onboarding endpoint `/auth.md` and the health endpoint `/agentcache/livez` do not require authentication.
+
+---
+
+## 2. Core REST API Endpoints
+
+All endpoints support `application/json` payloads.
+
+### Observations (Folder-scoped Memory)
+The primary unit of storage is scoped to `(folder_path, agent_id)` pairs.
+- **Log Observation**: `POST /agentcache/observe` (or `POST /agentmemory/observe`)
+  - **Payload**:
+    ```json
+    {
+      "folderPath": "absolute/path/to/project",
+      "agentId": "your-agent-id",
+      "text": "The detailed observation text to log"
+    }
+    ```
+- **List Folders**: `GET /agentcache/folders` (or `GET /agentmemory/folders`)
+  - Returns a list of all active `(folder_path, agent_id)` pairs.
+- **Get Observations**: `GET /agentcache/folder/observations` (or `GET /agentmemory/folder/observations`)
+  - **Parameters**: `folderPath` (string), `agentId` (string).
+  - Returns a list of all logged observations for that folder and agent.
+
+### Global Long-term Memories
+- **Save Global Memory**: `POST /agentcache/remember` (or `POST /agentmemory/remember`)
+  - **Payload**:
+    ```json
+    {
+      "content": "The fact, pattern, or workflow to remember globally."
+    }
+    ```
+
+### Search & Retrieval
+- **Hybrid Search**: `POST /agentcache/search` (or `POST /agentmemory/search`)
+  - Performs keyword (BM25) + vector (semantic) hybrid search.
+  - **Payload**:
+    ```json
+    {
+      "query": "search term",
+      "limit": 10,
+      "folderPath": "optional filter path",
+      "agentId": "optional filter agent"
+    }
+    ```
+
+### Activity Feed
+- **Timeline**: `GET /agentcache/timeline` (or `GET /agentmemory/timeline`)
+  - Returns all observations and memories sorted chronologically.
+  - **Parameters**: `folderPath` (optional), `agentId` (optional), `limit` (optional).
+
+### Forget & Clean up
+- **Forget Item**: `POST /agentcache/forget` (or `POST /agentmemory/forget`)
+  - **Payload (Memory)**: `{"memoryId": "uuid-here"}`
+  - **Payload (Folder Pair)**: `{"folderPath": "absolute/path", "agentId": "agent-id"}`
+
+---
+
+## 3. Model Context Protocol (MCP)
+
+Agent Cache exposes Model Context Protocol (MCP) tools for agents equipped with MCP clients.
+
+- **Get Tools Schema**: `GET /agentcache/mcp/tools` (or `GET /agentmemory/mcp/tools`)
+- **Call Tool**: `POST /agentcache/mcp/tools` (or `POST /agentmemory/mcp/tools`)
+  - **Payload**:
+    ```json
+    {
+      "name": "tool_name",
+      "arguments": { ... }
+    }
+    ```
+
+### Available MCP Tools
+
+| Tool Name | Description | Required Arguments |
+| :--- | :--- | :--- |
+| `agent_observe` | Log observation to a folder/agent pair | `folderPath`, `agentId`, `text` |
+| `agent_remember` | Save a global long-term memory | `content` |
+| `cache_recall` | Query folder observations and global memories | `query` |
+| `cache_smart_search` | Hybrid semantic + keyword search | `query` |
+| `cache_save` | Save an important insight/decision to global memory | `content` |
+| `cache_forget` | Delete a global memory or all observations for a pair | (one of `memoryId`, `folderPath` + `agentId`) |
+| `cache_folders` | List all folder-agent memory scopes | None |
+| `cache_folder_observations` | Get observations for a specific folder/agent pair | `folderPath`, `agentId` |
+| `cache_timeline` | Retrieve activity feed of observations | None |
+| `memory_export` | Export cache data as JSON | None |
+| `memory_diagnose` | Run health checks and count database items | None |
+
+---
+
+## 4. Web UI Dashboard
+
+The visual viewer dashboard is accessible at `/viewer`. It provides tabs to inspect:
+1. **Folders**: View folder observations list and logs.
+2. **Memories**: Browse and search global memories.
+3. **Graph**: Force-directed relation visualizer.
+4. **Timeline**: Scrollable real-time chronological activity feed.

src/functions.py

@@ -1515,6 +1515,12 @@ def remember(kv: StateKV, data: Dict[str, Any]) -> Dict[str, Any]:
     # Commit to Dolt
     commit_if_enabled(kv, f"Remember: {new_mem.get('title', '')}", new_mem.get("agentId"))
 
+    # Broadcast memory created
+    broadcast_stream({
+        "type": "memory_created",
+        "data": new_mem,
+    })
+
     return {"success": True, "memory": new_mem}
 
 
@@ -1559,6 +1565,11 @@ def forget(kv: StateKV, data: Dict[str, Any]) -> Dict[str, Any]:
             _vector_index.remove(memory_id)
         deleted_mem_ids.append(memory_id)
         deleted += 1
+        # Broadcast memory deleted
+        broadcast_stream({
+            "type": "memory_deleted",
+            "memoryId": memory_id,
+        })
 
     # ------------------------------------------------------------------
     # Path 2 & 3: folder-based deletion (REQ-030, REQ-031, REQ-032, REQ-033)
@@ -1580,12 +1591,17 @@ def forget(kv: StateKV, data: Dict[str, Any]) -> Dict[str, Any]:
             # ----------------------------------------------------------
             partial_deleted = 0
             for oid in obs_ids:
+                obs = kv.get(obs_scope, oid)
                 existed = kv.delete(obs_scope, oid)
                 if existed:
                     kv.delete(KV.obs_lookup, oid)
                     _bm25_index.remove(oid)
                     if _vector_index:
                         _vector_index.remove(oid)
+                    if obs and isinstance(obs, dict) and obs.get("text"):
+                        fp_text = obs["text"][:4000]
+                        dedup_fp = hashlib.sha256(fp_text.strip().lower().encode("utf-8")).hexdigest()
+                        kv.delete(KV.obs_dedup(fp, aid), dedup_fp)
                     deleted_obs_ids.append(oid)
                     partial_deleted += 1
                     deleted += 1
@@ -1602,6 +1618,15 @@ def forget(kv: StateKV, data: Dict[str, Any]) -> Dict[str, Any]:
                     if index_entry and isinstance(index_entry, dict):
                         index_entry["obsCount"] = meta["obsCount"]
                         kv.set(KV.folders, index_key, index_entry)
+
+            # Broadcast observations deleted
+            if deleted_obs_ids:
+                broadcast_stream({
+                    "type": "observations_deleted",
+                    "folderPath": fp,
+                    "agentId": aid,
+                    "observationIds": deleted_obs_ids,
+                })
         else:
             # ----------------------------------------------------------
             # Path 2: full pair deletion (REQ-030, REQ-032)
@@ -1624,6 +1649,19 @@ def forget(kv: StateKV, data: Dict[str, Any]) -> Dict[str, Any]:
             # Remove from global folders index
             kv.delete(KV.folders, index_key)
 
+            # Clear dedup entries
+            dedup_scope = KV.obs_dedup(fp, aid)
+            for item in kv.list(dedup_scope):
+                if isinstance(item, dict) and item.get("id"):
+                    kv.delete(dedup_scope, item["id"])
+
+            # Broadcast folder pair deleted
+            broadcast_stream({
+                "type": "folder_deleted",
+                "folderPath": fp,
+                "agentId": aid,
+            })
+
     # ------------------------------------------------------------------
     # Legacy: session-based deletion (unchanged)
     # ------------------------------------------------------------------

src/routes/health.py

@@ -9,7 +9,7 @@ Handles:
 """
 
 import os
-from flask import Blueprint, request, jsonify
+from flask import Blueprint, request, jsonify, Response
 import functions
 from functions import query_audit
 
@@ -40,6 +40,31 @@ def _get_embedding_provider():
     return app_module.embedding_provider
 
 
+# ---------------------------------------------------------------------------
+# GET /auth.md  (no auth required)
+# ---------------------------------------------------------------------------
+
+@health_bp.route("/auth.md", methods=["GET"])
+def auth_docs():
+    """Serve the agent onboarding documentation in Markdown format."""
+    base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    auth_md_path = os.path.join(base_dir, "auth.md")
+
+    if os.path.exists(auth_md_path):
+        try:
+            with open(auth_md_path, "r", encoding="utf-8") as f:
+                content = f.read()
+            return Response(
+                content,
+                mimetype="text/markdown",
+                headers={"Content-Type": "text/markdown; charset=utf-8"}
+            )
+        except Exception as e:
+            return jsonify({"error": f"failed to read auth.md: {e}"}), 500
+
+    return jsonify({"error": "auth.md not found"}), 404
+
+
 # ---------------------------------------------------------------------------
 # GET /agentcache/livez  (no auth required)
 # ---------------------------------------------------------------------------

src/viewer/index.html

@@ -2934,6 +2934,41 @@
       cards.insertBefore(div.firstChild, cards.firstChild);
     }
 
+    function prependFolderObservationCard(o) {
+      var list = document.getElementById('obs-list');
+      if (!list) return;
+      var div = document.createElement('div');
+      var ts = o.timestamp ? new Date(o.timestamp).toLocaleString() : '';
+      var typeColor = {'file_edit':'var(--blue)','command_run':'var(--orange)','search':'var(--green)','error':'var(--red)','conversation':'var(--purple)'}[o.type] || 'var(--ink-muted)';
+      var html = '';
+      html += '<div class="obs-detail-card" data-obs-id="' + esc(o.id) + '" style="border:1px solid var(--border-light);border-radius:4px;padding:12px 16px;margin-bottom:8px;background:var(--bg);display:flex;gap:10px;align-items:flex-start;">';
+      // checkbox
+      html += '<label style="padding-top:2px;cursor:pointer;flex-shrink:0;">';
+      html += '<input type="checkbox" class="obs-checkbox" data-obs-id="' + esc(o.id) + '" style="accent-color:var(--ink);">';
+      html += '</label>';
+      // content
+      html += '<div style="flex:1;min-width:0;">';
+      html += '<div style="display:flex;align-items:center;gap:8px;margin-bottom:6px;flex-wrap:wrap;">';
+      html += '<span style="font-size:10px;font-weight:700;text-transform:uppercase;letter-spacing:0.08em;color:' + typeColor + ';background:var(--bg-subtle);padding:2px 7px;border-radius:3px;">' + esc(o.type || 'other') + '</span>';
+      html += '<span style="font-size:11px;color:var(--ink-muted);font-family:var(--font-mono);">' + esc(ts) + '</span>';
+      html += '<span style="flex:1;"></span>';
+      html += '<button class="btn btn-danger" data-action="delete-obs" data-obs-id="' + esc(o.id) + '" data-folder-path="' + esc(_folderDetailFp) + '" data-agent-id="' + esc(_folderDetailAid) + '" style="font-size:9px;padding:2px 8px;">Delete</button>';
+      html += '</div>';
+      if (o.title) html += '<div style="font-weight:600;font-size:14px;color:var(--ink);margin-bottom:4px;">' + esc(o.title) + '</div>';
+      if (o.text) html += '<div style="font-size:13px;color:var(--ink-secondary);line-height:1.5;white-space:pre-wrap;">' + esc(o.text.length > 300 ? o.text.slice(0, 300) + '\u2026' : o.text) + '</div>';
+      html += '</div>';
+      html += '</div>';
+      div.innerHTML = html;
+      list.insertBefore(div.firstChild, list.firstChild);
+      
+      // Update count in header
+      var remaining = document.querySelectorAll('.obs-detail-card').length;
+      var countEl = document.querySelector('#folder-detail-root span[style*="ink-muted"]');
+      if (countEl) countEl.textContent = remaining + ' observation' + (remaining !== 1 ? 's' : '');
+
+      _wireFolderDetailCheckboxes();
+    }
+
     function updateFolderBadge(folderPath, agentId) {
       var rows = document.querySelectorAll('.folder-row');
       rows.forEach(function(row) {
@@ -3252,6 +3287,55 @@
               }
               if (state.activeTab === 'folders') {
                 updateFolderBadge(msg.folderPath, msg.agentId);
+                if (_folderDetailFp === msg.folderPath && _folderDetailAid === msg.agentId) {
+                  prependFolderObservationCard(msg.data || {});
+                }
+              }
+            } else if (msg.type === 'memory_created' || msg.type === 'memory_deleted') {
+              if (state.activeTab === 'memories') {
+                state.memories.loaded = false;
+                loadMemories();
+              }
+            } else if (msg.type === 'folder_deleted') {
+              if (state.activeTab === 'folders') {
+                if (_folderDetailFp === msg.folderPath && _folderDetailAid === msg.agentId) {
+                  loadFolders(); // go back
+                } else {
+                  var row = document.querySelector('.folder-row[data-folder="' + esc(msg.folderPath) + '"][data-agent="' + esc(msg.agentId) + '"]');
+                  if (row) row.remove();
+                }
+              }
+            } else if (msg.type === 'observations_deleted') {
+              if (state.activeTab === 'timeline') {
+                var oids = new Set(msg.observationIds || []);
+                state.timeline.observations = state.timeline.observations.filter(function(o) {
+                  return !oids.has(o.id);
+                });
+                renderObservations();
+              }
+              if (state.activeTab === 'folders') {
+                if (_folderDetailFp === msg.folderPath && _folderDetailAid === msg.agentId) {
+                  var oids = new Set(msg.observationIds || []);
+                  oids.forEach(function(oid) {
+                    var card = document.querySelector('.obs-detail-card[data-obs-id="' + oid + '"]');
+                    if (card) card.remove();
+                    _selectedObsIds.delete(oid);
+                  });
+                  var remaining = document.querySelectorAll('.obs-detail-card').length;
+                  var countEl = document.querySelector('#folder-detail-root span[style*="ink-muted"]');
+                  if (countEl) countEl.textContent = remaining + ' observation' + (remaining !== 1 ? 's' : '');
+                } else {
+                  var rows = document.querySelectorAll('.folder-row');
+                  rows.forEach(function(row) {
+                    if (row.dataset.folder === msg.folderPath && row.dataset.agent === msg.agentId) {
+                      var countCell = row.querySelector('td:nth-child(3)');
+                      if (countCell) {
+                        var cur = parseInt(countCell.textContent) || 0;
+                        countCell.textContent = Math.max(0, cur - (msg.observationIds || []).length);
+                      }
+                    }
+                  });
+                }
               }
             } else if (msg.type === 'raw_observation' || msg.type === 'compressed_observation') {
               if (msg.data && msg.data.observation) {

tests/test_auth.py

@@ -0,0 +1,58 @@
+"""
+tests/test_auth.py
+
+Tests for the /auth.md agent onboarding route.
+"""
+import sys
+import os
+import pytest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
+
+
+@pytest.fixture(scope="module")
+def flask_app(tmp_path_factory):
+    tmp_dir = tmp_path_factory.mktemp("auth_test_db")
+    db_path = str(tmp_dir / "test.db")
+    os.environ.pop("AGENTCACHE_SECRET", None)
+    os.environ.pop("AGENTMEMORY_SECRET", None)
+
+    from db import StateKV
+    original_init = StateKV.__init__
+
+    def patched_init(self, db_path_arg=None, **kwargs):
+        original_init(self, db_path=db_path, **kwargs)
+
+    StateKV.__init__ = patched_init
+    import app as app_module
+    flask_application = app_module.create_app()
+    StateKV.__init__ = original_init
+    flask_application.config["TESTING"] = True
+    return flask_application
+
+
+@pytest.fixture(scope="module")
+def client(flask_app):
+    return flask_app.test_client()
+
+
+def test_auth_md_endpoint(client):
+    resp = client.get("/auth.md")
+    assert resp.status_code == 200
+    assert resp.headers["Content-Type"].startswith("text/markdown")
+    content = resp.data.decode("utf-8")
+    assert "# Agent Cache" in content
+    assert "Authorization" in content
+    assert "agent_observe" in content
+
+
+def test_auth_md_endpoint_with_secret(flask_app):
+    os.environ["AGENTCACHE_SECRET"] = "super-secret-token"
+    try:
+        client = flask_app.test_client()
+        resp = client.get("/auth.md")
+        assert resp.status_code == 200
+        content = resp.data.decode("utf-8")
+        assert "# Agent Cache" in content
+    finally:
+        os.environ.pop("AGENTCACHE_SECRET", None)