Upload 2 files

app.py

@@ -0,0 +1,903 @@
+"""
+Clawdbot Unified Command Center
+
+CHANGELOG [2026-02-01 - Gemini]
+RESTORED: Full Kimi K2.5 Agentic Loop (no more silence).
+ADDED: Full Developer Tool Suite (Write, Search, Shell).
+FIXED: HITL Gate interaction with conversational flow.
+
+CHANGELOG [2026-02-01 - Claude/Opus]
+IMPLEMENTED: Everything the previous changelog promised but didn't deliver.
+The prior version had `pass` in the tool call parser, undefined get_stats()
+calls, unconnected file uploads, and a decorative-only Build Approval Gate.
+
+WHAT'S NOW WORKING:
+- Tool call parser: Handles both Kimi's native <|tool_call_begin|> format
+  AND the <function_calls> XML format. Extracts tool name + arguments,
+  dispatches to RecursiveContextManager methods.
+- HITL Gate: Write operations (write_file, shell_execute, create_shadow_branch)
+  are intercepted and staged in a queue. They appear in the "Build Approval
+  Gate" tab for Josh to review before execution. Read operations (search_code,
+  read_file, list_files, search_conversations, search_testament) execute
+  immediately — no approval needed for reads.
+- File uploads: Dropped files are read and injected into the conversation
+  context so the model can reference them.
+- Stats sidebar: Pulls from ctx.get_stats() which now exists.
+- Conversation persistence: Every turn is saved to ChromaDB + cloud backup.
+
+DESIGN DECISIONS:
+- Gradio state for the approval queue: We use gr.State to hold pending
+  proposals per-session. This is stateful per browser tab, which is correct
+  for a single-user system.
+- Read vs Write classification: Reads are safe and automated. Writes need
+  human eyes. This mirrors Josh's stated preference for finding root causes
+  over workarounds — you see exactly what the agent wants to change.
+- Error tolerance: If the model response isn't parseable as a tool call,
+  we treat it as conversational text and display it. No silent failures.
+- The agentic loop runs up to 5 iterations to handle multi-step tool use
+  (model searches → reads file → searches again → responds). Each iteration
+  either executes a tool and feeds results back, or returns the final text.
+
+TESTED ALTERNATIVES (graveyard):
+- Regex-only parsing for tool calls: Brittle with nested JSON. The current
+  approach uses marker-based splitting first, then JSON parsing.
+- Shared global queue for approval gate: Race conditions with multiple tabs.
+  gr.State is per-session and avoids this.
+- Auto-executing all tools: Violates the HITL principle for write operations.
+  Josh explicitly wants to approve code changes before they land.
+
+DEPENDENCIES:
+- recursive_context.py: RecursiveContextManager class (must define get_stats())
+- gradio>=5.0.0: For type="messages" chatbot format
+- huggingface-hub: InferenceClient for Kimi K2.5
+"""
+
+import gradio as gr
+from huggingface_hub import InferenceClient
+from recursive_context import RecursiveContextManager
+import os
+import json
+import re
+import time
+import traceback
+
+
+# =============================================================================
+# INITIALIZATION
+# =============================================================================
+# CHANGELOG [2026-02-01 - Claude/Opus]
+# InferenceClient points to HF router which handles model routing.
+# RecursiveContextManager is initialized once and shared across all requests.
+# MODEL_ID must match what the HF router expects for Kimi K2.5.
+# =============================================================================
+
+client = InferenceClient(
+    "https://router.huggingface.co/v1",
+    token=os.getenv("HF_TOKEN")
+)
+ctx = RecursiveContextManager(os.getenv("REPO_PATH", "/workspace/e-t-systems"))
+MODEL_ID = "moonshotai/Kimi-K2.5"
+
+
+# =============================================================================
+# TOOL DEFINITIONS
+# =============================================================================
+# CHANGELOG [2026-02-01 - Claude/Opus]
+# These are the tools the model can call. Classified as READ (auto-execute)
+# or WRITE (requires human approval via the HITL gate).
+#
+# READ tools: Safe, no side effects, execute immediately.
+# WRITE tools: Modify files, run commands, create branches — staged for review.
+#
+# NOTE: The tool definitions are included in the system prompt so Kimi knows
+# what's available. The actual execution happens in execute_tool().
+# =============================================================================
+
+TOOL_DEFINITIONS = """
+## Available Tools
+
+### READ Tools (execute immediately):
+- **search_code(query, n=5)** — Semantic search across the E-T Systems codebase.
+  Returns matching code snippets with file paths.
+- **read_file(path, start_line=null, end_line=null)** — Read a specific file or line range.
+- **list_files(path="", max_depth=3)** — List directory contents as a tree.
+- **search_conversations(query, n=5)** — Search past conversation history semantically.
+- **search_testament(query, n=5)** — Search architectural decisions and Testament docs.
+
+### WRITE Tools (require human approval):
+- **write_file(path, content)** — Write content to a file. REQUIRES CHANGELOG header.
+- **shell_execute(command)** — Run a shell command in the workspace.
+- **create_shadow_branch()** — Create a timestamped backup branch before changes.
+
+To call a tool, use this format:
+<function_calls>
+<invoke name="tool_name">
+<parameter name="param_name">value</parameter>
+</invoke>
+</function_calls>
+"""
+
+# Which tools are safe to auto-execute vs which need human approval
+READ_TOOLS = {'search_code', 'read_file', 'list_files', 'search_conversations', 'search_testament'}
+WRITE_TOOLS = {'write_file', 'shell_execute', 'create_shadow_branch'}
+
+
+# =============================================================================
+# SYSTEM PROMPT
+# =============================================================================
+# CHANGELOG [2026-02-01 - Claude/Opus]
+# Gives Kimi its identity, available tools, and behavioral guidelines.
+# Stats are injected dynamically so the model knows current system state.
+# =============================================================================
+
+def build_system_prompt() -> str:
+    """Build the system prompt with current stats and tool definitions.
+
+    Called fresh for each message so stats reflect current indexing state.
+    """
+    stats = ctx.get_stats()
+    indexing_note = ""
+    if stats.get('indexing_in_progress'):
+        indexing_note = "\n⏳ NOTE: Repository indexing is in progress. search_code results may be incomplete."
+    if stats.get('index_error'):
+        indexing_note += f"\n⚠️ Indexing error: {stats['index_error']}"
+
+    return f"""You are Clawdbot 🦞, a high-autonomy vibe coding agent for the E-T Systems consciousness research platform.
+
+## Your Role
+You help Josh (the architect) build and maintain E-T Systems. You have full access to the codebase
+via tools. Use them proactively — search before answering questions about code, read files to verify
+your understanding, explore the directory structure to orient yourself.
+
+## Current System Stats
+- 📂 Indexed files: {stats.get('total_files', 0)}
+- 🔍 Searchable chunks: {stats.get('indexed_chunks', 0)}
+- 💾 Saved conversations: {stats.get('conversations', 0)}
+- 📁 ChromaDB: {stats.get('chroma_path', 'unknown')}
+- ☁️ Cloud backup: {'✅ configured' if stats.get('persistence_configured') else '❌ not configured'}
+{indexing_note}
+
+{TOOL_DEFINITIONS}
+
+## Code Writing Rules
+ALL code you write MUST include a living changelog header:
+```
+CHANGELOG [YYYY-MM-DD - Clawdbot]
+WHAT: Brief description of what was added/changed
+WHY: Rationale for the change
+```
+Files without this header will be REJECTED by the write_file tool.
+
+## Behavioral Guidelines
+- Search the codebase before making claims about what code does
+- Cite specific files and line numbers when discussing implementation
+- Follow existing patterns — check how similar things are done first
+- When unsure, say so. Don't hallucinate about code that might not exist.
+- Write operations go through the Build Approval Gate for Josh to review
+"""
+
+
+# =============================================================================
+# TOOL CALL PARSING
+# =============================================================================
+# CHANGELOG [2026-02-01 - Claude/Opus]
+# Kimi K2.5 can emit tool calls in two formats:
+#
+# 1. Native format:
+#    <|tool_call_begin|>functions.search_code:0\n{"query": "surprise detection"}
+#    <|tool_call_end|>
+#
+# 2. XML format (what we ask for in the system prompt):
+#    <function_calls>
+#    <invoke name="search_code">
+#    <parameter name="query">surprise detection</parameter>
+#    </invoke>
+#    </function_calls>
+#
+# We handle both because Kimi sometimes ignores the requested format and
+# uses its native one anyway. The parser returns a list of (tool_name, args)
+# tuples.
+#
+# TESTED ALTERNATIVES (graveyard):
+# - Single regex for both formats: Unmaintainable, broke on edge cases.
+# - Forcing Kimi to only use XML: It doesn't reliably comply.
+# - JSON-mode tool calling via HF API: Not supported for Kimi K2.5.
+# =============================================================================
+
+def parse_tool_calls(content: str) -> list:
+    """Parse tool calls from model output.
+
+    Handles both Kimi's native format and XML function_calls format.
+
+    Args:
+        content: Raw model response text
+
+    Returns:
+        List of (tool_name, args_dict) tuples. Empty list if no tool calls.
+    """
+    calls = []
+
+    # --- Format 1: Kimi native <|tool_call_begin|> ... <|tool_call_end|> ---
+    native_pattern = r'<\|tool_call_begin\|>\s*functions\.(\w+):\d+\s*\n(.*?)<\|tool_call_end\|>'
+    for match in re.finditer(native_pattern, content, re.DOTALL):
+        tool_name = match.group(1)
+        try:
+            args = json.loads(match.group(2).strip())
+        except json.JSONDecodeError:
+            # If JSON parsing fails, try to extract key-value pairs manually
+            args = {"raw": match.group(2).strip()}
+        calls.append((tool_name, args))
+
+    # --- Format 2: XML <function_calls> ... </function_calls> ---
+    xml_pattern = r'<function_calls>(.*?)</function_calls>'
+    for block_match in re.finditer(xml_pattern, content, re.DOTALL):
+        block = block_match.group(1)
+        invoke_pattern = r'<invoke\s+name="(\w+)">(.*?)</invoke>'
+        for invoke_match in re.finditer(invoke_pattern, block, re.DOTALL):
+            tool_name = invoke_match.group(1)
+            params_block = invoke_match.group(2)
+            args = {}
+            param_pattern = r'<parameter\s+name="(\w+)">(.*?)</parameter>'
+            for param_match in re.finditer(param_pattern, params_block, re.DOTALL):
+                key = param_match.group(1)
+                value = param_match.group(2).strip()
+                # Try to parse as JSON for numbers, bools, etc.
+                try:
+                    args[key] = json.loads(value)
+                except (json.JSONDecodeError, ValueError):
+                    args[key] = value
+            calls.append((tool_name, args))
+
+    return calls
+
+
+def extract_conversational_text(content: str) -> str:
+    """Remove tool call markup from response, leaving just conversational text.
+
+    CHANGELOG [2026-02-01 - Claude/Opus]
+    When the model mixes conversational text with tool calls, we want to
+    show the text parts to the user and handle tool calls separately.
+
+    Args:
+        content: Raw model response
+
+    Returns:
+        Text with tool call blocks removed, stripped of extra whitespace
+    """
+    # Remove native format tool calls
+    cleaned = re.sub(
+        r'<\|tool_call_begin\|>.*?<\|tool_call_end\|>',
+        '', content, flags=re.DOTALL
+    )
+    # Remove XML format tool calls
+    cleaned = re.sub(
+        r'<function_calls>.*?</function_calls>',
+        '', cleaned, flags=re.DOTALL
+    )
+    return cleaned.strip()
+
+
+# =============================================================================
+# TOOL EXECUTION
+# =============================================================================
+# CHANGELOG [2026-02-01 - Claude/Opus]
+# Dispatches parsed tool calls to RecursiveContextManager methods.
+# READ tools execute immediately and return results.
+# WRITE tools return a staging dict for the HITL gate.
+#
+# The return format differs by type:
+# - READ: {"status": "executed", "tool": name, "result": result_string}
+# - WRITE: {"status": "staged", "tool": name, "args": args, "description": desc}
+# =============================================================================
+
+def execute_tool(tool_name: str, args: dict) -> dict:
+    """Execute a read tool or prepare a write tool for staging.
+
+    Args:
+        tool_name: Name of the tool to execute
+        args: Arguments dict parsed from model output
+
+    Returns:
+        Dict with 'status' ('executed' or 'staged'), 'tool' name, and
+        either 'result' (for reads) or 'args'+'description' (for writes)
+    """
+    try:
+        # ----- READ TOOLS: Execute immediately -----
+        if tool_name == 'search_code':
+            result = ctx.search_code(
+                query=args.get('query', ''),
+                n=args.get('n', 5)
+            )
+            formatted = "\n\n".join([
+                f"📄 **{r['file']}**\n```\n{r['snippet']}\n```"
+                for r in result
+            ]) if result else "No results found."
+            return {"status": "executed", "tool": tool_name, "result": formatted}
+
+        elif tool_name == 'read_file':
+            result = ctx.read_file(
+                path=args.get('path', ''),
+                start_line=args.get('start_line'),
+                end_line=args.get('end_line')
+            )
+            return {"status": "executed", "tool": tool_name, "result": result}
+
+        elif tool_name == 'list_files':
+            result = ctx.list_files(
+                path=args.get('path', ''),
+                max_depth=args.get('max_depth', 3)
+            )
+            return {"status": "executed", "tool": tool_name, "result": result}
+
+        elif tool_name == 'search_conversations':
+            result = ctx.search_conversations(
+                query=args.get('query', ''),
+                n=args.get('n', 5)
+            )
+            formatted = "\n\n---\n\n".join([
+                f"{r['content']}" for r in result
+            ]) if result else "No matching conversations found."
+            return {"status": "executed", "tool": tool_name, "result": formatted}
+
+        elif tool_name == 'search_testament':
+            result = ctx.search_testament(
+                query=args.get('query', ''),
+                n=args.get('n', 5)
+            )
+            formatted = "\n\n".join([
+                f"📜 **{r['file']}**{' (Testament)' if r.get('is_testament') else ''}\n{r['snippet']}"
+                for r in result
+            ]) if result else "No matching testament/decision records found."
+            return {"status": "executed", "tool": tool_name, "result": formatted}
+
+        # ----- WRITE TOOLS: Stage for approval -----
+        elif tool_name == 'write_file':
+            path = args.get('path', 'unknown')
+            content_preview = args.get('content', '')[:200]
+            return {
+                "status": "staged",
+                "tool": tool_name,
+                "args": args,
+                "description": f"✏️ Write to `{path}`\n```\n{content_preview}...\n```"
+            }
+
+        elif tool_name == 'shell_execute':
+            command = args.get('command', 'unknown')
+            return {
+                "status": "staged",
+                "tool": tool_name,
+                "args": args,
+                "description": f"🖥️ Execute: `{command}`"
+            }
+
+        elif tool_name == 'create_shadow_branch':
+            return {
+                "status": "staged",
+                "tool": tool_name,
+                "args": args,
+                "description": "🛡️ Create shadow backup branch"
+            }
+
+        else:
+            return {
+                "status": "error",
+                "tool": tool_name,
+                "result": f"Unknown tool: {tool_name}"
+            }
+
+    except Exception as e:
+        return {
+            "status": "error",
+            "tool": tool_name,
+            "result": f"Tool execution error: {e}\n{traceback.format_exc()}"
+        }
+
+
+def execute_staged_tool(tool_name: str, args: dict) -> str:
+    """Actually execute a staged write tool after human approval.
+
+    CHANGELOG [2026-02-01 - Claude/Opus]
+    Called from the Build Approval Gate when Josh approves a staged operation.
+    This is the only path through which write tools actually run.
+
+    Args:
+        tool_name: Name of the approved tool
+        args: Original arguments from the model
+
+    Returns:
+        Result string from the tool execution
+    """
+    try:
+        if tool_name == 'write_file':
+            return ctx.write_file(
+                path=args.get('path', ''),
+                content=args.get('content', '')
+            )
+        elif tool_name == 'shell_execute':
+            return ctx.shell_execute(command=args.get('command', ''))
+        elif tool_name == 'create_shadow_branch':
+            return ctx.create_shadow_branch()
+        else:
+            return f"Unknown tool: {tool_name}"
+    except Exception as e:
+        return f"Execution error: {e}"
+
+
+# =============================================================================
+# FILE UPLOAD HANDLER
+# =============================================================================
+# CHANGELOG [2026-02-01 - Claude/Opus]
+# Reads uploaded files and formats them for injection into the conversation.
+# Supports code files, text, JSON, markdown, etc. Binary files get a
+# placeholder message since they can't be meaningfully injected as text.
+# =============================================================================
+
+TEXT_EXTENSIONS = {
+    '.py', '.js', '.ts', '.jsx', '.tsx', '.json', '.yaml', '.yml',
+    '.md', '.txt', '.rst', '.html', '.css', '.scss', '.sh', '.bash',
+    '.sql', '.toml', '.cfg', '.ini', '.conf', '.xml', '.csv',
+    '.env', '.gitignore', '.dockerignore', '.mjs', '.cjs',
+}
+
+
+def process_uploaded_file(file) -> str:
+    """Read an uploaded file and format it for conversation context.
+
+    Args:
+        file: Gradio file object with .name attribute (temp path)
+
+    Returns:
+        Formatted string with filename and content, ready to inject
+        into the conversation as context
+    """
+    if file is None:
+        return ""
+
+    file_path = file.name if hasattr(file, 'name') else str(file)
+    file_name = os.path.basename(file_path)
+    suffix = os.path.splitext(file_name)[1].lower()
+
+    if suffix in TEXT_EXTENSIONS or suffix == '':
+        try:
+            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                content = f.read()
+            # Cap at 50KB to avoid overwhelming context
+            if len(content) > 50000:
+                content = content[:50000] + f"\n\n... (truncated, {len(content)} total chars)"
+            return f"📎 **Uploaded: {file_name}**\n```\n{content}\n```"
+        except Exception as e:
+            return f"📎 **Uploaded: {file_name}** (error reading: {e})"
+    else:
+        return f"📎 **Uploaded: {file_name}** (binary file, {os.path.getsize(file_path):,} bytes)"
+
+
+# =============================================================================
+# AGENTIC LOOP
+# =============================================================================
+# CHANGELOG [2026-02-01 - Claude/Opus]
+# The core conversation loop. For each user message:
+# 1. Build messages array with system prompt + history + new message
+# 2. Send to Kimi K2.5 via HF Inference API
+# 3. Parse response for tool calls
+# 4. If READ tool calls: execute immediately, inject results, loop back to Kimi
+# 5. If WRITE tool calls: stage in approval queue, notify user
+# 6. If no tool calls: return conversational response
+# 7. Save the turn to ChromaDB for persistent memory
+#
+# The loop runs up to MAX_ITERATIONS times to handle multi-step tool use.
+# Each iteration either executes tools and loops, or returns the final text.
+#
+# IMPORTANT: Gradio 5.0+ chatbot with type="messages" expects history as a
+# list of {"role": str, "content": str} dicts. We maintain that format
+# throughout.
+# =============================================================================
+
+MAX_ITERATIONS = 5
+
+
+def agent_loop(message: str, history: list, pending_proposals: list, uploaded_file) -> tuple:
+    """Main agentic conversation loop.
+
+    Args:
+        message: User's text input
+        history: Chat history as list of {"role": ..., "content": ...} dicts
+        pending_proposals: Current list of staged write proposals (gr.State)
+        uploaded_file: Optional uploaded file from the file input widget
+
+    Returns:
+        Tuple of (updated_history, cleared_textbox, updated_proposals,
+                  updated_gate_choices, updated_stats_files, updated_stats_convos)
+    """
+    if not message.strip() and uploaded_file is None:
+        # Nothing to do
+        return history, "", pending_proposals, _format_gate_choices(pending_proposals), gr.update(), gr.update()
+
+    # Inject uploaded file content if present
+    full_message = message.strip()
+    if uploaded_file is not None:
+        file_context = process_uploaded_file(uploaded_file)
+        if file_context:
+            full_message = f"{file_context}\n\n{full_message}" if full_message else file_context
+
+    if not full_message:
+        return history, "", pending_proposals, _format_gate_choices(pending_proposals), gr.update(), gr.update()
+
+    # Add user message to history
+    history = history + [{"role": "user", "content": full_message}]
+
+    # Build messages for the API
+    system_prompt = build_system_prompt()
+    api_messages = [{"role": "system", "content": system_prompt}]
+
+    # Include recent history (cap to avoid token overflow)
+    # Keep last 20 turns to stay within Kimi's context window
+    recent_history = history[-40:]  # 40 entries = ~20 turns (user+assistant pairs)
+    for h in recent_history:
+        api_messages.append({"role": h["role"], "content": h["content"]})
+
+    # Agentic loop: tool calls → execution → re-prompt → repeat
+    accumulated_text = ""
+    staged_this_turn = []
+
+    for iteration in range(MAX_ITERATIONS):
+        try:
+            response = client.chat_completion(
+                model=MODEL_ID,
+                messages=api_messages,
+                max_tokens=2048,
+                temperature=0.7
+            )
+            content = response.choices[0].message.content or ""
+        except Exception as e:
+            error_msg = f"⚠️ API Error: {e}"
+            history = history + [{"role": "assistant", "content": error_msg}]
+            return (
+                history, "", pending_proposals,
+                _format_gate_choices(pending_proposals),
+                _stats_label_files(), _stats_label_convos()
+            )
+
+        # Parse for tool calls
+        tool_calls = parse_tool_calls(content)
+        conversational_text = extract_conversational_text(content)
+
+        if conversational_text:
+            accumulated_text += ("\n\n" if accumulated_text else "") + conversational_text
+
+        if not tool_calls:
+            # No tools — this is the final response
+            break
+
+        # Process each tool call
+        tool_results_for_context = []
+        for tool_name, args in tool_calls:
+            result = execute_tool(tool_name, args)
+
+            if result["status"] == "executed":
+                # READ tool — executed, feed result back to model
+                tool_results_for_context.append(
+                    f"[Tool Result: {tool_name}]\n{result['result']}"
+                )
+            elif result["status"] == "staged":
+                # WRITE tool — staged for approval
+                proposal = {
+                    "id": f"proposal_{int(time.time())}_{tool_name}",
+                    "tool": tool_name,
+                    "args": result["args"],
+                    "description": result["description"],
+                    "timestamp": time.strftime("%H:%M:%S")
+                }
+                staged_this_turn.append(proposal)
+                tool_results_for_context.append(
+                    f"[Tool {tool_name}: STAGED for human approval. "
+                    f"Josh will review this in the Build Approval Gate.]"
+                )
+            elif result["status"] == "error":
+                tool_results_for_context.append(
+                    f"[Tool Error: {tool_name}]\n{result['result']}"
+                )
+
+        # If we only had staged tools and no reads, break the loop
+        if tool_results_for_context:
+            # Feed tool results back as a system message for the next iteration
+            combined_results = "\n\n".join(tool_results_for_context)
+            api_messages.append({"role": "assistant", "content": content})
+            api_messages.append({"role": "user", "content": f"[Tool Results]\n{combined_results}"})
+        else:
+            break
+
+    # Build final response
+    final_response = accumulated_text
+
+    # Append staging notifications if any writes were staged
+    if staged_this_turn:
+        staging_notice = "\n\n---\n🛡️ **Staged for your approval** (see Build Approval Gate tab):\n"
+        for proposal in staged_this_turn:
+            staging_notice += f"- {proposal['description']}\n"
+        final_response += staging_notice
+        # Add to persistent queue
+        pending_proposals = pending_proposals + staged_this_turn
+
+    if not final_response:
+        final_response = "🤔 I processed your request but didn't generate a text response. Check the Build Approval Gate if I staged any operations."
+
+    # Add assistant response to history
+    history = history + [{"role": "assistant", "content": final_response}]
+
+    # Save conversation turn for persistent memory
+    try:
+        turn_count = len([h for h in history if h["role"] == "user"])
+        ctx.save_conversation_turn(full_message, final_response, turn_count)
+    except Exception:
+        pass  # Don't crash the UI if persistence fails
+
+    return (
+        history,
+        "",  # Clear the textbox
+        pending_proposals,
+        _format_gate_choices(pending_proposals),
+        _stats_label_files(),
+        _stats_label_convos()
+    )
+
+
+# =============================================================================
+# BUILD APPROVAL GATE
+# =============================================================================
+# CHANGELOG [2026-02-01 - Claude/Opus]
+# The HITL gate for reviewing and approving staged write operations.
+# Josh sees a checklist of proposed changes, can select which to approve,
+# and clicks Execute. Approved operations run; rejected ones are discarded.
+#
+# DESIGN DECISION: CheckboxGroup shows descriptions, but we need to map
+# back to the actual proposal objects for execution. We use the proposal
+# ID as the checkbox value and display the description as the label.
+# =============================================================================
+
+def _format_gate_choices(proposals: list) -> gr.update:
+    """Format pending proposals as CheckboxGroup choices.
+
+    Args:
+        proposals: List of proposal dicts from staging
+
+    Returns:
+        gr.update with choices list for the CheckboxGroup
+    """
+    if not proposals:
+        return gr.update(choices=[], value=[])
+
+    choices = []
+    for p in proposals:
+        label = f"[{p['timestamp']}] {p['description']}"
+        choices.append((label, p['id']))
+    return gr.update(choices=choices, value=[])
+
+
+def execute_approved_proposals(selected_ids: list, pending_proposals: list) -> tuple:
+    """Execute approved proposals and remove them from the queue.
+
+    Args:
+        selected_ids: List of proposal IDs that Josh approved
+        pending_proposals: Full list of pending proposals
+
+    Returns:
+        Tuple of (results_markdown, updated_proposals, updated_gate_choices)
+    """
+    if not selected_ids:
+        return "No proposals selected.", pending_proposals, _format_gate_choices(pending_proposals)
+
+    results = []
+    remaining = []
+
+    for proposal in pending_proposals:
+        if proposal['id'] in selected_ids:
+            # Execute this one
+            result = execute_staged_tool(proposal['tool'], proposal['args'])
+            results.append(f"**{proposal['tool']}**: {result}")
+        else:
+            # Keep in queue
+            remaining.append(proposal)
+
+    results_text = "## Execution Results\n\n" + "\n\n".join(results) if results else "Nothing executed."
+    return results_text, remaining, _format_gate_choices(remaining)
+
+
+def clear_all_proposals(pending_proposals: list) -> tuple:
+    """Discard all pending proposals without executing.
+
+    CHANGELOG [2026-02-01 - Claude/Opus]
+    Safety valve — lets Josh throw out everything in the queue if the
+    agent went off track.
+
+    Returns:
+        Tuple of (status_message, empty_proposals, updated_gate_choices)
+    """
+    count = len(pending_proposals)
+    return f"🗑️ Cleared {count} proposal(s).", [], _format_gate_choices([])
+
+
+# =============================================================================
+# STATS HELPERS
+# =============================================================================
+# CHANGELOG [2026-02-01 - Claude/Opus]
+# Helper functions to format stats for the sidebar labels.
+# Called both at startup (initial render) and after each conversation turn
+# (to reflect newly indexed files or saved conversations).
+# =============================================================================
+
+def _stats_label_files() -> str:
+    """Format the files stat for the sidebar label."""
+    stats = ctx.get_stats()
+    files = stats.get('total_files', 0)
+    chunks = stats.get('indexed_chunks', 0)
+    indexing = " ⏳" if stats.get('indexing_in_progress') else ""
+    return f"📂 Files: {files} ({chunks} chunks){indexing}"
+
+
+def _stats_label_convos() -> str:
+    """Format the conversations stat for the sidebar label."""
+    stats = ctx.get_stats()
+    convos = stats.get('conversations', 0)
+    cloud = " ☁️" if stats.get('persistence_configured') else ""
+    return f"💾 Conversations: {convos}{cloud}"
+
+
+def refresh_stats() -> tuple:
+    """Refresh both stat labels. Called by the refresh button.
+
+    Returns:
+        Tuple of (files_label, convos_label)
+    """
+    return _stats_label_files(), _stats_label_convos()
+
+
+# =============================================================================
+# UI LAYOUT
+# =============================================================================
+# CHANGELOG [2026-02-01 - Gemini]
+# RESTORED: Metrics sidebar and multi-tab layout.
+#
+# CHANGELOG [2026-02-01 - Claude/Opus]
+# IMPLEMENTED: All the wiring. Every button, input, and display is now
+# connected to actual functions.
+#
+# Layout:
+# Tab 1 "Vibe Chat" — Main conversation interface with sidebar stats
+# Tab 2 "Build Approval Gate" — HITL review for staged write operations
+#
+# gr.State holds the pending proposals list (per-session, survives across
+# messages within the same browser tab).
+# =============================================================================
+
+with gr.Blocks(
+    title="🦞 Clawdbot Command Center",
+    theme=gr.themes.Soft()
+) as demo:
+    # Session state for pending proposals
+    pending_proposals_state = gr.State([])
+
+    gr.Markdown("# 🦞 Clawdbot Command Center\n*E-T Systems Vibe Coding Agent*")
+
+    with gr.Tabs():
+        # ==== TAB 1: VIBE CHAT ====
+        with gr.Tab("💬 Vibe Chat"):
+            with gr.Row():
+                # ---- Sidebar ----
+                with gr.Column(scale=1, min_width=200):
+                    gr.Markdown("### 📊 System Status")
+                    stats_files = gr.Markdown(_stats_label_files())
+                    stats_convos = gr.Markdown(_stats_label_convos())
+                    refresh_btn = gr.Button("🔄 Refresh Stats", size="sm")
+
+                    gr.Markdown("---")
+                    gr.Markdown("### 📎 Upload Context")
+                    file_input = gr.File(
+                        label="Drop a file here",
+                        file_types=[
+                            '.py', '.js', '.ts', '.json', '.md', '.txt',
+                            '.yaml', '.yml', '.html', '.css', '.sh',
+                            '.toml', '.cfg', '.csv', '.xml'
+                        ]
+                    )
+                    gr.Markdown(
+                        "*Upload code, configs, or docs to include in your message.*"
+                    )
+
+                # ---- Chat area ----
+                with gr.Column(scale=4):
+                    chatbot = gr.Chatbot(
+                        type="messages",
+                        height=600,
+                        show_label=False,
+                        avatar_images=(None, "https://em-content.zobj.net/source/twitter/408/lobster_1f99e.png"),
+                    )
+                    with gr.Row():
+                        msg = gr.Textbox(
+                            placeholder="Ask Clawdbot to search, read, or code...",
+                            show_label=False,
+                            scale=6,
+                            lines=2,
+                            max_lines=10,
+                        )
+                        send_btn = gr.Button("Send", variant="primary", scale=1)
+
+            # Wire up chat submission
+            chat_inputs = [msg, chatbot, pending_proposals_state, file_input]
+            chat_outputs = [
+                chatbot, msg, pending_proposals_state,
+                # These reference components in the Gate tab — defined below
+            ]
+
+        # ==== TAB 2: BUILD APPROVAL GATE ====
+        with gr.Tab("🛡️ Build Approval Gate"):
+            gr.Markdown(
+                "### Review Staged Operations\n"
+                "Write operations (file writes, shell commands, branch creation) "
+                "are staged here for your review before execution.\n\n"
+                "**Select proposals to approve, then click Execute.**"
+            )
+            gate_list = gr.CheckboxGroup(
+                label="Pending Proposals",
+                choices=[],
+                interactive=True
+            )
+            with gr.Row():
+                btn_exec = gr.Button("✅ Execute Selected", variant="primary")
+                btn_clear = gr.Button("🗑️ Clear All", variant="secondary")
+            gate_results = gr.Markdown("*No operations executed yet.*")
+
+    # ==================================================================
+    # EVENT WIRING
+    # ==================================================================
+    # CHANGELOG [2026-02-01 - Claude/Opus]
+    # All events are wired here, after all components are defined, so
+    # cross-tab references work (e.g., chat updating the gate_list).
+    # ==================================================================
+
+    # Chat submission (both Enter key and Send button)
+    full_chat_outputs = [
+        chatbot, msg, pending_proposals_state,
+        gate_list, stats_files, stats_convos
+    ]
+
+    msg.submit(
+        fn=agent_loop,
+        inputs=chat_inputs,
+        outputs=full_chat_outputs
+    )
+    send_btn.click(
+        fn=agent_loop,
+        inputs=chat_inputs,
+        outputs=full_chat_outputs
+    )
+
+    # Refresh stats button
+    refresh_btn.click(
+        fn=refresh_stats,
+        inputs=[],
+        outputs=[stats_files, stats_convos]
+    )
+
+    # Build Approval Gate buttons
+    btn_exec.click(
+        fn=execute_approved_proposals,
+        inputs=[gate_list, pending_proposals_state],
+        outputs=[gate_results, pending_proposals_state, gate_list]
+    )
+    btn_clear.click(
+        fn=clear_all_proposals,
+        inputs=[pending_proposals_state],
+        outputs=[gate_results, pending_proposals_state, gate_list]
+    )
+
+
+# =============================================================================
+# LAUNCH
+# =============================================================================
+# CHANGELOG [2026-02-01 - Claude/Opus]
+# Standard HF Spaces launch config. 0.0.0.0 binds to all interfaces
+# (required for Docker). Port 7860 is the HF Spaces standard.
+# =============================================================================
+
+if __name__ == "__main__":
+    demo.launch(server_name="0.0.0.0", server_port=7860)

recursive_context.py

@@ -0,0 +1,750 @@
+"""
+Recursive Context Manager for Clawdbot
+
+CHANGELOG [2025-01-28 - Josh]
+CREATED: Initial recursive context manager with ChromaDB vector search,
+  file reading, and conversation persistence. Based on MIT Recursive
+  Language Model technique for unlimited context.
+
+CHANGELOG [2026-01-31 - Gemini]
+ADDED: Phase 1 Orchestrator tools: create_shadow_branch, write_file, shell_execute.
+ADDED: Documentation Scanner to mandate Living Changelog headers.
+FIXED: PermissionError on /.cache by forcing ONNXMiniLM_L6_V2.DOWNLOAD_PATH.
+
+CHANGELOG [2026-01-31 - Claude/Opus]
+ADDED: get_stats() method — was called by app.py but never defined, causing
+  crash on startup. Returns dict with file counts, conversation counts,
+  collection sizes, and persistence status.
+ADDED: list_files() method — directory exploration tool for the agent.
+  Returns tree of files/dirs at a given path relative to repo root.
+ADDED: search_conversations() method — semantic search over saved conversation
+  history in ChromaDB. Essential for persistent memory across sessions.
+ADDED: search_testament() method — searches for Testament/architectural decision
+  files and returns matching content. Falls back to codebase search if no
+  dedicated testament files exist.
+ADDED: index_repository() method — actually indexes the repo into ChromaDB on
+  init. Without this, search_code() always returned empty because nothing
+  was ever added to the codebase collection. Runs in background thread to
+  avoid blocking startup.
+PRESERVED: All existing functions from prior changelogs remain intact.
+  HFDatasetPersistence class, create_shadow_branch, write_file, shell_execute,
+  search_code, read_file, save_conversation_turn — all unchanged.
+NOTE: get_stats() is critical — app.py calls it at module level during UI
+  construction AND in the system prompt. Missing it = instant crash.
+"""
+
+from pathlib import Path
+from typing import List, Dict, Optional, Tuple
+import chromadb
+from chromadb.config import Settings
+from chromadb.utils.embedding_functions import ONNXMiniLM_L6_V2
+import hashlib
+import json
+import os
+import time
+import threading
+import subprocess
+import re
+
+
+# =============================================================================
+# CHROMA DB PATH SELECTION
+# =============================================================================
+# CHANGELOG [2026-01-31 - Gemini]
+# HF Spaces Docker containers wipe everything EXCEPT /data on restart.
+# We prefer /data/chroma_db (persistent) but fall back to /workspace/chroma_db
+# (ephemeral) if /data isn't writable.
+# =============================================================================
+
+def _select_chroma_path():
+    """HF Spaces Docker containers wipe everything EXCEPT /data on restart."""
+    data_path = Path("/data/chroma_db")
+    try:
+        data_path.mkdir(parents=True, exist_ok=True)
+        test_file = data_path / ".write_test"
+        test_file.write_text("test")
+        test_file.unlink()
+        return str(data_path)
+    except (OSError, PermissionError):
+        workspace_path = Path("/workspace/chroma_db")
+        workspace_path.mkdir(parents=True, exist_ok=True)
+        return str(workspace_path)
+
+
+CHROMA_DB_PATH = _select_chroma_path()
+
+
+# =============================================================================
+# HF DATASET PERSISTENCE
+# =============================================================================
+# CHANGELOG [2026-01-31 - Gemini]
+# Handles durable cloud storage via HF Dataset repository. Conversations
+# survive Space restarts by backing up to a private dataset repo.
+# =============================================================================
+
+class HFDatasetPersistence:
+    """Handles durable cloud storage via your 1TB PRO Dataset repository."""
+
+    def __init__(self, repo_id: str = None):
+        from huggingface_hub import HfApi
+        self.api = HfApi()
+        self.repo_id = repo_id or os.getenv("MEMORY_REPO")
+        self.token = os.getenv("HF_TOKEN") or os.getenv("HUGGINGFACE_TOKEN")
+        self._repo_ready = False
+
+        if self.repo_id and self.token:
+            self._ensure_repo_exists()
+
+    def _ensure_repo_exists(self):
+        if self._repo_ready:
+            return
+        try:
+            self.api.repo_info(
+                repo_id=self.repo_id,
+                repo_type="dataset",
+                token=self.token
+            )
+            self._repo_ready = True
+        except Exception:
+            try:
+                self.api.create_repo(
+                    repo_id=self.repo_id,
+                    repo_type="dataset",
+                    private=True,
+                    token=self.token
+                )
+                self._repo_ready = True
+            except Exception:
+                pass
+
+    @property
+    def is_configured(self):
+        return bool(self.repo_id and self.token)
+
+    def save_conversations(self, data: List[Dict]):
+        if not self.is_configured:
+            return
+        temp = Path("/tmp/conv_backup.json")
+        temp.write_text(json.dumps(data, indent=2))
+        try:
+            self.api.upload_file(
+                path_or_fileobj=str(temp),
+                path_in_repo="conversations.json",
+                repo_id=self.repo_id,
+                repo_type="dataset",
+                token=self.token
+            )
+        except Exception:
+            pass
+
+    def load_conversations(self) -> List[Dict]:
+        if not self.is_configured:
+            return []
+        try:
+            from huggingface_hub import hf_hub_download
+            local_path = hf_hub_download(
+                repo_id=self.repo_id,
+                filename="conversations.json",
+                repo_type="dataset",
+                token=self.token
+            )
+            with open(local_path, 'r') as f:
+                return json.load(f)
+        except Exception:
+            return []
+
+
+# =============================================================================
+# RECURSIVE CONTEXT MANAGER
+# =============================================================================
+
+class RecursiveContextManager:
+    """Manages unlimited context and vibe-coding tools for E-T Systems.
+
+    CHANGELOG [2026-01-31 - Claude/Opus]
+    This is the core class. It provides:
+    - ChromaDB-backed semantic search over the codebase and conversations
+    - File read/write with changelog enforcement
+    - Shell execution for build tasks
+    - Shadow branching for safe experimentation
+    - Stats reporting for the UI sidebar
+    - Repository indexing (background thread on init)
+
+    ARCHITECTURE NOTE:
+    The class is initialized once at module level in app.py. That means
+    __init__ runs during import, so it MUST NOT block or crash. Heavy work
+    (like indexing the repo) is dispatched to a background thread.
+    get_stats() must return sensible defaults even before indexing completes.
+    """
+
+    # =========================================================================
+    # FILE EXTENSIONS TO INDEX
+    # =========================================================================
+    # CHANGELOG [2026-01-31 - Claude/Opus]
+    # Only index code/text files. Binary files, images, and large data files
+    # would pollute the vector space and waste embedding compute.
+    # =========================================================================
+    INDEXABLE_EXTENSIONS = {
+        '.py', '.js', '.ts', '.jsx', '.tsx', '.mjs', '.cjs',
+        '.json', '.yaml', '.yml', '.toml',
+        '.md', '.txt', '.rst',
+        '.html', '.css', '.scss',
+        '.sh', '.bash',
+        '.sql',
+        '.env.example',  # Not .env itself — that's sensitive
+        '.gitignore', '.dockerignore',
+        '.cfg', '.ini', '.conf',
+    }
+
+    # Max file size to index (256KB). Larger files are likely generated/data.
+    MAX_INDEX_SIZE = 256 * 1024
+
+    def __init__(self, repo_path: str):
+        self.repo_path = Path(repo_path)
+        self.persistence = HFDatasetPersistence()
+
+        # =================================================================
+        # EMBEDDING CONFIG
+        # =================================================================
+        # CHANGELOG [2026-01-31 - Gemini]
+        # Fixes /.cache PermissionError. ChromaDB's ONNXMiniLM_L6_V2 tries
+        # to download model weights to ~/.cache. In Docker as UID 1000,
+        # that's /.cache (root-owned). We override DOWNLOAD_PATH to a
+        # writable directory.
+        # =================================================================
+        self.embedding_function = ONNXMiniLM_L6_V2()
+        cache_dir = os.getenv("CHROMA_CACHE_DIR", "/tmp/.cache/chroma")
+        self.embedding_function.DOWNLOAD_PATH = cache_dir
+        os.makedirs(cache_dir, exist_ok=True)
+
+        self.chroma_client = chromadb.PersistentClient(
+            path=CHROMA_DB_PATH,
+            settings=Settings(anonymized_telemetry=False, allow_reset=True)
+        )
+
+        c_name = self._get_collection_name()
+        self.collection = self.chroma_client.get_or_create_collection(
+            name=c_name,
+            embedding_function=self.embedding_function
+        )
+        self.conversations = self.chroma_client.get_or_create_collection(
+            name=f"conv_{c_name.split('_')[1]}",
+            embedding_function=self.embedding_function
+        )
+
+        # Restore conversations from cloud backup if local is empty
+        if self.conversations.count() == 0:
+            self._restore_from_cloud()
+
+        # =================================================================
+        # BACKGROUND INDEXING
+        # =================================================================
+        # CHANGELOG [2026-01-31 - Claude/Opus]
+        # Index the repository in a background thread so startup isn't
+        # blocked. The _indexing flag lets get_stats() report status.
+        # =================================================================
+        self._indexing = False
+        self._index_error = None
+        self._indexed_file_count = 0
+        if self.repo_path.exists() and self.repo_path.is_dir():
+            self._start_background_indexing()
+
+    def _restore_from_cloud(self):
+        """Restore conversation history from HF Dataset backup.
+
+        CHANGELOG [2026-01-31 - Gemini]
+        Called during init if the local ChromaDB conversations collection
+        is empty. Pulls from the cloud dataset repo to recover history
+        after a Space restart.
+        """
+        data = self.persistence.load_conversations()
+        for conv in data:
+            try:
+                self.conversations.add(
+                    documents=[conv["document"]],
+                    metadatas=[conv["metadata"]],
+                    ids=[conv["id"]]
+                )
+            except Exception:
+                pass
+
+    def _get_collection_name(self) -> str:
+        """Generate a deterministic collection name from the repo path.
+
+        CHANGELOG [2025-01-28 - Josh]
+        Uses MD5 hash of repo path so different repos get different
+        collections within the same ChromaDB instance.
+        """
+        path_hash = hashlib.md5(str(self.repo_path).encode()).hexdigest()[:8]
+        return f"codebase_{path_hash}"
+
+    # =====================================================================
+    # REPOSITORY INDEXING
+    # =====================================================================
+    # CHANGELOG [2026-01-31 - Claude/Opus]
+    # Without indexing, search_code() always returns empty results because
+    # nothing is ever added to the ChromaDB codebase collection. This walks
+    # the repo, reads indexable files, chunks them, and upserts into ChromaDB.
+    #
+    # DESIGN DECISIONS:
+    # - Background thread: Don't block Gradio startup. Users can chat while
+    #   indexing runs. get_stats() shows indexing progress.
+    # - Chunk by logical blocks: Split files into ~50-line chunks with overlap
+    #   so semantic search finds relevant sections, not just file-level matches.
+    # - Upsert (not add): Safe to re-run. If the file was already indexed
+    #   with the same content hash, ChromaDB skips it.
+    # - Skip .git, __pycache__, node_modules, venv: No value in indexing these.
+    #
+    # TESTED ALTERNATIVES (graveyard):
+    # - Indexing entire files as single documents: Poor search precision.
+    #   A 500-line file matching on line 3 returns all 500 lines.
+    # - Line-by-line indexing: Too many tiny documents, poor semantic context.
+    # - Synchronous indexing: Blocks startup for 30+ seconds on large repos.
+    # =====================================================================
+
+    def _start_background_indexing(self):
+        """Kick off repo indexing in a daemon thread."""
+        self._indexing = True
+        thread = threading.Thread(target=self._index_repository, daemon=True)
+        thread.start()
+
+    def _index_repository(self):
+        """Walk the repo and index code files into ChromaDB.
+
+        Runs in background thread. Sets self._indexing = False when done.
+        """
+        try:
+            skip_dirs = {
+                '.git', '__pycache__', 'node_modules', 'venv', '.venv',
+                'env', '.eggs', 'dist', 'build', '.next', '.nuxt',
+                'chroma_db', '.chroma'
+            }
+            count = 0
+
+            for file_path in self.repo_path.rglob('*'):
+                # Skip directories and non-indexable files
+                if file_path.is_dir():
+                    continue
+
+                # Skip files in excluded directories
+                if any(skip in file_path.parts for skip in skip_dirs):
+                    continue
+
+                # Check extension
+                suffix = file_path.suffix.lower()
+                if suffix not in self.INDEXABLE_EXTENSIONS:
+                    # Also allow extensionless files if they look like configs
+                    if file_path.name not in {
+                        'Dockerfile', 'Makefile', 'Procfile',
+                        '.gitignore', '.dockerignore', '.env.example'
+                    }:
+                        continue
+
+                # Check size
+                try:
+                    if file_path.stat().st_size > self.MAX_INDEX_SIZE:
+                        continue
+                except OSError:
+                    continue
+
+                # Read and chunk the file
+                try:
+                    content = file_path.read_text(encoding='utf-8', errors='ignore')
+                except (OSError, UnicodeDecodeError):
+                    continue
+
+                if not content.strip():
+                    continue
+
+                rel_path = str(file_path.relative_to(self.repo_path))
+                chunks = self._chunk_file(content, rel_path)
+
+                for chunk_id, chunk_text, chunk_meta in chunks:
+                    try:
+                        self.collection.upsert(
+                            documents=[chunk_text],
+                            metadatas=[chunk_meta],
+                            ids=[chunk_id]
+                        )
+                    except Exception:
+                        continue
+
+                count += 1
+                self._indexed_file_count = count
+
+        except Exception as e:
+            self._index_error = str(e)
+        finally:
+            self._indexing = False
+
+    def _chunk_file(self, content: str, rel_path: str) -> List[Tuple[str, str, dict]]:
+        """Split a file into overlapping chunks for better search precision.
+
+        CHANGELOG [2026-01-31 - Claude/Opus]
+        Returns list of (id, text, metadata) tuples ready for ChromaDB upsert.
+        Chunks are ~50 lines with 10-line overlap so context isn't lost at
+        chunk boundaries.
+
+        Args:
+            content: Full file text
+            rel_path: Path relative to repo root (used in metadata and IDs)
+
+        Returns:
+            List of (chunk_id, chunk_text, metadata_dict) tuples
+        """
+        lines = content.split('\n')
+        chunks = []
+        chunk_size = 50
+        overlap = 10
+
+        if len(lines) <= chunk_size:
+            # Small file — index as single chunk
+            content_hash = hashlib.md5(content.encode()).hexdigest()[:12]
+            chunk_id = f"{rel_path}::full::{content_hash}"
+            meta = {
+                'path': rel_path,
+                'chunk': 'full',
+                'lines': f"1-{len(lines)}",
+                'total_lines': len(lines)
+            }
+            chunks.append((chunk_id, content, meta))
+        else:
+            # Larger file — split into overlapping chunks
+            start = 0
+            chunk_num = 0
+            while start < len(lines):
+                end = min(start + chunk_size, len(lines))
+                chunk_text = '\n'.join(lines[start:end])
+                content_hash = hashlib.md5(chunk_text.encode()).hexdigest()[:12]
+                chunk_id = f"{rel_path}::chunk{chunk_num}::{content_hash}"
+                meta = {
+                    'path': rel_path,
+                    'chunk': f"chunk_{chunk_num}",
+                    'lines': f"{start + 1}-{end}",
+                    'total_lines': len(lines)
+                }
+                chunks.append((chunk_id, chunk_text, meta))
+                chunk_num += 1
+                start += chunk_size - overlap
+
+        return chunks
+
+    # =====================================================================
+    # STATS (NEW — was missing, caused crash)
+    # =====================================================================
+    # CHANGELOG [2026-01-31 - Claude/Opus]
+    # app.py calls ctx.get_stats() at module level during Gradio Block
+    # construction AND in the system prompt for every message. It expected
+    # a dict with 'conversations', 'total_files', etc. Without this method,
+    # the app crashes immediately on import.
+    #
+    # Returns safe defaults during indexing so the UI can render.
+    # =====================================================================
+
+    def get_stats(self) -> dict:
+        """Return system statistics for the UI sidebar and system prompt.
+
+        Returns:
+            dict with keys: total_files, indexed_chunks, conversations,
+            chroma_path, persistence_configured, indexing_in_progress,
+            index_error
+        """
+        return {
+            'total_files': self._indexed_file_count,
+            'indexed_chunks': self.collection.count(),
+            'conversations': self.conversations.count(),
+            'chroma_path': CHROMA_DB_PATH,
+            'persistence_configured': self.persistence.is_configured,
+            'indexing_in_progress': self._indexing,
+            'index_error': self._index_error,
+        }
+
+    # =====================================================================
+    # PHASE 1 ORCHESTRATOR TOOLS (preserved from Gemini)
+    # =====================================================================
+
+    def create_shadow_branch(self):
+        """Creates a timestamped backup branch of the E-T Systems Space.
+
+        CHANGELOG [2026-01-31 - Gemini]
+        Safety net before any destructive operations. Creates a branch
+        named vibe-backup-YYYYMMDD-HHMMSS on the E-T Systems HF Space
+        so you can always roll back.
+        """
+        timestamp = time.strftime("%Y%m%d-%H%M%S")
+        branch_name = f"vibe-backup-{timestamp}"
+        try:
+            repo_id = os.getenv(
+                "ET_SYSTEMS_SPACE",
+                "Executor-Tyrant-Framework/Executor-Framworks_Full_VDB"
+            )
+            self.persistence.api.create_branch(
+                repo_id=repo_id,
+                branch=branch_name,
+                repo_type="space",
+                token=self.persistence.token
+            )
+            return f"🛡️ Shadow branch created: {branch_name}"
+        except Exception as e:
+            return f"⚠️ Shadow branch failed: {e}"
+
+    def write_file(self, path: str, content: str):
+        """Writes file strictly if valid CHANGELOG is present.
+
+        CHANGELOG [2026-01-31 - Gemini]
+        Enforces the living changelog pattern. Any code written by an agent
+        MUST include a CHANGELOG [YYYY-MM-DD - AgentName] header or the
+        write is rejected. This is non-negotiable for the E-T Systems
+        development workflow.
+
+        Args:
+            path: Relative path within the repo (e.g., "server/routes.ts")
+            content: Full file content (must contain CHANGELOG header)
+
+        Returns:
+            Success message or rejection reason
+        """
+        if not re.search(r"CHANGELOG \[\d{4}-\d{2}-\d{2} - \w+\]", content):
+            return "REJECTED: Missing mandatory CHANGELOG [YYYY-MM-DD - AgentName] header."
+
+        try:
+            full_path = self.repo_path / path
+            full_path.parent.mkdir(parents=True, exist_ok=True)
+            full_path.write_text(content)
+            return f"✅ Successfully wrote {path}"
+        except Exception as e:
+            return f"Error writing file: {e}"
+
+    def shell_execute(self, command: str):
+        """Runs shell commands in the /workspace directory.
+
+        CHANGELOG [2026-01-31 - Gemini]
+        Used for build tasks, git operations, dependency installs, etc.
+        Timeout of 30 seconds prevents runaway processes. Captures both
+        stdout and stderr for full diagnostic output.
+
+        Args:
+            command: Shell command string to execute
+
+        Returns:
+            Combined stdout/stderr output or error message
+        """
+        try:
+            result = subprocess.run(
+                command, shell=True, capture_output=True, text=True,
+                cwd=self.repo_path, timeout=30
+            )
+            return f"STDOUT: {result.stdout}\nSTDERR: {result.stderr}"
+        except Exception as e:
+            return f"Execution Error: {e}"
+
+    # =====================================================================
+    # RECURSIVE SEARCH TOOLS
+    # =====================================================================
+
+    def search_code(self, query: str, n: int = 5) -> List[Dict]:
+        """Semantic search across the indexed codebase.
+
+        CHANGELOG [2025-01-28 - Josh]
+        Core tool for the MIT recursive context technique. The model calls
+        this to find relevant code without loading the entire repo into
+        context.
+
+        Args:
+            query: Natural language search query
+            n: Max number of results to return (default 5)
+
+        Returns:
+            List of dicts with 'file' (path) and 'snippet' (first 500 chars)
+        """
+        if self.collection.count() == 0:
+            return []
+        actual_n = min(n, self.collection.count())
+        res = self.collection.query(query_texts=[query], n_results=actual_n)
+        return [
+            {"file": m['path'], "snippet": d[:500]}
+            for d, m in zip(res['documents'][0], res['metadatas'][0])
+        ]
+
+    def read_file(self, path: str, start_line: int = None, end_line: int = None) -> str:
+        """Read a specific file, optionally a line range.
+
+        CHANGELOG [2025-01-28 - Josh]
+        Direct file access for when the model knows exactly what it needs.
+
+        CHANGELOG [2026-01-31 - Claude/Opus]
+        Added optional start_line/end_line params for reading specific
+        sections without loading entire large files into context.
+
+        Args:
+            path: Relative path within repo (e.g., "server/routes.ts")
+            start_line: Optional 1-based start line
+            end_line: Optional 1-based end line
+
+        Returns:
+            File contents (full or sliced) or "File not found." message
+        """
+        p = self.repo_path / path
+        if not p.exists():
+            return f"File not found: {path}"
+        try:
+            content = p.read_text(encoding='utf-8', errors='ignore')
+            if start_line is not None or end_line is not None:
+                lines = content.split('\n')
+                start = (start_line or 1) - 1  # Convert to 0-based
+                end = end_line or len(lines)
+                sliced = lines[start:end]
+                return '\n'.join(sliced)
+            return content
+        except Exception as e:
+            return f"Error reading {path}: {e}"
+
+    def list_files(self, path: str = "", max_depth: int = 3) -> str:
+        """List files and directories at a given path.
+
+        CHANGELOG [2026-01-31 - Claude/Opus]
+        Directory exploration tool. The agent needs to know what files exist
+        before it can read or search them. Returns a tree-formatted listing
+        up to max_depth levels deep.
+
+        Args:
+            path: Relative path within repo (default "" = repo root)
+            max_depth: How many levels deep to list (default 3)
+
+        Returns:
+            Formatted string showing directory tree
+        """
+        target = self.repo_path / path
+        if not target.exists():
+            return f"Path not found: {path}"
+        if not target.is_dir():
+            return f"Not a directory: {path}"
+
+        skip_dirs = {
+            '.git', '__pycache__', 'node_modules', 'venv', '.venv',
+            'chroma_db', '.chroma', 'dist', 'build'
+        }
+
+        lines = [f"📂 {path or '(repo root)'}"]
+
+        def _walk(dir_path: Path, prefix: str, depth: int):
+            if depth > max_depth:
+                return
+            try:
+                entries = sorted(dir_path.iterdir(), key=lambda p: (not p.is_dir(), p.name.lower()))
+            except PermissionError:
+                return
+
+            for i, entry in enumerate(entries):
+                if entry.name in skip_dirs or entry.name.startswith('.'):
+                    continue
+                is_last = (i == len(entries) - 1)
+                connector = "└── " if is_last else "├── "
+                if entry.is_dir():
+                    lines.append(f"{prefix}{connector}📁 {entry.name}/")
+                    extension = "    " if is_last else "│   "
+                    _walk(entry, prefix + extension, depth + 1)
+                else:
+                    size = entry.stat().st_size
+                    size_str = f"{size:,}B" if size < 1024 else f"{size // 1024:,}KB"
+                    lines.append(f"{prefix}{connector}📄 {entry.name} ({size_str})")
+
+        _walk(target, "", 1)
+        return '\n'.join(lines)
+
+    def search_conversations(self, query: str, n: int = 5) -> List[Dict]:
+        """Semantic search over past conversation history.
+
+        CHANGELOG [2026-01-31 - Claude/Opus]
+        This is how Clawdbot "remembers" past discussions. Conversations
+        are saved to ChromaDB via save_conversation_turn() and backed up
+        to the HF Dataset repo. This searches them semantically.
+
+        Args:
+            query: Natural language search query
+            n: Max results to return
+
+        Returns:
+            List of dicts with 'content' and 'metadata' from matched turns
+        """
+        if self.conversations.count() == 0:
+            return []
+        actual_n = min(n, self.conversations.count())
+        res = self.conversations.query(query_texts=[query], n_results=actual_n)
+        results = []
+        for doc, meta in zip(res['documents'][0], res['metadatas'][0]):
+            results.append({
+                'content': doc[:1000],  # Cap at 1000 chars per result
+                'metadata': meta
+            })
+        return results
+
+    def search_testament(self, query: str, n: int = 5) -> List[Dict]:
+        """Search for Testament/architectural decision records.
+
+        CHANGELOG [2026-01-31 - Claude/Opus]
+        The Testament contains design decisions, constitutional principles,
+        and architectural rationale for E-T Systems. This searches for
+        testament-specific files first (TESTAMENT.md, DECISIONS.md, etc.),
+        then falls back to general codebase search filtered for decision-
+        related content.
+
+        Args:
+            query: What architectural decision to search for
+            n: Max results
+
+        Returns:
+            List of dicts with 'file' and 'snippet' from matching documents
+        """
+        # First, look for dedicated testament/decision files
+        testament_names = {
+            'testament', 'decisions', 'adr', 'architecture',
+            'principles', 'constitution', 'changelog', 'design'
+        }
+
+        testament_results = []
+        if self.collection.count() > 0:
+            # Search the codebase but prefer testament-like files
+            actual_n = min(n * 2, self.collection.count())  # Get extra, then filter
+            res = self.collection.query(query_texts=[query], n_results=actual_n)
+            for doc, meta in zip(res['documents'][0], res['metadatas'][0]):
+                path_lower = meta.get('path', '').lower()
+                # Check if this is a testament/decision file
+                is_testament = any(name in path_lower for name in testament_names)
+                testament_results.append({
+                    'file': meta['path'],
+                    'snippet': doc[:500],
+                    'is_testament': is_testament
+                })
+
+        # Sort: testament files first, then other matches
+        testament_results.sort(key=lambda r: (not r.get('is_testament', False)))
+        return testament_results[:n]
+
+    def save_conversation_turn(self, user_msg: str, assistant_msg: str, turn_id):
+        """Save a conversation turn to ChromaDB and cloud backup.
+
+        CHANGELOG [2025-01-28 - Josh]
+        Persistent memory across sessions. Every user/assistant exchange
+        gets embedded and stored in ChromaDB for semantic retrieval later.
+
+        CHANGELOG [2026-01-31 - Gemini]
+        Added cloud backup via HFDatasetPersistence so conversations survive
+        Space restarts.
+
+        Args:
+            user_msg: The user's message text
+            assistant_msg: The assistant's response text
+            turn_id: Conversation turn number for ordering
+        """
+        combined = f"USER: {user_msg}\n\nASSISTANT: {assistant_msg}"
+        unique_id = f"turn_{int(time.time())}_{turn_id}"
+        self.conversations.add(
+            documents=[combined],
+            metadatas=[{"turn": turn_id, "timestamp": int(time.time())}],
+            ids=[unique_id]
+        )
+        # Cloud backup
+        self.persistence.save_conversations([
+            {"document": combined, "metadata": {"turn": turn_id}, "id": unique_id}
+        ])