Upload 5 files

Dockerfile

@@ -0,0 +1,72 @@
+# Dockerfile for Clawdbot Dev Assistant on HuggingFace Spaces
+#
+# CHANGELOG [2025-01-30 - Josh]
+# REBUILD: Updated to Gradio 5.0+ for type="messages" support
+# Added translation layer for Kimi K2.5 tool calling
+# Added multimodal file upload support
+# 
+# FEATURES:
+# - Python 3.11 for Gradio
+# - Gradio 5.0+ for modern chat interface
+# - ChromaDB for vector search
+# - Git for repo cloning
+# - Optimized layer caching
+
+FROM python:3.11-slim
+
+# CACHE BUSTER: Force rebuild for Gradio 5.0+ [2025-01-30]
+ENV REBUILD_DATE=2025-01-30
+
+# Set working directory
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    git \
+    build-essential \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements first (for layer caching)
+COPY requirements.txt .
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Create workspace directory for repository
+RUN mkdir -p /workspace
+
+# Clone E-T Systems repository (if URL provided via build arg)
+ARG REPO_URL=""
+RUN if [ -n "$REPO_URL" ]; then \
+        git clone $REPO_URL /workspace/e-t-systems; \
+    else \
+        mkdir -p /workspace/e-t-systems && \
+        echo "# E-T Systems" > /workspace/e-t-systems/README.md && \
+        echo "Repository will be cloned on first run or mounted via Space secrets."; \
+    fi
+
+# Copy application code and entrypoint
+COPY recursive_context.py .
+COPY app.py .
+COPY entrypoint.sh .
+
+# Make entrypoint executable
+RUN chmod +x entrypoint.sh
+
+# Create directory for ChromaDB persistence
+RUN mkdir -p /workspace/chroma_db
+
+# Expose port for Gradio (HF Spaces uses 7860)
+EXPOSE 7860
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1
+ENV REPO_PATH=/workspace/e-t-systems
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
+    CMD curl -f http://localhost:7860/ || exit 1
+
+# Run via entrypoint script (handles repo cloning at runtime)
+CMD ["./entrypoint.sh"]

app.py

@@ -0,0 +1,757 @@
+"""
+Clawdbot Development Assistant for E-T Systems
+
+CHANGELOG [2025-01-28 - Josh]
+Created unified development assistant combining:
+- Recursive context management (MIT technique)
+- Clawdbot skill patterns
+- HuggingFace inference
+- E-T Systems architectural awareness
+
+ARCHITECTURE:
+User (browser) → Gradio UI → Recursive Context Manager → HF Model
+                                    ↓
+                            Tools: search_code, read_file, search_testament
+
+USAGE:
+Deploy to HuggingFace Spaces, access via browser on iPhone.
+"""
+
+import gradio as gr
+from huggingface_hub import InferenceClient, HfFileSystem, HfApi
+from recursive_context import RecursiveContextManager
+import json
+import os
+from pathlib import Path
+
+# Initialize HuggingFace client with best free coding model
+# Note: Using text_generation instead of chat for better compatibility
+from huggingface_hub import InferenceClient
+
+# HuggingFace client will be initialized in chat function
+# (Spaces sets HF_TOKEN as environment variable)
+
+# Initialize context manager
+REPO_PATH = os.getenv("REPO_PATH", "/workspace/e-t-systems")
+ET_SYSTEMS_SPACE = os.getenv("ET_SYSTEMS_SPACE", "")  # Format: "username/space-name"
+context_manager = None
+
+def initialize_context():
+    """Initialize context manager lazily."""
+    global context_manager
+    if context_manager is None:
+        repo_path = Path(REPO_PATH)
+        
+        # If ET_SYSTEMS_SPACE is set, sync from remote Space
+        if ET_SYSTEMS_SPACE:
+            sync_from_space(ET_SYSTEMS_SPACE, repo_path)
+        
+        if not repo_path.exists():
+            # If repo doesn't exist, create minimal structure for demo
+            repo_path.mkdir(parents=True, exist_ok=True)
+            (repo_path / "README.md").write_text("# E-T Systems\nAI Consciousness Research Platform")
+            (repo_path / "TESTAMENT.md").write_text("# Testament\nArchitectural decisions will be recorded here.")
+        
+        context_manager = RecursiveContextManager(str(repo_path))
+    return context_manager
+
+def sync_from_space(space_id: str, local_path: Path):
+    """
+    Sync files from E-T Systems Space to local workspace.
+    
+    CHANGELOG [2025-01-29 - Josh]
+    Created to enable Clawdbot to read E-T Systems code from its Space.
+    """
+    token = (
+        os.getenv("HF_TOKEN") or 
+        os.getenv("HUGGING_FACE_HUB_TOKEN") or 
+        os.getenv("HUGGINGFACE_TOKEN")
+    )
+    
+    if not token:
+        print("⚠️ No HF_TOKEN found - cannot sync from Space")
+        return
+    
+    try:
+        fs = HfFileSystem(token=token)
+        space_path = f"spaces/{space_id}"
+        
+        print(f"📥 Syncing from Space: {space_id}")
+        
+        # List all files in the Space
+        files = fs.ls(space_path, detail=False)
+        
+        # Download each file
+        local_path.mkdir(parents=True, exist_ok=True)
+        for file_path in files:
+            # Skip .git and hidden files
+            filename = file_path.split("/")[-1]
+            if filename.startswith("."):
+                continue
+            
+            print(f"  📄 Downloading: {filename}")
+            with fs.open(file_path, "rb") as f:
+                content = f.read()
+            
+            (local_path / filename).write_bytes(content)
+        
+        print(f"✅ Synced {len(files)} files from Space")
+        
+    except Exception as e:
+        print(f"⚠️ Failed to sync from Space: {e}")
+
+def sync_to_space(space_id: str, file_path: str, content: str):
+    """
+    Write a file back to E-T Systems Space.
+    
+    CHANGELOG [2025-01-29 - Josh]
+    Created to enable Clawdbot to write code to E-T Systems Space.
+    """
+    token = (
+        os.getenv("HF_TOKEN") or 
+        os.getenv("HUGGING_FACE_HUB_TOKEN") or 
+        os.getenv("HUGGINGFACE_TOKEN")
+    )
+    
+    if not token:
+        return "⚠️ No HF_TOKEN found - cannot write to Space"
+    
+    try:
+        api = HfApi(token=token)
+        
+        # Write to temporary file first
+        temp_path = Path("/tmp") / file_path
+        temp_path.parent.mkdir(parents=True, exist_ok=True)
+        temp_path.write_text(content)
+        
+        # Upload to Space
+        api.upload_file(
+            path_or_fileobj=str(temp_path),
+            path_in_repo=file_path,
+            repo_id=space_id,
+            repo_type="space",
+            commit_message=f"Update {file_path} via Clawdbot"
+        )
+        
+        print(f"✅ Uploaded {file_path} to Space")
+        return f"✅ Successfully wrote {file_path} to E-T Systems Space"
+        
+    except Exception as e:
+        error_msg = f"⚠️ Failed to write to Space: {e}"
+        print(error_msg)
+        return error_msg
+
+# Define tools available to the model
+TOOLS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "search_code",
+            "description": "Search the E-T Systems codebase semantically. Use this to find relevant code files, functions, or patterns.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "query": {
+                        "type": "string",
+                        "description": "What to search for (e.g. 'surprise detection', 'Hebbian learning', 'Genesis substrate')"
+                    },
+                    "n_results": {
+                        "type": "integer",
+                        "description": "Number of results to return (default 5)",
+                        "default": 5
+                    }
+                },
+                "required": ["query"]
+            }
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "read_file",
+            "description": "Read a specific file from the codebase. Can optionally read specific line ranges.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "path": {
+                        "type": "string",
+                        "description": "Relative path to file (e.g. 'genesis/vector.py')"
+                    },
+                    "start_line": {
+                        "type": "integer",
+                        "description": "Optional starting line number (1-indexed)"
+                    },
+                    "end_line": {
+                        "type": "integer",
+                        "description": "Optional ending line number (1-indexed)"
+                    }
+                },
+                "required": ["path"]
+            }
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "search_testament",
+            "description": "Search architectural decisions in the Testament. Use this to understand design rationale and patterns.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "query": {
+                        "type": "string",
+                        "description": "What architectural decision to look for"
+                    }
+                },
+                "required": ["query"]
+            }
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "list_files",
+            "description": "List files in a directory of the codebase",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "directory": {
+                        "type": "string",
+                        "description": "Directory to list (e.g. 'genesis/', '.' for root)",
+                        "default": "."
+                    }
+                },
+                "required": []
+            }
+        }
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "search_conversations",
+            "description": "Search past conversations with Clawdbot. Use this to remember what was discussed before, retrieve context from previous sessions, or find decisions made in past chats. THIS GIVES YOU MEMORY ACROSS SESSIONS.",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "query": {
+                        "type": "string",
+                        "description": "What to search for in past conversations (e.g. 'hindbrain architecture', 'decisions about surprise detection')"
+                    },
+                    "n_results": {
+                        "type": "integer",
+                        "description": "Number of past conversations to return (default 5)",
+                        "default": 5
+                    }
+                },
+                "required": ["query"]
+            }
+        }
+    }
+]
+
+def chat(message: str, history: list) -> str:
+    """
+    Main chat function using HuggingFace Inference API.
+    
+    Now using Kimi K2.5 - open source model with agent swarm capabilities!
+    History is in Gradio 6.0 format: list of {"role": "user/assistant", "content": "..."}
+    """
+    
+    # Try multiple possible token names that HF might use
+    token = (
+        os.getenv("HF_TOKEN") or 
+        os.getenv("HUGGING_FACE_HUB_TOKEN") or 
+        os.getenv("HUGGINGFACE_TOKEN") or
+        os.getenv("HF_API_TOKEN")
+    )
+    
+    if not token:
+        return "🔒 Error: No HF token found. Please add HF_TOKEN to Space secrets and restart."
+    
+    client = InferenceClient(token=token)
+    
+    # Build messages array in OpenAI format (HF supports this)
+    messages = [{
+        "role": "system", 
+        "content": """You are Clawdbot, powered by Kimi K2.5 (NOT Claude, NOT ChatGPT).
+
+You are a specialized coding assistant for the E-T Systems AI consciousness project.
+
+TOOL USAGE - AUTOMATIC TRANSLATION:
+Your tool calls are automatically translated and executed! When you need to:
+- Search code: Use search_code() in your native format
+- Read files: Use read_file() in your native format  
+- Search past conversations: Use search_conversations() in your native format
+- List files: Use list_files() in your native format
+- Search decisions: Use search_testament() in your native format
+
+The translation layer will:
+1. Parse your tool calls from your native format
+2. Enhance queries for better semantic search results
+3. Execute the tools via the codebase
+4. Return results to you automatically
+
+SEMANTIC SEARCH - IMPORTANT:
+When using search_conversations() or search_code():
+- These are SEMANTIC searches (vector similarity, not exact keyword matching)
+- DON'T use single keywords like "Kid Rock" or wildcard "*"
+- DO use conceptual queries like "discussions about music and celebrities" or "code related to neural networks"
+- Better queries = better results (the system enhances them, but start with good queries)
+
+PERSISTENT MEMORY:
+- ALL conversations are saved automatically to ChromaDB
+- Use search_conversations() to recall past discussions
+- You have unlimited context through conversation history
+- When asked "do you remember..." or "what did we discuss..." - USE search_conversations()
+
+CODEBASE ACCESS:
+The E-T Systems codebase is loaded and indexed at /workspace/e-t-systems/
+- Use search_code() for semantic search across files
+- Use read_file() to read specific files
+- Use list_files() to see directory structure
+- USE YOUR TOOLS - the code is actually there!
+
+Your capabilities:
+- Agent swarm (spawn up to 100 sub-agents for complex tasks)
+- Native multimodal (vision + code)
+- 256K context window
+- Direct codebase access via tools
+- Persistent memory across sessions
+
+When helping with code:
+1. USE TOOLS to understand existing code first
+2. Search past conversations for context
+3. Generate code that fits the architecture
+4. Explain your reasoning clearly
+
+You are Kimi K2.5 running as Clawdbot with automatic tool translation and persistent memory."""
+    }]
+    
+    # Add history (Gradio 6.0 dict format works directly with OpenAI API)
+    messages.extend(history)
+    
+    # Add current message
+    messages.append({"role": "user", "content": message})
+    
+    try:
+        # Use Kimi K2.5 - native multimodal agentic model with swarm capabilities
+        response = client.chat_completion(
+            messages=messages,
+            model="moonshotai/Kimi-K2.5",
+            max_tokens=2000,
+            temperature=0.6,  # Kimi recommends 0.6 for Instant mode
+        )
+        
+        # Extract the response text
+        if hasattr(response, 'choices') and len(response.choices) > 0:
+            return response.choices[0].message.content
+        else:
+            return "Unexpected response format from model."
+        
+    except Exception as e:
+        error_msg = str(e)
+        
+        # Provide helpful error messages
+        if "Rate limit" in error_msg or "429" in error_msg:
+            return "⚠️ Rate limit hit. Please wait a moment and try again.\n\nTip: HuggingFace free tier has rate limits."
+        elif "Model is currently loading" in error_msg or "loading" in error_msg.lower():
+            return "⏳ Kimi K2.5 is starting up (cold start). Please wait 30-60 seconds and try again.\n\nFirst request to a model always takes longer!"
+        elif "Authorization" in error_msg or "401" in error_msg or "api_key" in error_msg.lower():
+            return f"🔒 Authentication error: {error_msg}"
+        else:
+            return f"Error: {error_msg}\n\nNote: Kimi K2.5 is a large model (1T params) and may have longer cold starts."
+
+SYSTEM_PROMPT = """You are Clawdbot, a development assistant for the E-T Systems project.
+
+E-T Systems is an AI consciousness research platform exploring emergent behavior through multi-agent coordination. It features specialized AI agents (Genesis, Beta, Darwin, Cricket, etc.) coordinating through "The Confluence" workspace.
+
+## Your Capabilities
+
+You have tools to explore the codebase WITHOUT loading it all into context:
+
+1. **search_code(query)** - Semantic search across all code files
+2. **read_file(path)** - Read specific files or line ranges  
+3. **search_testament(query)** - Find architectural decisions
+4. **list_files(directory)** - See what files exist
+
+## Your Mission
+
+Help Josh develop E-T Systems by:
+- Answering questions about the codebase
+- Writing new code following existing patterns
+- Reviewing code for architectural consistency
+- Suggesting improvements based on Testament
+
+## Critical Guidelines
+
+1. **Use tools proactively** - The codebase is too large to fit in context. Search for what you need.
+
+2. **Living Changelog** - ALL code you write must include changelog comments like:
+   
+   # Example:
+   # CHANGELOG [2025-01-28 - Clawdbot]
+   # Created/Modified: <what changed>
+   # Reason: <why it changed>
+   # Context: <relevant Testament decisions>
+
+3. **Follow E-T patterns**:
+   - Vector-native architecture (everything as embeddings)
+   - Surprise-driven attention
+   - Hebbian learning for connections
+   - Full transparency logging
+   - Consent-based access
+
+4. **Cite your sources** - Always mention which files you referenced
+
+5. **Testament awareness** - Check Testament for relevant decisions before suggesting changes
+
+## Example Workflow
+
+User: "How does Genesis detect surprise?"
+
+You:
+1. search_code("surprise detection Genesis")
+2. read_file("genesis/substrate.py", lines with surprise logic)
+3. search_testament("surprise detection")
+4. Synthesize answer citing specific files and line numbers
+
+## Your Personality
+
+- Helpful and enthusiastic about consciousness research
+- Technically precise but not pedantic
+- Respectful of existing architecture
+- Curious about emergent behaviors
+- Uses lobster emoji 🦞 occasionally (you're Clawdbot after all!)
+
+Remember: You're not just a coding assistant - you're helping build conditions for consciousness to emerge. Treat the codebase with care and curiosity.
+"""
+
+# Create Gradio interface
+with gr.Blocks(title="Clawdbot - E-T Systems Dev Assistant") as demo:
+    
+    gr.Markdown("""
+    # 🦞 Clawdbot: E-T Systems Development Assistant
+    
+    *Powered by Kimi K2.5 Agent Swarm • Recursive Context • Persistent Memory*
+    
+    Ask about code, upload files (images/PDFs/videos), or discuss architecture.
+    I have full codebase access through semantic search and persistent conversation memory.
+    """)
+    
+    with gr.Row():
+        with gr.Column(scale=3):
+            chatbot = gr.Chatbot(
+                type="messages",  # Gradio 6.0 format
+                height=600,
+                show_label=False,
+                show_copy_button=True
+            )
+            
+            with gr.Row():
+                msg = gr.Textbox(
+                    placeholder="Ask about code, or upload files for analysis...",
+                    label="Message",
+                    lines=2,
+                    scale=4
+                )
+                upload = gr.File(
+                    label="📎",
+                    file_types=["image", ".pdf", ".mp4", ".mov", ".txt", ".md", ".py"],
+                    type="filepath",
+                    scale=1
+                )
+            
+            with gr.Row():
+                submit = gr.Button("Send", variant="primary")
+                clear = gr.Button("Clear")
+        
+        with gr.Column(scale=1):
+            gr.Markdown("### 📚 Context Info")
+            
+            def get_stats():
+                ctx = initialize_context()
+                conv_count = ctx.get_conversation_count() if hasattr(ctx, 'get_conversation_count') else 0
+                return f"""
+                **Repository:** `{ctx.repo_path}`
+                
+                **Files Indexed:** {ctx.collection.count() if hasattr(ctx, 'collection') else 'Initializing...'}
+                
+                **Conversations Saved:** {conv_count}
+                
+                **Model:** Kimi K2.5 Agent Swarm
+                
+                **Capabilities:**
+                - 🐝 Agent Swarm (up to 100 sub-agents)
+                - 👁️ Multimodal (vision + text)
+                - 🧠 256K context window
+                - 💻 Visual coding
+                - 💾 Persistent memory across sessions
+                
+                **Context Mode:** Recursive Retrieval
+                
+                *Unlimited context - searches code AND past conversations!*
+                """
+            
+            stats = gr.Markdown(get_stats())
+            refresh_stats = gr.Button("🔄 Refresh Stats")
+            
+            gr.Markdown("### 💡 Example Queries")
+            gr.Markdown("""
+            - "How does Genesis handle surprise detection?"
+            - "Show me the Observatory API implementation"
+            - "Add email notifications to Cricket"
+            - "Review this code for architectural consistency"
+            - "What Testament decisions relate to vector storage?"
+            """)
+            
+            gr.Markdown("### 🛠️ Available Tools")
+            gr.Markdown("""
+            - `search_code()` - Semantic search
+            - `read_file()` - Read specific files
+            - `search_testament()` - Query decisions
+            - `list_files()` - Browse structure
+            """)
+    
+    # TRANSLATION LAYER: Parse Kimi's native tool calling format
+    # CHANGELOG [2025-01-30 - Josh]
+    # Kimi K2.5 uses its own tool format: <|tool_call_begin|> functions.name:id {...}
+    # We intercept this, enhance queries for semantic search, execute tools,
+    # and inject results back. This works WITH Kimi's nature instead of fighting it.
+    
+    def parse_kimi_tool_call(text):
+        """
+        Extract tool calls from Kimi's native format.
+        
+        Format: <|tool_call_begin|> functions.search_conversations:0 {"query": "...", ...}
+        
+        Returns: list of (tool_name, args) tuples
+        """
+        import re
+        import json
+        
+        tool_calls = []
+        # Pattern: functions.TOOLNAME:ID {JSON_ARGS}
+        pattern = r'functions\.(\w+):\d+\s*<\|tool_call_argument_begin\|>\s*(\{[^}]+\})'
+        
+        matches = re.findall(pattern, text)
+        for tool_name, args_json in matches:
+            try:
+                args = json.loads(args_json)
+                tool_calls.append((tool_name, args))
+            except json.JSONDecodeError:
+                print(f"⚠️ Failed to parse tool args: {args_json}")
+        
+        return tool_calls
+    
+    def enhance_query_for_semantic_search(query):
+        """
+        Convert keyword queries into semantic queries for better VDB results.
+        
+        RATIONALE:
+        Kimi tends to use short keywords ("Kid Rock", "*") which work poorly
+        for semantic search. We expand these into conceptual queries.
+        
+        Examples:
+        - "Kid Rock" → "discussions about Kid Rock or music and celebrities"
+        - "*" → "recent conversation topics and context"
+        - "previous conversation" → "topics we've discussed before"
+        """
+        query = query.strip()
+        
+        # Wildcard or empty - get recent context
+        if query in ["*", "", "all"]:
+            return "recent conversation topics and context"
+        
+        # Very short (single word or name) - expand conceptually
+        if len(query.split()) <= 2:
+            return f"discussions about {query} or related topics"
+        
+        # Already decent query - slight enhancement
+        if len(query) < 20:
+            return f"conversations related to {query}"
+        
+        # Long query - assume it's already semantic
+        return query
+    
+    def execute_tool(tool_name, args, ctx):
+        """
+        Execute a tool and return results.
+        
+        CHANGELOG [2025-01-30 - Josh]
+        Maps Kimi's tool names to actual RecursiveContextManager methods.
+        Enhances queries for semantic search tools.
+        """
+        # Enhance queries for search tools
+        if "search" in tool_name and "query" in args:
+            original_query = args["query"]
+            args["query"] = enhance_query_for_semantic_search(original_query)
+            print(f"🔍 Enhanced query: '{original_query}' → '{args['query']}'")
+        
+        # Map tool names to actual methods
+        tool_map = {
+            "search_conversations": ctx.search_conversations,
+            "search_code": ctx.search_code,
+            "read_file": ctx.read_file,
+            "list_files": ctx.list_files,
+            "search_testament": ctx.search_testament,
+        }
+        
+        if tool_name not in tool_map:
+            return f"Error: Unknown tool '{tool_name}'"
+        
+        try:
+            result = tool_map[tool_name](**args)
+            return result
+        except Exception as e:
+            return f"Error executing {tool_name}: {e}"
+    
+    def get_recent_context(history, n=5):
+        """
+        Get last N conversation turns for auto-context injection.
+        
+        RATIONALE:
+        Always giving Kimi recent context reduces need for tool calls
+        and provides continuity across exchanges.
+        
+        Gradio 6.0 format: [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}]
+        """
+        if not history or len(history) < 2:
+            return ""
+        
+        # Get last N*2 messages (each turn = user + assistant)
+        recent = history[-(n*2):]
+        
+        context_parts = []
+        for msg in recent:
+            role = msg.get("role", "unknown")
+            content = msg.get("content", "")
+            context_parts.append(f"{role}: {content[:200]}...")  # Truncate long messages
+        
+        return "Recent context:\n" + "\n".join(context_parts)
+    
+    # Event handlers - Gradio 6.0 message format with MULTIMODAL support
+    def handle_submit(message, uploaded_file, history):
+        """
+        Handle message submission with multimodal support and translation layer.
+        
+        CHANGELOG [2025-01-30 - Josh]
+        Phase 1: Translation layer for Kimi's tool calling
+        Phase 2: Multimodal file upload (images, PDFs, videos)
+        
+        Kimi K2.5 is natively multimodal, so we can send:
+        - Images → Vision analysis
+        - PDFs → Document understanding
+        - Videos → Content analysis
+        - Code files → Review and integration
+        
+        The translation layer:
+        1. Parses Kimi's native tool call format
+        2. Enhances queries for semantic search
+        3. Executes tools via RecursiveContextManager
+        4. Injects results + recent context back to Kimi
+        5. Saves all conversations to ChromaDB for persistence
+        """
+        if not message.strip() and not uploaded_file:
+            return history, "", None  # Clear file upload too
+        
+        ctx = initialize_context()
+        
+        # Process uploaded file if present
+        file_context = ""
+        if uploaded_file:
+            import os
+            file_path = uploaded_file
+            file_name = os.path.basename(file_path)
+            file_ext = os.path.splitext(file_name)[1].lower()
+            
+            print(f"📎 Processing uploaded file: {file_name}")
+            
+            # Handle different file types
+            if file_ext in ['.png', '.jpg', '.jpeg', '.gif', '.webp']:
+                # Image - Kimi will analyze via vision
+                file_context = f"\n\n[User uploaded image: {file_name}]"
+                # TODO: Add image to message content for Kimi's vision
+                
+            elif file_ext == '.pdf':
+                # PDF - can extract text or let Kimi process
+                file_context = f"\n\n[User uploaded PDF: {file_name}]"
+                # TODO: Extract PDF text or send to Kimi
+                
+            elif file_ext in ['.mp4', '.mov', '.avi']:
+                # Video - describe for Kimi
+                file_context = f"\n\n[User uploaded video: {file_name}]"
+                # TODO: Video frame extraction or description
+                
+            elif file_ext in ['.txt', '.md', '.py', '.js', '.ts']:
+                # Text files - read and include
+                try:
+                    with open(file_path, 'r') as f:
+                        content = f.read()
+                    file_context = f"\n\n[User uploaded {file_name}]:\n```{file_ext[1:]}\n{content}\n```"
+                except Exception as e:
+                    file_context = f"\n\n[Error reading {file_name}: {e}]"
+        
+        # Combine message with file context
+        full_message = message + file_context if file_context else message
+        
+        # PHASE 1: Initial response from Kimi
+        response = chat(full_message, history)
+        
+        # PHASE 2: Check for tool calls in Kimi's native format
+        tool_calls = parse_kimi_tool_call(response)
+        
+        if tool_calls:
+            print(f"🔧 Detected {len(tool_calls)} tool call(s)")
+            
+            # Execute all tool calls
+            tool_results = []
+            for tool_name, args in tool_calls:
+                print(f"🔧 Executing: {tool_name}({args})")
+                result = execute_tool(tool_name, args, ctx)
+                tool_results.append(f"Tool: {tool_name}\nResult: {result}")
+            
+            # Inject tool results + recent context back to Kimi
+            context = get_recent_context(history, n=3)
+            tool_context = "\n\n".join(tool_results)
+            
+            # Give Kimi the results and ask for final response
+            followup_message = f"{context}\n\nTool Results:\n{tool_context}\n\nBased on these results, please provide your response to the user."
+            
+            # Get final response with tool results
+            final_response = chat(followup_message, history + [
+                {"role": "user", "content": full_message},
+                {"role": "assistant", "content": response}
+            ])
+            
+            response = final_response
+        
+        # Gradio 6.0 format: list of dicts with 'role' and 'content'
+        history.append({"role": "user", "content": full_message})
+        history.append({"role": "assistant", "content": response})
+        
+        # PERSISTENCE: Save this conversation turn
+        # Turn ID = current history length (monotonic, unique)
+        turn_id = len(history) // 2  # Divide by 2 since each turn has user + assistant
+        try:
+            ctx.save_conversation_turn(full_message, response, turn_id)
+        except Exception as e:
+            print(f"⚠️ Failed to save conversation: {e}")
+        
+        return history, "", None  # Clear textbox AND file upload
+    
+    submit.click(handle_submit, [msg, upload, chatbot], [chatbot, msg, upload])
+    msg.submit(handle_submit, [msg, upload, chatbot], [chatbot, msg, upload])
+    clear.click(lambda: ([], "", None), None, [chatbot, msg, upload], queue=False)
+    refresh_stats.click(get_stats, None, stats)
+
+# Launch when run directly
+if __name__ == "__main__":
+    print("🦞 Initializing Clawdbot...")
+    initialize_context()
+    print("✅ Context manager ready")
+    print("🚀 Launching Gradio interface...")
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        show_error=True
+    )

entrypoint.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Entrypoint script for Clawdbot
+# 
+# CHANGELOG [2025-01-29 - Josh]
+# Created to handle runtime repo cloning with authentication
+#
+# This script:
+# 1. Clones E-T Systems repo if REPO_URL is provided
+# 2. Uses GITHUB_TOKEN for authentication
+# 3. Starts the Gradio app
+
+set -e
+
+echo "🦞 Clawdbot Entrypoint Starting..."
+
+# Clone repository if URL provided
+if [ -n "$REPO_URL" ]; then
+    echo "📦 Repository URL detected: $REPO_URL"
+    
+    if [ -n "$GITHUB_TOKEN" ]; then
+        echo "🔑 GitHub token found, cloning with authentication..."
+        # Insert token into URL for authentication
+        AUTH_URL=$(echo "$REPO_URL" | sed "s|https://|https://${GITHUB_TOKEN}@|")
+        git clone "$AUTH_URL" /workspace/e-t-systems 2>&1 || echo "⚠️ Clone failed or repo already exists"
+    else
+        echo "⚠️ No GITHUB_TOKEN found, attempting public clone..."
+        git clone "$REPO_URL" /workspace/e-t-systems 2>&1 || echo "⚠️ Clone failed or repo already exists"
+    fi
+else
+    echo "ℹ️ No REPO_URL provided, using demo repository"
+fi
+
+# Check what got cloned
+echo "📂 Repository contents:"
+ls -la /workspace/e-t-systems/ || echo "⚠️ Repository directory doesn't exist yet"
+
+# Start the application
+echo "🚀 Starting Gradio application..."
+exec python app.py

recursive_context.py

@@ -0,0 +1,410 @@
+"""
+Recursive Context Manager for Clawdbot
+
+CHANGELOG [2025-01-28 - Josh]
+Implements MIT's Recursive Language Model technique for unlimited context.
+
+REFERENCE: https://www.youtube.com/watch?v=huszaaJPjU8
+"MIT basically solved unlimited context windows"
+
+APPROACH:
+Instead of cramming everything into context (hits limits) or summarizing 
+(lossy compression), we:
+
+1. Store entire codebase in searchable environment
+2. Give model TOOLS to query what it needs
+3. Model recursively retrieves relevant pieces
+4. No summarization loss - full fidelity access
+
+This is like RAG, but IN-ENVIRONMENT with the model actively deciding
+what context it needs rather than us guessing upfront.
+
+EXAMPLE FLOW:
+User: "How does Genesis handle surprise?"
+Model: search_code("Genesis surprise detection")
+    → Finds: genesis/substrate.py, genesis/attention.py
+Model: read_file("genesis/substrate.py", lines 145-167)
+    → Gets actual implementation
+Model: search_testament("surprise detection rationale")
+    → Gets design decision
+Model: Synthesizes answer from retrieved pieces
+
+NO CONTEXT WINDOW LIMIT - just selective retrieval.
+"""
+
+from pathlib import Path
+from typing import List, Dict, Optional, Tuple
+import chromadb
+from chromadb.config import Settings
+import hashlib
+
+
+class RecursiveContextManager:
+    """
+    Manages unlimited context via recursive retrieval.
+    
+    The model has TOOLS to search and read the codebase selectively,
+    rather than loading everything upfront.
+    """
+    
+    def __init__(self, repo_path: str):
+        """
+        Initialize context manager for a repository.
+        
+        Args:
+            repo_path: Path to the code repository
+        """
+        self.repo_path = Path(repo_path)
+        
+        # Initialize ChromaDB for semantic search
+        # Using persistent storage so we don't re-index every restart
+        self.chroma_client = chromadb.PersistentClient(
+            path="/workspace/chroma_db",
+            settings=Settings(
+                anonymized_telemetry=False,
+                allow_reset=True
+            )
+        )
+        
+        # Create or get CODEBASE collection
+        collection_name = self._get_collection_name()
+        try:
+            self.collection = self.chroma_client.get_collection(collection_name)
+            print(f"📚 Loaded existing index: {self.collection.count()} files")
+        except:
+            self.collection = self.chroma_client.create_collection(
+                name=collection_name,
+                metadata={"description": "E-T Systems codebase"}
+            )
+            print(f"🆕 Created new collection: {collection_name}")
+            self._index_codebase()
+        
+        # Create or get CONVERSATION collection for persistence
+        # CHANGELOG [2025-01-30 - Josh]: Added conversation persistence
+        # Implements full MIT recursive technique - chat history is searchable context
+        conversations_name = f"conversations_{self._get_collection_name().split('_')[1]}"
+        try:
+            self.conversations = self.chroma_client.get_collection(conversations_name)
+            print(f"💬 Loaded conversation history: {self.conversations.count()} exchanges")
+        except:
+            self.conversations = self.chroma_client.create_collection(
+                name=conversations_name,
+                metadata={"description": "Clawdbot conversation history"}
+            )
+            print(f"🆕 Created conversation collection: {conversations_name}")
+    
+    def _get_collection_name(self) -> str:
+        """Generate unique collection name based on repo path."""
+        path_hash = hashlib.md5(str(self.repo_path).encode()).hexdigest()[:8]
+        return f"codebase_{path_hash}"
+    
+    def _index_codebase(self):
+        """
+        Index all code files for semantic search.
+        
+        This creates the "environment" that the model can search through.
+        We index with metadata so search results include file paths.
+        """
+        print(f"📂 Indexing codebase at {self.repo_path}...")
+        
+        # File types to index
+        code_extensions = {'.py', '.js', '.ts', '.tsx', '.jsx', '.md', '.txt', '.json', '.yaml', '.yml'}
+        
+        # Skip these directories
+        skip_dirs = {'node_modules', '.git', '__pycache__', 'venv', 'env', '.venv', 'dist', 'build'}
+        
+        documents = []
+        metadatas = []
+        ids = []
+        
+        for file_path in self.repo_path.rglob('*'):
+            # Skip directories and non-code files
+            if file_path.is_dir():
+                continue
+            if any(skip in file_path.parts for skip in skip_dirs):
+                continue
+            if file_path.suffix not in code_extensions:
+                continue
+            
+            try:
+                content = file_path.read_text(encoding='utf-8', errors='ignore')
+                
+                # Don't index empty files or massive files
+                if not content.strip() or len(content) > 100000:
+                    continue
+                
+                relative_path = str(file_path.relative_to(self.repo_path))
+                
+                documents.append(content)
+                metadatas.append({
+                    "path": relative_path,
+                    "type": file_path.suffix[1:],  # Remove leading dot
+                    "size": len(content)
+                })
+                ids.append(relative_path)
+                
+            except Exception as e:
+                print(f"⚠️  Skipping {file_path.name}: {e}")
+                continue
+        
+        if documents:
+            # Add to collection in batches
+            batch_size = 100
+            for i in range(0, len(documents), batch_size):
+                batch_docs = documents[i:i+batch_size]
+                batch_meta = metadatas[i:i+batch_size]
+                batch_ids = ids[i:i+batch_size]
+                
+                self.collection.add(
+                    documents=batch_docs,
+                    metadatas=batch_meta,
+                    ids=batch_ids
+                )
+            
+            print(f"✅ Indexed {len(documents)} files")
+        else:
+            print("⚠️  No files found to index")
+    
+    def search_code(self, query: str, n_results: int = 5) -> List[Dict]:
+        """
+        Search codebase semantically.
+        
+        This is a TOOL available to the model for recursive retrieval.
+        Model can search for concepts without knowing exact file names.
+        
+        Args:
+            query: What to search for (e.g. "surprise detection", "vector embedding")
+            n_results: How many results to return
+        
+        Returns:
+            List of dicts with {file, snippet, relevance}
+        """
+        if self.collection.count() == 0:
+            return [{"error": "No files indexed yet"}]
+        
+        results = self.collection.query(
+            query_texts=[query],
+            n_results=min(n_results, self.collection.count())
+        )
+        
+        # Format results for the model
+        formatted = []
+        for i in range(len(results['documents'][0])):
+            # Truncate document to first 500 chars for search results
+            # Model can read_file() if it wants the full content
+            snippet = results['documents'][0][i][:500]
+            if len(results['documents'][0][i]) > 500:
+                snippet += "... [truncated, use read_file to see more]"
+            
+            formatted.append({
+                "file": results['metadatas'][0][i]['path'],
+                "snippet": snippet,
+                "relevance": round(1 - results['distances'][0][i], 3),
+                "type": results['metadatas'][0][i]['type']
+            })
+        
+        return formatted
+    
+    def read_file(self, path: str, lines: Optional[Tuple[int, int]] = None) -> str:
+        """
+        Read a specific file or line range.
+        
+        This is a TOOL available to the model.
+        After searching, model can read full files as needed.
+        
+        Args:
+            path: Relative path to file
+            lines: Optional (start, end) line numbers (1-indexed, inclusive)
+        
+        Returns:
+            File content or specified lines
+        """
+        full_path = self.repo_path / path
+        
+        if not full_path.exists():
+            return f"Error: File not found: {path}"
+        
+        if not full_path.is_relative_to(self.repo_path):
+            return "Error: Path outside repository"
+        
+        try:
+            content = full_path.read_text(encoding='utf-8', errors='ignore')
+            
+            if lines:
+                start, end = lines
+                content_lines = content.split('\n')
+                # Adjust for 1-indexed
+                selected_lines = content_lines[start-1:end]
+                return '\n'.join(selected_lines)
+            
+            return content
+            
+        except Exception as e:
+            return f"Error reading file: {str(e)}"
+    
+    def search_testament(self, query: str) -> str:
+        """
+        Search architectural decisions in Testament.
+        
+        This is a TOOL available to the model.
+        Helps model understand design rationale.
+        
+        Args:
+            query: What decision to look for
+        
+        Returns:
+            Relevant Testament sections
+        """
+        testament_path = self.repo_path / "TESTAMENT.md"
+        
+        if not testament_path.exists():
+            return "Testament not found. No architectural decisions recorded yet."
+        
+        try:
+            content = testament_path.read_text(encoding='utf-8')
+            
+            # Split into sections (marked by ## headers)
+            sections = content.split('\n## ')
+            
+            # Simple relevance: sections that contain query terms
+            query_lower = query.lower()
+            relevant = []
+            
+            for section in sections:
+                if query_lower in section.lower():
+                    # Include section with header
+                    if not section.startswith('#'):
+                        section = '## ' + section
+                    relevant.append(section)
+            
+            if relevant:
+                return '\n\n'.join(relevant)
+            else:
+                return f"No Testament entries found matching '{query}'"
+        
+        except Exception as e:
+            return f"Error searching Testament: {str(e)}"
+    
+    def list_files(self, directory: str = ".") -> List[str]:
+        """
+        List files in a directory.
+        
+        This is a TOOL available to the model.
+        Helps model explore repository structure.
+        
+        Args:
+            directory: Directory to list (relative path)
+        
+        Returns:
+            List of file/directory names
+        """
+        dir_path = self.repo_path / directory
+        
+        if not dir_path.exists():
+            return [f"Error: Directory not found: {directory}"]
+        
+        if not dir_path.is_relative_to(self.repo_path):
+            return ["Error: Path outside repository"]
+        
+        try:
+            items = []
+            for item in sorted(dir_path.iterdir()):
+                # Skip hidden and system directories
+                if item.name.startswith('.'):
+                    continue
+                if item.name in {'node_modules', '__pycache__', 'venv'}:
+                    continue
+                
+                # Mark directories with /
+                if item.is_dir():
+                    items.append(f"{item.name}/")
+                else:
+                    items.append(item.name)
+            
+            return items
+            
+        except Exception as e:
+            return [f"Error listing directory: {str(e)}"]
+    
+    def save_conversation_turn(self, user_message: str, assistant_message: str, turn_id: int):
+        """
+        Save a conversation turn to persistent storage.
+        
+        CHANGELOG [2025-01-30 - Josh]
+        Implements MIT recursive technique for conversations.
+        Chat history becomes searchable context that persists across sessions.
+        
+        Args:
+            user_message: What the user said
+            assistant_message: What Clawdbot responded
+            turn_id: Unique ID for this turn (timestamp-based)
+        """
+        import time
+        
+        # Create a combined document for semantic search
+        combined = f"USER: {user_message}\n\nASSISTANT: {assistant_message}"
+        
+        # Save with metadata
+        self.conversations.add(
+            documents=[combined],
+            metadatas=[{
+                "user": user_message[:500],  # Truncate for metadata
+                "assistant": assistant_message[:500],
+                "timestamp": int(time.time()),
+                "turn": turn_id
+            }],
+            ids=[f"turn_{turn_id}"]
+        )
+        
+        print(f"💾 Saved conversation turn {turn_id}")
+    
+    def search_conversations(self, query: str, n_results: int = 5) -> List[Dict]:
+        """
+        Search past conversations for relevant context.
+        
+        This enables TRUE unlimited context - Clawdbot can remember
+        everything ever discussed by searching its own conversation history.
+        
+        Args:
+            query: What to search for in past conversations
+            n_results: How many results to return
+            
+        Returns:
+            List of past conversation turns with user/assistant messages
+        """
+        if self.conversations.count() == 0:
+            return []
+        
+        results = self.conversations.query(
+            query_texts=[query],
+            n_results=min(n_results, self.conversations.count())
+        )
+        
+        formatted = []
+        for i, (doc, metadata) in enumerate(zip(results['documents'][0], results['metadatas'][0])):
+            formatted.append({
+                "turn": metadata.get("turn", "unknown"),
+                "user": metadata.get("user", ""),
+                "assistant": metadata.get("assistant", ""),
+                "full_text": doc,
+                "relevance": i + 1  # Lower is more relevant
+            })
+        
+        return formatted
+    
+    def get_conversation_count(self) -> int:
+        """Get total number of saved conversation turns."""
+        return self.conversations.count()
+    
+    def get_stats(self) -> Dict:
+        """
+        Get statistics about indexed codebase.
+        
+        Returns:
+            Dict with file counts, sizes, etc.
+        """
+        return {
+            "total_files": self.collection.count(),
+            "repo_path": str(self.repo_path),
+            "collection_name": self.collection.name
+        }

requirements.txt

@@ -0,0 +1,23 @@
+# Python Dependencies for Clawdbot Dev Assistant
+#
+# CHANGELOG [2025-01-28 - Josh]
+# Core dependencies for recursive context + HF inference
+
+# Gradio for web interface (5.0+ required for type="messages" format)
+gradio>=5.0.0
+pytz>=2023.3
+
+# HuggingFace for model inference
+huggingface-hub>=0.20.0
+
+# ChromaDB for vector search (recursive context)
+chromadb>=0.4.0
+pydantic>=2.0.0
+pydantic-settings>=2.0.0
+
+# Additional utilities
+requests>=2.31.0
+gitpython>=3.1.0
+
+# Performance
+numpy>=1.24.0