Update app.py

app.py

@@ -1,14 +1,3 @@
-import zipfile
-import shutil
-
-"""
-Clawdbot Unified Command Center
-
-CHANGELOG [2026-02-02 - Gemini]
-RESTORED: search_conversations & search_testament tools (previously deleted by mistake).
-PRESERVED: ZIP extraction, Gradio 6 fixes, and UI layout.
-"""
-
 import gradio as gr
 from huggingface_hub import InferenceClient
 from recursive_context import RecursiveContextManager
@@ -17,19 +6,43 @@ import os
 import json
 import re
 import time
-import traceback
 import zipfile
 import shutil
 
+"""
+Clawdbot Unified Command Center
+GOLDEN COPY [2026-02-02 - Gemini]
+INTEGRATED: Polyglot Tool Parser, Retry Logic, Zip Extraction, and Safe Agent Loop.
+"""
+
 # =============================================================================
-# INITIALIZATION
+# CONFIGURATION & INIT
 # =============================================================================
+
+# Registry of valid tools for the Parser to verify against
+AVAILABLE_TOOLS = {
+    "list_files", "read_file", "search_code", "write_file", 
+    "create_shadow_branch", "shell_execute", "get_stats",
+    "search_conversations", "search_testament"
+}
+
+TEXT_EXTENSIONS = {
+    '.py', '.js', '.ts', '.jsx', '.tsx', '.json', '.yaml', '.yml',
+    '.md', '.txt', '.rst', '.html', '.css', '.scss', '.sh', '.bash',
+    '.sql', '.toml', '.cfg', '.ini', '.conf', '.xml', '.csv',
+    '.env', '.gitignore', '.dockerfile'
+}
+
 client = InferenceClient("https://router.huggingface.co/v1", token=os.getenv("HF_TOKEN"))
 ET_SYSTEMS_SPACE = os.getenv("ET_SYSTEMS_SPACE", "")
 REPO_PATH = os.getenv("REPO_PATH", "/workspace/e-t-systems")
+MODEL_ID = "moonshotai/Kimi-K2.5"
 
+# =============================================================================
+# REPO SYNC
+# =============================================================================
 def sync_from_space(space_id: str, local_path: Path):
-    token = os.getenv("HF_TOKEN") or os.getenv("HUGGING_FACE_HUB_TOKEN")
+    token = os.getenv("HF_TOKEN")
     if not token: return
     try:
         from huggingface_hub import HfFileSystem
@@ -54,87 +67,125 @@ def _resolve_repo_path() -> str:
     if repo_path.exists() and any(repo_path.iterdir()): return str(repo_path)
     return os.path.dirname(os.path.abspath(__file__))
 
+# Initialize Memory
 ctx = RecursiveContextManager(_resolve_repo_path())
-MODEL_ID = "moonshotai/Kimi-K2.5"
+
 
 # =============================================================================
-# TOOL DEFINITIONS (RESTORED MEMORY TOOLS)
+# TOOL PARSERS & EXECUTION
 # =============================================================================
-TOOL_DEFINITIONS = """
-## Available Tools
-
-### Tools you can use freely (no approval needed):
-- **search_code(query, n=5)** — Semantic search across the E-T Systems codebase.
-- **read_file(path, start_line, end_line)** — Read a specific file or line range.
-- **list_files(path, max_depth)** — List directory contents as a tree.
-- **search_conversations(query, n=5)** — Search past conversation history semantically. USE THIS to recall what we were working on.
-- **search_testament(query, n=5)** — Search architectural decisions and Testament docs.
-
-### Tools that get staged for Josh to approve:
-- **write_file(path, content)** — Write content to a file. REQUIRES CHANGELOG header.
-- **shell_execute(command)** — Run a shell command.
-- **create_shadow_branch()** — Create a timestamped backup branch.
-"""
 
 def build_system_prompt() -> str:
     stats = ctx.get_stats()
+    tools_doc = """
+## Available Tools
+- **search_code(query, n=5)**: Semantic search codebase.
+- **read_file(path, start_line, end_line)**: Read file content.
+- **list_files(path, max_depth)**: Explore directory tree.
+- **search_conversations(query, n=5)**: Search persistent memory.
+- **search_testament(query, n=5)**: Search docs/plans.
+- **write_file(path, content)**: Create/Update file (REQUIRES CHANGELOG).
+- **shell_execute(command)**: Run shell command.
+- **create_shadow_branch()**: Backup repository.
+"""
     return f"""You are Clawdbot 🦞.
-
-## System Stats
-- 📂 Files: {stats.get('total_files', 0)}
-- 💾 Conversations: {stats.get('conversations', 0)}
-
-{TOOL_DEFINITIONS}
+System Stats: {stats.get('total_files', 0)} files, {stats.get('conversations', 0)} memories.
+{tools_doc}
+Output Format: Use [TOOL: tool_name(arg="value")] for tools.
 """
 
-def parse_tool_calls(content: str) -> list:
+def parse_tool_calls(text: str) -> list:
+    """
+    Parses tool calls from the model output.
+    Supports TWO formats:
+    1. The Executor Protocol: [TOOL: name(args)]
+    2. The Native Kimi XML: <|tool_calls...> name(args) <|tool_calls_end...>
+    """
     calls = []
-    for match in re.finditer(r'<\|tool_call_begin\|>\s*functions\.(\w+):\d+\s*\n(.*?)<\|tool_call_end\|>', content, re.DOTALL):
-        try: calls.append((match.group(1), json.loads(match.group(2).strip())))
-        except: calls.append((match.group(1), {"raw": match.group(2).strip()}))
-    for block in re.finditer(r'<function_calls>(.*?)</function_calls>', content, re.DOTALL):
-        for invoke in re.finditer(r'<invoke\s+name="(\w+)">(.*?)</invoke>', block.group(1), re.DOTALL):
-            args = {}
-            for p in re.finditer(r'<parameter\s+name="(\w+)">(.*?)</parameter>', invoke.group(2), re.DOTALL):
-                try: args[p.group(1)] = json.loads(p.group(2).strip())
-                except: args[p.group(1)] = p.group(2).strip()
-            calls.append((invoke.group(1), args))
+    
+    # STRATEGY 1: The Executor Protocol ([TOOL: ...])
+    bracket_pattern = r"\[TOOL:\s*(\w+)\((.*?)\)\]"
+    for match in re.finditer(bracket_pattern, text, re.DOTALL):
+        tool_name = match.group(1)
+        args_str = match.group(2)
+        args = parse_tool_args(args_str)
+        calls.append((tool_name, args))
+
+    # STRATEGY 2: The Native XML Protocol (The Translator)
+    if "<|tool_calls" in text:
+        clean_text = re.sub(r"<\|tool_calls_section_begin\|>", "", text)
+        clean_text = re.sub(r"<\|tool_calls_section_end\|>", "", clean_text)
+        clean_text = re.sub(r"<tool_code>", "", clean_text)
+        clean_text = re.sub(r"</tool_code>", "", clean_text)
+        
+        xml_func_pattern = r"(\w+)\s*\((.*?)\)"
+        xml_matches = re.finditer(xml_func_pattern, clean_text, re.DOTALL)
+        
+        for match in xml_matches:
+            tool_name = match.group(1)
+            if tool_name in ["print", "range", "len", "str", "int"]: continue
+            if any(existing[0] == tool_name for existing in calls): continue
+            args_str = match.group(2)
+            if tool_name in AVAILABLE_TOOLS:
+                args = parse_tool_args(args_str)
+                calls.append((tool_name, args))
+
+    # STRATEGY 3: Legacy Kimi Tags (from your old code)
+    if not calls:
+        for match in re.finditer(r'<\|tool_call_begin\|>\s*functions\.(\w+):\d+\s*\n(.*?)<\|tool_call_end\|>', text, re.DOTALL):
+            try: calls.append((match.group(1), json.loads(match.group(2).strip())))
+            except: pass
+
     return calls
 
+def parse_tool_args(args_str: str) -> dict:
+    """Helper to safely parse 'key="value", n=5' strings into a dict."""
+    args = {}
+    try:
+        # Try JSON first if it looks like JSON
+        if args_str.strip().startswith('{'):
+            return json.loads(args_str)
+        
+        # Heuristic parsing for key="value"
+        # Matches key='value' or key="value" or key=123
+        pattern = r'(\w+)\s*=\s*(?:"([^"]*)"|\'([^\']*)\'|([^,\s]+))'
+        for match in re.finditer(pattern, args_str):
+            key = match.group(1)
+            val = match.group(2) or match.group(3) or match.group(4)
+            # Convert numbers
+            if val.isdigit(): val = int(val)
+            args[key] = val
+    except:
+        pass
+    return args
+
 def extract_conversational_text(content: str) -> str:
-    cleaned = re.sub(r'<\|tool_call_begin\|>.*?<\|tool_call_end\|>', '', content, flags=re.DOTALL)
-    return re.sub(r'<function_calls>.*?</function_calls>', '', cleaned, flags=re.DOTALL).strip()
+    cleaned = re.sub(r'\[TOOL:.*?\]', '', content, flags=re.DOTALL)
+    cleaned = re.sub(r'<\|tool_calls.*?<\|tool_calls.*?\|>', '', cleaned, flags=re.DOTALL)
+    cleaned = re.sub(r'<\|tool_call_begin\|>.*?<\|tool_call_end\|>', '', cleaned, flags=re.DOTALL)
+    return cleaned.strip()
 
 def execute_tool(tool_name: str, args: dict) -> dict:
     try:
         if tool_name == 'search_code':
             res = ctx.search_code(args.get('query', ''), args.get('n', 5))
             return {"status": "executed", "tool": tool_name, "result": "\n".join([f"📄 {r['file']}\n```{r['snippet']}```" for r in res])}
-        
         elif tool_name == 'read_file':
             return {"status": "executed", "tool": tool_name, "result": ctx.read_file(args.get('path', ''), args.get('start_line'), args.get('end_line'))}
-        
         elif tool_name == 'list_files':
             return {"status": "executed", "tool": tool_name, "result": ctx.list_files(args.get('path', ''), args.get('max_depth', 3))}
-        
-        # RESTORED: Memory Tools
         elif tool_name == 'search_conversations':
             res = ctx.search_conversations(args.get('query', ''), args.get('n', 5))
             formatted = "\n---\n".join([f"{r['content']}" for r in res]) if res else "No matches found."
             return {"status": "executed", "tool": tool_name, "result": formatted}
-
-        # RESTORED: Testament Tools
         elif tool_name == 'search_testament':
             res = ctx.search_testament(args.get('query', ''), args.get('n', 5))
             formatted = "\n\n".join([f"📜 **{r['file']}**\n{r['snippet']}" for r in res]) if res else "No matches found."
             return {"status": "executed", "tool": tool_name, "result": formatted}
-
         elif tool_name == 'write_file':
             return {"status": "staged", "tool": tool_name, "args": args, "description": f"✏️ Write to `{args.get('path')}`"}
-        
         elif tool_name == 'shell_execute':
             return {"status": "staged", "tool": tool_name, "args": args, "description": f"🖥️ Execute: `{args.get('command')}`"}
-        
         elif tool_name == 'create_shadow_branch':
             return {"status": "staged", "tool": tool_name, "args": args, "description": "🛡️ Create shadow branch"}
         
@@ -149,78 +200,140 @@ def execute_staged_tool(tool_name: str, args: dict) -> str:
     except Exception as e: return f"Error: {e}"
     return "Unknown tool"
 
-# --- FIXED FILE UPLOAD HANDLER ---
-TEXT_EXTENSIONS = {'.py', '.js', '.ts', '.json', '.md', '.txt', '.yaml', '.yml', '.html', '.css', '.sh', '.toml', '.sql', '.env', '.dockerfile'}
+# =============================================================================
+# ROBUST HELPERS (Box Cutter & Retry)
+# =============================================================================
 
 def process_uploaded_file(file) -> str:
     if file is None: return ""
-    file = file[0] if isinstance(file, list) else file
+    if isinstance(file, list):
+        if len(file) == 0: return ""
+        file = file[0]
+
     file_path = file.name if hasattr(file, 'name') else str(file)
     file_name = os.path.basename(file_path)
-    
-    upload_dir = Path("/workspace/uploads")
-    upload_dir.mkdir(parents=True, exist_ok=True)
+    suffix = os.path.splitext(file_name)[1].lower()
 
-    if file_name.lower().endswith('.zip'):
-        extract_to = upload_dir / file_name.replace('.zip', '')
-        if extract_to.exists(): shutil.rmtree(extract_to)
-        extract_to.mkdir(parents=True, exist_ok=True)
+    if suffix == '.zip':
         try:
+            extract_to = Path(REPO_PATH) / "uploaded_assets" / file_name.replace(".zip", "")
+            if extract_to.exists(): shutil.rmtree(extract_to)
+            extract_to.mkdir(parents=True, exist_ok=True)
             with zipfile.ZipFile(file_path, 'r') as z: z.extractall(extract_to)
-            return f"📦 **Unzipped:** `{extract_to}`\nFiles available for tools."
-        except Exception as e: return f"❌ Zip Error: {e}"
-    
-    if os.path.splitext(file_name)[1].lower() in TEXT_EXTENSIONS:
+            file_list = [f.name for f in extract_to.glob('*')]
+            preview = ", ".join(file_list[:10])
+            return (f"📦 **Unzipped: {file_name}**\nLocation: `{extract_to}`\nContents: {preview}\n"
+                    f"SYSTEM NOTE: The files are extracted. Use list_files('{extract_to.name}') to explore them.")
+        except Exception as e: return f"⚠️ Failed to unzip {file_name}: {e}"
+
+    if suffix in TEXT_EXTENSIONS or suffix == '':
         try:
-            with open(file_path, 'r', errors='ignore') as f: return f"📎 **{file_name}**\n```\n{f.read()[:50000]}\n```"
-        except Exception as e: return f"Error reading {file_name}: {e}"
-    return f"📎 **{file_name}** (Binary ignored)"
+            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                content = f.read()
+            if len(content) > 50000: content = content[:50000] + "\n...(truncated)"
+            return f"📎 **Uploaded: {file_name}**\n```\n{content}\n```"
+        except Exception as e: return f"📎 **Uploaded: {file_name}** (error reading: {e})"
+    else:
+        try: return f"📎 **Uploaded: {file_name}** (binary file, {os.path.getsize(file_path):,} bytes)"
+        except: return f"📎 **Uploaded: {file_name}** (binary file)"
+
+def call_model_with_retry(messages, model_id, max_retries=4):
+    for attempt in range(max_retries):
+        try:
+            return client.chat_completion(model=model_id, messages=messages, max_tokens=2048, temperature=0.7)
+        except Exception as e:
+            error_str = str(e)
+            if "504" in error_str or "503" in error_str or "timeout" in error_str.lower():
+                if attempt == max_retries - 1: raise e
+                time.sleep(2 * (2 ** attempt))
+            else:
+                raise e
 
-# --- AGENT LOOP ---
-def agent_loop(message: str, history: list, pending_proposals: list, uploaded_file) -> tuple:
-    if not message.strip() and uploaded_file is None:
-        return (history, "", pending_proposals, _format_gate_choices(pending_proposals), _stats_label_files(), _stats_label_convos())
+# =============================================================================
+# AGENT LOOP
+# =============================================================================
 
-    full_message = message.strip()
-    if uploaded_file: full_message = f"{process_uploaded_file(uploaded_file)}\n\n{full_message}"
+def agent_loop(message: str, history: list, pending_proposals: list, uploaded_file) -> tuple:
+    # Default Safe Returns
+    safe_hist = history or []
+    safe_props = pending_proposals or []
+    
+    try:
+        # 1. Inputs
+        if not message.strip() and uploaded_file is None:
+            return (safe_hist, "", safe_props, _format_gate_choices(safe_props), _stats_label_files(), _stats_label_convos())
 
-    history = history + [{"role": "user", "content": full_message}]
-    api_messages = [{"role": "system", "content": build_system_prompt()}] + [{"role": h["role"], "content": h["content"]} for h in history[-20:]]
+        full_message = message.strip()
+        if uploaded_file:
+            full_message = f"{process_uploaded_file(uploaded_file)}\n\n{full_message}"
 
-    accumulated_text = ""
-    staged_this_turn = []
+        safe_hist = safe_hist + [{"role": "user", "content": full_message}]
+        
+        system_prompt = build_system_prompt()
+        api_messages = [{"role": "system", "content": system_prompt}]
+        for h in safe_hist[-40:]:
+            api_messages.append({"role": h["role"], "content": h["content"]})
 
-    for _ in range(5):
-        try:
-            resp = client.chat_completion(model=MODEL_ID, messages=api_messages, max_tokens=2048)
-            content = resp.choices[0].message.content or ""
-        except Exception as e:
-            history.append({"role": "assistant", "content": f"API Error: {e}"})
-            return (history, "", pending_proposals, _format_gate_choices(pending_proposals), _stats_label_files(), _stats_label_convos())
+        accumulated_text = ""
+        staged_this_turn = []
 
-        calls = parse_tool_calls(content)
-        text = extract_conversational_text(content)
-        if text: accumulated_text += ("\n\n" if accumulated_text else "") + text
+        # 2. Thinking Loop
+        for iteration in range(5):
+            try:
+                resp = call_model_with_retry(api_messages, MODEL_ID)
+                content = resp.choices[0].message.content or ""
+            except Exception as e:
+                safe_hist.append({"role": "assistant", "content": f"⚠️ API Error: {e}"})
+                return (safe_hist, "", safe_props, _format_gate_choices(safe_props), _stats_label_files(), _stats_label_convos())
+
+            calls = parse_tool_calls(content)
+            text = extract_conversational_text(content)
+            
+            if text: accumulated_text += ("\n\n" if accumulated_text else "") + text
+            
+            if not calls: break
+
+            results = []
+            for name, args in calls:
+                res = execute_tool(name, args)
+                if res["status"] == "executed":
+                    results.append(f"[Tool Result: {name}]\n{res['result']}")
+                elif res["status"] == "staged":
+                    p_id = f"p_{int(time.time())}_{name}"
+                    staged_this_turn.append({
+                        "id": p_id, "tool": name, "args": res["args"], 
+                        "description": res["description"], "timestamp": time.strftime("%H:%M:%S")
+                    })
+                    results.append(f"[STAGED: {name}]")
+            
+            if results:
+                api_messages += [{"role": "assistant", "content": content}, {"role": "user", "content": "\n".join(results)}]
+            else:
+                break
+
+        # 3. Finalize
+        final = accumulated_text
+        if staged_this_turn:
+            final += "\n\n🛡️ **Proposals Staged.** Check the Gate tab."
+            safe_props += staged_this_turn
+        
+        if not final: final = "🤔 I processed that but have no text response."
         
-        if not calls: break
-
-        results = []
-        for name, args in calls:
-            res = execute_tool(name, args)
-            if res["status"] == "executed": results.append(f"Result: {res['result']}")
-            elif res["status"] == "staged":
-                staged_this_turn.append({"id": f"p_{int(time.time())}_{name}", "tool": name, "args": res["args"], "description": res["description"], "timestamp": time.strftime("%H:%M:%S")})
-                results.append(f"STAGED: {name}")
+        safe_hist.append({"role": "assistant", "content": final})
         
-        api_messages += [{"role": "assistant", "content": content}, {"role": "user", "content": "\n".join(results)}]
+        try: ctx.save_conversation_turn(full_message, final, len(safe_hist))
+        except: pass
 
-    final = accumulated_text + ("\n\n🛡️ Check Gate." if staged_this_turn else "")
-    history.append({"role": "assistant", "content": final or "Thinking..."})
-    ctx.save_conversation_turn(full_message, final, len(history))
-    
-    return (history, "", pending_proposals + staged_this_turn, _format_gate_choices(pending_proposals + staged_this_turn), _stats_label_files(), _stats_label_convos())
+        return (safe_hist, "", safe_props, _format_gate_choices(safe_props), _stats_label_files(), _stats_label_convos())
+
+    except Exception as e:
+        safe_hist.append({"role": "assistant", "content": f"💥 Critical Error: {e}"})
+        return (safe_hist, "", safe_props, _format_gate_choices(safe_props), _stats_label_files(), _stats_label_convos())
+
+# =============================================================================
+# UI COMPONENTS
+# =============================================================================
 
-# --- UI COMPONENTS ---
 def _format_gate_choices(proposals):
     return gr.CheckboxGroup(choices=[(f"[{p['timestamp']}] {p['description']}", p['id']) for p in proposals], value=[])
 
@@ -228,21 +341,26 @@ def execute_approved_proposals(ids, proposals, history):
     if not ids: return "No selection.", proposals, _format_gate_choices(proposals), history
     results, remaining = [], []
     for p in proposals:
-        if p['id'] in ids: results.append(f"**{p['tool']}**: {execute_staged_tool(p['tool'], p['args'])}")
+        if p['id'] in ids:
+            out = execute_staged_tool(p['tool'], p['args'])
+            results.append(f"**{p['tool']}**: {out}")
         else: remaining.append(p)
     if results: history.append({"role": "assistant", "content": "✅ **Executed:**\n" + "\n".join(results)})
     return "Done.", remaining, _format_gate_choices(remaining), history
 
 def auto_continue_after_approval(history, proposals):
-    last = history[-1].get("content", "")
-    text = last[0].get("text", "") if isinstance(last, list) else str(last)
-    if not text.startswith("✅"): return history, "", proposals, _format_gate_choices(proposals), _stats_label_files(), _stats_label_convos()
-    return agent_loop("[Approved. Continue.]", history, proposals, None)
+    last = history[-1].get("content", "") if history else ""
+    if "✅ **Executed:**" in str(last):
+        return agent_loop("[System: Tools executed. Continue.]", history, proposals, None)
+    return history, "", proposals, _format_gate_choices(proposals), _stats_label_files(), _stats_label_convos()
 
 def _stats_label_files(): return f"📂 Files: {ctx.get_stats().get('total_files', 0)}"
 def _stats_label_convos(): return f"💾 Convos: {ctx.get_stats().get('conversations', 0)}"
 
-# --- UI LAYOUT ---
+# =============================================================================
+# GRADIO INTERFACE
+# =============================================================================
+
 with gr.Blocks(title="🦞 Clawdbot") as demo:
     state_proposals = gr.State([])
     gr.Markdown("# 🦞 Clawdbot Command Center")
@@ -253,9 +371,9 @@ with gr.Blocks(title="🦞 Clawdbot") as demo:
                     stat_f = gr.Markdown(_stats_label_files())
                     stat_c = gr.Markdown(_stats_label_convos())
                     btn_ref = gr.Button("🔄")
-                    file_in = gr.File(label="Upload", file_count="multiple", file_types=['.py', '.js', '.json', '.md', '.txt', '.yaml', '.sh', '.zip', '.env', '.toml', '.sql'])
+                    file_in = gr.File(label="Upload", file_count="multiple")
                 with gr.Column(scale=4):
-                    chat = gr.Chatbot(height=600, avatar_images=(None, "https://em-content.zobj.net/source/twitter/408/lobster_1f99e.png"))
+                    chat = gr.Chatbot(height=600, type="messages", avatar_images=(None, "https://em-content.zobj.net/source/twitter/408/lobster_1f99e.png"))
                     with gr.Row():
                         txt = gr.Textbox(scale=6, placeholder="Prompt...")
                         btn_send = gr.Button("Send", scale=1)