gateway.py

import os
#!/usr/bin/env python3
# ============================================================
# DOLOR3V GATEWAY v6 – Claude Code features
# ============================================================
from dotenv import load_dotenv
load_dotenv('/tmp/dolor3v/.env')
import asyncio
import os, sys, json, time, re, math, subprocess, threading, hashlib, itertools
import urllib.request, urllib.parse, urllib.error
from pathlib import Path
from collections import defaultdict
from datetime import datetime
from fastapi import FastAPI, Request, HTTPException
from fastapi.middleware.cors import CORSMiddleware
import uvicorn, requests
from bs4 import BeautifulSoup
from load_master_prompt import load_master_prompt
from colorthief import ColorThief

PORT = int(os.environ.get("PORT", 7860))

DOLOR3V_KEY = os.getenv("DOLOR3V_KEY", "d3v-master-dolordprince-2026")

GROQ_KEYS = [
    k for k in [
        os.getenv("GROQ_API_KEY", ""),
        os.getenv("GROQ_API_KEY_2", ""),
        os.getenv("GROQ_API_KEY_3", "")
    ] if k
]
GROQ_KEY = GROQ_KEYS[0] if GROQ_KEYS else ""
_groq_idx = 0

OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY", "")
OPENROUTER_KEY = OPENROUTER_API_KEY
OPENROUTER_API_BASE = os.getenv("OPENROUTER_API_BASE", "https://openrouter.ai/api/v1")
OPENROUTER_MODEL = os.getenv("OPENROUTER_MODEL", "qwen/qwen3-coder:free")

OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "")
CEREBRAS_API_KEY = os.getenv("CEREBRAS_API_KEY", "")
ZAI_API_KEY = os.getenv("ZAI_API_KEY", "")

GITHUB_TOKEN = os.getenv("GITHUB_TOKEN", "")
GITHUB_USER = os.getenv("GITHUB_USER", "Daviddolor")

HF_TOKEN = os.getenv("HF_TOKEN", "")
HF_USER = os.getenv("HF_USER", "Daviddolor")

SURGE_TOKEN = os.getenv("SURGE_TOKEN", "")
SURGE_EMAIL = os.getenv("SURGE_EMAIL", "personaldolor@gmail.com")

print("GitHub token loaded:", bool(GITHUB_TOKEN))
print("HF token loaded:", bool(HF_TOKEN))
print("OpenRouter loaded:", bool(OPENROUTER_API_KEY))
print("Groq loaded:", bool(GROQ_KEY))
print("Cerebras loaded:", bool(CEREBRAS_API_KEY))

OLLAMA_URL    = "http://localhost:11434"
TABBY_URL     = "http://localhost:9090"
EVENT_BUS_URL = "http://localhost:9091"
PROJECTS_DIR = os.getenv("PROJECTS_DIR", "/tmp/dolor3v/projects")
MEMORY_DB = "/tmp/dolor3v/memory.json"
LOG_FILE      = "/tmp/dolor3v/mcp.log"
SAFE_MODE     = os.environ.get("DOLOR3V_SAFE_MODE", "true").lower() not in ("0","false","no")
MAX_CONTEXT_CHARS = 4000   # smaller limit to avoid 413 from Groq

os.makedirs(PROJECTS_DIR,        exist_ok=True)
os.makedirs("/tmp/dolor3v",      exist_ok=True)
os.makedirs("/tmp/dolor3v/data", exist_ok=True)

def log(msg):
    ts = datetime.now().strftime("%H:%M:%S")
    line = f"[{ts}] {msg}"
    print(line, flush=True)
    try:
        with open(LOG_FILE,"a") as f: f.write(line+"\n")
    except: pass

app = FastAPI(title="DOLOR3V MCP Gateway v6.0")
app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"])

class Memory:
    _lock = threading.Lock()
    @staticmethod
    def _load():
        try:
            with open(MEMORY_DB) as f: return json.load(f)
        except: return {"kv":{}, "docs":[]}
    @staticmethod
    def _save(data):
        with open(MEMORY_DB,"w") as f: json.dump(data, f, indent=2)
    @staticmethod
    def set(key, value):
        with Memory._lock:
            d = Memory._load()
            d["kv"][key] = {"value": value, "ts": time.time()}
            Memory._save(d)
    @staticmethod
    def get(key):
        d = Memory._load()
        entry = d["kv"].get(key)
        return entry["value"] if entry else None
    @staticmethod
    def ingest(text, source="user"):
        with Memory._lock:
            d = Memory._load()
            d.setdefault("docs", [])
            d["docs"].append({"id": hashlib.md5(text.encode()).hexdigest()[:8],
                               "text": text[:2000], "source": source, "ts": time.time()})
            if len(d["docs"]) > 200: d["docs"] = d["docs"][-200:]
            Memory._save(d)
    @staticmethod
    def search(query, top_k=3):
        d = Memory._load()
        docs = d.get("docs", [])
        if not docs: return []
        def tokenize(t): return re.findall(r'\w+', t.lower())
        q_tokens = set(tokenize(query))
        N = len(docs)
        df = defaultdict(int)
        for doc in docs:
            for tok in set(tokenize(doc["text"])):
                df[tok] += 1
        scores = []
        for doc in docs:
            tokens = tokenize(doc["text"])
            tf = defaultdict(int)
            for t in tokens: tf[t] += 1
            score = 0.0
            for tok in q_tokens:
                if tok in tf:
                    tfidf = (tf[tok]/len(tokens)) * math.log((N+1)/(df[tok]+1)+1)
                    score += tfidf
            scores.append((score, doc))
        scores.sort(key=lambda x: x[0], reverse=True)
        return [d for _, d in scores[:top_k] if _ > 0]
    @staticmethod
    def all_kv():
        return Memory._load().get("kv", {})

class Firewall:
    UA = "Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36"
    @staticmethod
    def search(query, max_results=5):
        try:
            q = urllib.parse.quote_plus(query)
            url = f"https://html.duckduckgo.com/html/?q={q}"
            req = urllib.request.Request(url, headers={"User-Agent": Firewall.UA})
            with urllib.request.urlopen(req, timeout=12) as r:
                html = r.read().decode("utf-8", errors="ignore")
            snippets = re.findall(r'class="result__snippet"[^>]*>(.*?)</a>', html, re.DOTALL)[:max_results]
            titles   = re.findall(r'class="result__a"[^>]*>(.*?)</a>', html, re.DOTALL)[:max_results]
            urls     = re.findall(r'class="result__url"[^>]*>(.*?)</span>', html, re.DOTALL)[:max_results]
            results = []
            for i in range(len(snippets)):
                results.append({"title": re.sub(r"<[^>]+>","", titles[i] if i<len(titles) else ""),
                                 "snippet": re.sub(r"<[^>]+>","", snippets[i]),
                                 "url": urls[i].strip() if i<len(urls) else ""})
            for r in results:
                Memory.ingest(f"{r['title']}: {r['snippet']}", "web_search")
            return results
        except Exception as e:
            log(f"Search error: {e}")
            return []
    @staticmethod
    def fetch(url):
        try:
            req = urllib.request.Request(url, headers={"User-Agent": Firewall.UA})
            with urllib.request.urlopen(req, timeout=15) as r:
                html = r.read().decode("utf-8", errors="ignore")
            text = re.sub(r"<style[^>]*>.*?</style>","", html, flags=re.DOTALL)
            text = re.sub(r"<script[^>]*>.*?</script>","", text, flags=re.DOTALL)
            text = re.sub(r"<[^>]+>"," ", text)
            text = re.sub(r"\s+"," ", text).strip()
            Memory.ingest(text[:1000], f"fetch:{url}")
            return text[:5000]
        except Exception as e:
            return f"Fetch error: {e}"
    @staticmethod
    def gather_context(prompt):
        keywords = re.sub(r'\b(make|build|create|a|an|the|for|with|and|to)\b','', prompt.lower())[:80]
        results = Firewall.search(f"{keywords} production best practices 2025")
        ctx = f"[FIREWALL: {keywords}]\n"
        for r in results[:3]:
            ctx += f"- {r['title']}: {r['snippet']}\n"
        return ctx
def _http_post(url, data, headers=None, timeout=120):
    body = json.dumps(data).encode()
    hdrs = {"Content-Type":"application/json"}
    if headers: hdrs.update(headers)
    req = urllib.request.Request(url, data=body, headers=hdrs)
    with urllib.request.urlopen(req, timeout=timeout) as r:
        return json.loads(r.read())

def llm_ollama(messages, model="dolor3v_coder:turbo"):
    result = _http_post(f"{OLLAMA_URL}/api/chat",
                        {"model": model, "messages": messages, "stream": False,
                         "options": {"num_ctx":1024,"temperature":0.2}})
    return result["message"]["content"]

def llm_groq(messages, model="llama-3.3-70b-versatile", tools=None, tool_choice=None):
    global _groq_idx
    last_err = None
    tries = len(GROQ_KEYS) or 1
    backoff = 4   # start with 4 seconds for 429
    for attempt in range(tries):
        key = GROQ_KEYS[_groq_idx % len(GROQ_KEYS)] if GROQ_KEYS else ""
        _groq_idx += 1
        try:
            payload = {"model": model, "messages": messages, "temperature": 0.2, "max_tokens": 4096}
            if tools:
                payload["tools"] = tools
                if tool_choice:
                    payload["tool_choice"] = tool_choice
            result = _http_post("https://api.groq.com/openai/v1/chat/completions",
                                payload,
                                {"Authorization": f"Bearer {key}", "User-Agent": "Mozilla/5.0 (compatible; dolor3v/1.0)"})
            return result["choices"][0]["message"]
        except urllib.error.HTTPError as e:
            if e.code == 429:
                log(f"Groq rate‑limited, sleeping {backoff}s")
                time.sleep(backoff)
                backoff = min(backoff * 2, 30)
                last_err = e
                # stay on the same key, just wait
            else:
                last_err = e
                log(f"Groq key failed, rotating: {e}")
                continue
        except Exception as e:
            last_err = e
            log(f"Groq key failed, rotating: {e}")
            continue
    raise last_err if last_err else Exception("No Groq keys configured")

def llm_openrouter(messages, model="nvidia/nemotron-super-49b-v1:free", tools=None, tool_choice=None):
    payload = {"model": model, "messages": messages, "temperature": 0.2, "max_tokens": 4096}
    if tools:
        payload["tools"] = tools
        if tool_choice:
            payload["tool_choice"] = tool_choice
    result = _http_post("https://openrouter.ai/api/v1/chat/completions",
                        payload,
                        {"Authorization": f"Bearer {OPENROUTER_KEY}", "HTTP-Referer": "https://dolor3v.com"})
    return result["choices"][0]["message"]

def llm_openai(messages, model="gpt-4o-mini"):
    result = _http_post("https://api.openai.com/v1/chat/completions",
                        {"model": model, "messages": messages, "temperature": 0.2, "max_tokens": 4096},
                        {"Authorization": f"Bearer {OPENAI_API_KEY}"})
    return result["choices"][0]["message"]["content"]

def llm_cerebras(messages, model="gpt-oss-120b"):
    result = _http_post("https://api.cerebras.ai/v1/chat/completions",
                        {"model": model, "messages": messages, "temperature": 0.2, "max_tokens": 4096},
                        {"Authorization": f"Bearer {CEREBRAS_API_KEY}", "User-Agent": "Mozilla/5.0 (compatible; dolor3v/1.0)"})
    msg = result["choices"][0]["message"]
    content = msg.get("content") or msg.get("reasoning") or ""
    if not content:
        raise Exception("Cerebras returned no usable content")
    return content

def llm_glm(messages, model="glm-4.7-flash"):
    result = _http_post("https://api.z.ai/api/paas/v4/chat/completions",
                        {"model": model, "messages": messages, "temperature": 0.2, "max_tokens": 4096},
                        {"Authorization": f"Bearer {ZAI_API_KEY}", "User-Agent": "Mozilla/5.0 (compatible; dolor3v/1.0)"})
    msg = result["choices"][0]["message"]
    content = msg.get("content") or ""
    if not content:
        raise Exception("GLM returned no usable content")
    return content

def llm_route(messages, model=None, tools=None, tool_choice=None):
    query = " ".join(m.get("content","") for m in messages if m.get("role")=="user")[-200:]
    try:
        mem_docs = Memory.search(query)
    except Exception as e:
        log(f"Memory.search failed, continuing without context: {e}")
        mem_docs = None
    if mem_docs:
        context_text = "\n".join([d["text"][:200] for d in mem_docs])
        messages = [{"role":"system","content":context_text}] + messages

    if tools:
        providers = [
            ("groq",       lambda: llm_groq(messages, model=model or "llama-3.3-70b-versatile", tools=tools, tool_choice=tool_choice)),
            ("openrouter", lambda: llm_openrouter(messages, model=model or "nvidia/nemotron-super-49b-v1:free", tools=tools, tool_choice=tool_choice)),
        ]
    else:
        providers = [
            ("groq", lambda: llm_groq(
                messages,
                model=model or "llama-3.3-70b-versatile"
            )),
            ("cerebras", lambda: {
                "role":"assistant",
                "content": llm_cerebras(
                    messages,
                    model=model or "gpt-oss-120b"
                )
            }),
            ("openrouter", lambda: llm_openrouter(
                messages,
                model=model or os.getenv(
                    "OPENROUTER_MODEL",
                    "qwen/qwen3-coder:free"
                )
            )),
        ]

    for name, fn in providers:
        try:
            log(f"LLM → {name}")
            message = fn()
            content_preview = message.get("content") or ""
            try:
                Memory.ingest(f"Q:{query[:100]} A:{content_preview[:200]}", "llm")
            except Exception as e:
                log(f"Memory.ingest failed, ignoring: {e}")
            emit_event("custom", {"provider":name,"tokens":len(content_preview)}, "mcp-llm")
            return message, name
        except Exception as e:
            import traceback
            traceback.print_exc()
            print(f"[LLM ERROR] {name}: {repr(e)}")
            log(f"LLM {name} failed: {repr(e)}")
            continue
    return {"role":"assistant","content":"All LLM providers failed"}, "none"

def llm_route_quality(messages, model=None):
    query = " ".join(m.get("content","") for m in messages if m.get("role")=="user")[-200:]
    mem_docs = Memory.search(query)
    if mem_docs:
        context_text = "\n".join([d["text"][:200] for d in mem_docs])
        messages = [{"role":"system","content":context_text}] + messages
    providers = [
        ("cerebras",   lambda: llm_cerebras(messages)),
        ("groq",       lambda: llm_groq(messages)),
        ("glm",        lambda: llm_glm(messages)),
        ("openrouter", lambda: llm_openrouter(messages)),
        ("openai",     lambda: llm_openai(messages)),
        ("ollama",     lambda: llm_ollama(messages))
    ]
    for name, fn in providers:
        try:
            log(f"LLM(quality) \u2192 {name}")
            result = fn()
            Memory.ingest(f"Q:{query[:100]} A:{result[:200]}", "llm")
            emit_event("custom", {"provider":name,"tokens":len(result)}, "mcp-llm")
            return result, name
        except Exception as e:
            log(f"LLM(quality) {name} failed: {e}")
            continue
    return "All LLM providers failed", "none"

def emit_event(etype, payload, source="mcp"):
    try:
        _http_post(f"{EVENT_BUS_URL}/emit",
                   {"type":etype,"payload":payload,"source":source}, timeout=3)
    except: pass

class GitHub:
    @staticmethod
    def create_repo(name, desc="DOLOR3V project"):
        data = json.dumps({"name": name, "description": desc, "private": False, "auto_init": True}).encode()
        req = urllib.request.Request("https://api.github.com/user/repos", data=data,
                                      headers={"Authorization": f"token {GITHUB_TOKEN}",
                                               "Accept": "application/vnd.github.v3+json",
                                               "Content-Type": "application/json",
                                               "User-Agent": "DOLOR3V-MCP"})
        with urllib.request.urlopen(req, timeout=15) as r:
            result = json.loads(r.read())
        return result.get("clone_url",""), result.get("html_url","")
    @staticmethod
    def push(project_path, repo_name, msg="DOLOR3V build"):
        if not GITHUB_TOKEN: return "❌ GITHUB_TOKEN not set"
        try:
            clone_url, html_url = GitHub.create_repo(repo_name)
            if not clone_url: return "❌ Could not create repo"
            auth_url = clone_url.replace("https://", f"https://{GITHUB_TOKEN}@")
            cmds = f"cd {project_path} && git init -q && git config user.email '{SURGE_EMAIL}' && git config user.name '{GITHUB_USER}' && git add -A && git commit -q -m '{msg}' && git branch -M main && git remote add origin {auth_url} 2>/dev/null || git remote set-url origin {auth_url} && git push -u origin main --force -q"
            r = subprocess.run(cmds, shell=True, capture_output=True, text=True, timeout=60)
            if r.returncode == 0:
                emit_event("deploy", {"github":html_url,"path":project_path}, "mcp-github")
                return f"✅ {html_url}"
            return f"❌ {r.stderr[:200]}"
        except Exception as e:
            return f"❌ GitHub error: {e}"
    @staticmethod
    def push_source():
        return GitHub.push("/tmp/dolor3v", "dolor3v-engine", "DOLOR3V: engine update")

class Surge:
    @staticmethod
    def deploy(project_path, subdomain=None):
        if not subdomain:
            slug = re.sub(r'[^a-z0-9-]','-', Path(project_path).name.lower())
            subdomain = f"{slug}-{int(time.time())}"
        domain = f"{subdomain}.surge.sh"
        env = os.environ.copy()
        if SURGE_TOKEN: env["SURGE_TOKEN"] = SURGE_TOKEN
        try:
            r = subprocess.run(["surge", project_path, domain], capture_output=True, text=True, env=env, timeout=120)
            if r.returncode == 0 or "Success" in r.stdout:
                url = f"https://{domain}"
                emit_event("deploy", {"url":url}, "mcp-surge")
                return url, None
            return None, r.stderr[:300] or r.stdout[:300]
        except FileNotFoundError:
            return None, "surge not installed"
        except Exception as e:
            return None, str(e)

def tabby_complete(prefix, suffix="", lang="python"):
    try:
        result = _http_post(f"{TABBY_URL}/v1beta/completions",
                            {"language": lang, "segments": {"prefix":prefix,"suffix":suffix}}, timeout=60)
        return result.get("choices",[{}])[0].get("text","")
    except Exception as e:
        return f"Tabby error: {e}"
TOOL_LIST = [
    "write_file","read_file","list_files","delete_file",
    "shell_exec","web_search","web_fetch","firewall_context",
    "memory_save","memory_recall","memory_search","memory_ingest",
    "github_push","github_push_source","git_commit",
    "surge_deploy","tabby_complete","hf_deploy",
    "watchdog_ping","code_parse","clone_website",
    "project_index","grep_code","spawn_agent","qa_review","build_project"
]

DANGEROUS_PATTERNS = [
    r'\brm\s+-rf\b', r'\brm\s+-r\b', r'\brm\s+.*-f\b',
    r'>\s*/dev/sd', r'\bmkfs\.', r'\bdd\s+if=',
    r'git\s+push\s+--force', r'git\s+push\s+-f',
    r'\bchmod\s+777', r'\bchown\s+-R',
    r':(){ :|:& };:'
]

def is_dangerous(cmd):
    return any(re.search(pat, cmd) for pat in DANGEROUS_PATTERNS)

def retry_tool(tool_func, *args, max_retries=2):
    for attempt in range(max_retries + 1):
        try:
            return tool_func(*args)
        except Exception as e:
            if attempt == max_retries:
                raise
            time.sleep(1)
            log(f"Retry {attempt+1}/{max_retries} for tool: {e}")

def project_index(path="."):
    path = os.path.expanduser(path)
    summary = []
    summary.append(f"Project root: {os.path.abspath(path)}")
    try:
        r = subprocess.run(["find", path, "-maxdepth", "2", "-not", "-path", "*/node_modules/*", "-not", "-path", "*/.git/*"],
                           capture_output=True, text=True, timeout=10)
        tree = r.stdout.strip()
        summary.append(f"File tree (depth 2):\n{tree[:2000]}")
    except:
        pass

    # Collect up to 15 important source files and include their first 50 lines
    source_files = []
    for root, dirs, files in os.walk(path):
        dirs[:] = [d for d in dirs if d not in ('.git', 'node_modules', '__pycache__')]
        for f in files:
            if f.endswith(('.py', '.js', '.ts', '.sh', '.yaml', '.yml', '.json', '.toml')):
                source_files.append(os.path.join(root, f))
        if len(source_files) >= 20:
            break
    for fpath in sorted(source_files)[:15]:
        try:
            with open(fpath) as f:
                flines = f.readlines()[:50]
            header = ''.join(flines)
            rel = os.path.relpath(fpath, path)
            summary.append(f"\n--- {rel} (first 50 lines) ---\n{header}")
        except:
            pass

    manifests = {
        "package.json": None,
        "requirements.txt": None,
        "Pipfile": None,
        "pyproject.toml": None,
        "go.mod": None,
        "Cargo.toml": None,
        "Makefile": None,
        "README.md": None,
        ".env.example": None,
        "docker-compose.yml": None,
    }
    for fname in manifests:
        fpath = os.path.join(path, fname)
        if os.path.isfile(fpath):
            try:
                with open(fpath) as f:
                    mcontent = f.read(2000)
                manifests[fname] = mcontent
            except:
                pass

    for name, mcontent in manifests.items():
        if mcontent:
            summary.append(f"\n--- {name} ---\n{mcontent}")

    try:
        r = subprocess.run(["git", "-C", path, "status", "--short"], capture_output=True, text=True, timeout=5)
        git_stat = r.stdout.strip()
        if git_stat:
            summary.append(f"\n--- git status ---\n{git_stat}")
    except:
        pass

    return "\n".join(summary)

def grep_code(directory, pattern, file_filter="*"):
    cmd = f"grep -rn --include='{file_filter}' '{pattern}' {directory}"
    r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=15)
    return r.stdout.strip() or "No matches found."

def spawn_agent(prompt, max_steps=3):
    # AGENT_SYSTEM_PROMPT will be defined later; we'll handle that after function definition
    msgs = [{"role":"system","content":AGENT_SYSTEM_PROMPT},
            {"role":"user","content":prompt}]
    for _ in range(max_steps):
        try:
            reply, _ = llm_route(msgs)
        except:
            return "Sub-agent failed: no LLM response."
        line = reply.strip()
        if line.startswith("{") and '"tool"' in line:
            try:
                tc = json.loads(line)
                tool_res = execute_tool(tc["tool"], tc.get("args",{}))
                msgs.append({"role":"assistant","content":line})
                msgs.append({"role":"user","content":f"Tool result:\n{tool_res}"})
                continue
            except:
                return reply
        return reply
    return "Sub-agent reached max steps."

# AGENT_SYSTEM_PROMPT placeholder – real definition will be in part 5
AGENT_SYSTEM_PROMPT = ""

def execute_tool(tool, args):
    log(f"Tool: {tool} args:{str(args)[:80]}")
    # Safety checks
    if SAFE_MODE and tool == "shell_exec":
        cmd = args.get("command","")
        if is_dangerous(cmd):
            return "🚫 SAFE MODE: Dangerous command blocked. Set DOLOR3V_SAFE_MODE=false to disable."
    if SAFE_MODE and tool == "delete_file":
        path = args.get("path","")
        if any(re.search(pat, path) for pat in [r'\/$', r'^\/(etc|boot|bin|sbin|lib|sys|dev|proc)']):
            return "🚫 SAFE MODE: Deletion of system path blocked."

    # ----- standard tools -----
    if tool == "write_file":
        filename = os.path.basename(args.get("path",""))
        file_path_arg = args.get("path", args.get("file_path",""))

        # Block dangerous system paths
        BLOCKED_PATHS = ["/etc/","/boot/","/bin/","/sbin/","/lib/","/sys/","/dev/","/proc/"]
        if any(file_path_arg.startswith(bp) for bp in BLOCKED_PATHS):
            return {"error": "BLOCKED", "message": f"System path blocked: {file_path_arg}"}

        # Allow all legitimate web/app file extensions
        ALLOWED_EXTENSIONS = {
            ".html",".css",".js",".ts",".tsx",".jsx",".json",".svg",
            ".md",".mdx",".txt",".env",".yaml",".yml",".toml",".lock",
            ".png",".jpg",".jpeg",".gif",".webp",".ico",".woff",".woff2"
        }
        ext = os.path.splitext(filename)[1].lower()
        if ext and ext not in ALLOWED_EXTENSIONS:
            return {"error": "BLOCKED", "message": f"File extension not allowed: {ext}", "blocked_file": filename}

        path = os.path.expanduser(args.get("path", args.get("file_path","")))
        content = args.get("content","")
        os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
        with open(path,"w") as f: f.write(content)
        return f"✅ Written: {path} ({len(content)} bytes)"
    elif tool == "read_file":
        path = os.path.expanduser(args.get("path", args.get("file_path","")))
        with open(path) as f: return f.read()[:2000][:3000]
    elif tool == "list_files":
        path = os.path.expanduser(args.get("path","."))
        items = list(Path(path).iterdir())
        return "\n".join(f"{'D' if i.is_dir() else 'F'} {i.name}" for i in sorted(items))
    elif tool == "delete_file":
        path = os.path.expanduser(args.get("path", args.get("file_path","")))
        os.remove(path)
        return f"✅ Deleted: {path}"
    elif tool == "shell_exec":
        cmd = args.get("command","")
        r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=120)
        out = (r.stdout + r.stderr).strip()
        return out[:3000] or "✅ Done (no output)"
    elif tool == "web_search":
        results = Firewall.search(args.get("query",""), args.get("max_results",5))
        return json.dumps(results, indent=2)
    elif tool == "web_fetch":
        return Firewall.fetch(args.get("url",""))
    elif tool == "firewall_context":
        return Firewall.gather_context(args.get("prompt",""))
    elif tool == "memory_save":
        Memory.set(args["key"], args["value"])
        return f"✅ Saved: {args['key']}"
    elif tool == "memory_recall":
        val = Memory.get(args["key"])
        return val if val else f"No memory for: {args['key']}"
    elif tool == "memory_search":
        results = Memory.search(args.get("query",""))
        return json.dumps([r["text"][:200] for r in results], indent=2)
    elif tool == "memory_ingest":
        Memory.ingest(args.get("text",""), args.get("source","user"))
        return "✅ Ingested into RAG"
    elif tool == "github_push":
        return GitHub.push(args.get("project_path",""),
                            args.get("repo_name", Path(args.get("project_path","x")).name),
                            args.get("message","DOLOR3V build"))
    elif tool == "github_push_source":
        return GitHub.push_source()
    elif tool == "git_commit":
        path = args.get("path",".")
        msg = args.get("message","update")
        r = subprocess.run(f"cd {path} && git add -A && git commit -m '{msg}'",
                            shell=True, capture_output=True, text=True)
        return r.stdout + r.stderr
    elif tool == "surge_deploy":
        url, err = Surge.deploy(args.get("project_path",""), args.get("subdomain"))
        return f"✅ Live: {url}" if url else f"❌ {err}"
    elif tool == "tabby_complete":
        return tabby_complete(args.get("prefix",""), args.get("suffix",""), args.get("lang","python"))
    elif tool == "hf_deploy":
        space = args.get("space", f"{HF_USER}/dolor3v")
        src = args.get("src_path", "/tmp/dolor3v")
        if not HF_TOKEN: return "❌ HF_TOKEN not set"
        hf_url = f"https://user:{HF_TOKEN}@huggingface.co/spaces/{space}"
        r = subprocess.run(f"cd {src} && git init && git add -A && git commit -m 'DOLOR3V deploy' && git remote add hf {hf_url} 2>/dev/null || git remote set-url hf {hf_url} && git push hf main --force",
                            shell=True, capture_output=True, text=True, timeout=120)
        return f"✅ Deployed to https://{space.replace('/','--')}.hf.space" if r.returncode==0 else f"❌ {r.stderr[:200]}"
    elif tool == "watchdog_ping":
        service = args.get("service","unknown")
        status = args.get("status","ok")
        emit_event("watchdog", {"service":service,"status":status}, "mcp-watchdog")
        return f"✅ Watchdog: {service}={status}"
    elif tool == "code_parse":
        path = os.path.expanduser(args.get("path", args.get("file_path","")))
        with open(path) as f: content = f.read()
        lines = content.split("\n")
        funcs = [l.strip() for l in lines if l.strip().startswith(("def ","class ","function ","const ","async "))]
        return json.dumps({"lines": len(lines), "size": len(content), "symbols": funcs[:20]}, indent=2)
    elif tool == "clone_website":
        url = args.get("url")
        proj_dir = args.get("project_dir")
        if not url: return "❌ missing url"
        try:
            resp = requests.get(url, headers={"User-Agent": Firewall.UA}, timeout=15)
            soup = BeautifulSoup(resp.text, "html.parser")
            title = soup.title.string.strip() if soup.title else url
            desc = soup.find("meta", attrs={"name": "description"})
            description = desc.get("content","") if desc else ""
            og_image = soup.find("meta", property="og:image")
            hero_url = urllib.parse.urljoin(url, og_image["content"]) if og_image else None
            colors = ["#3B82F6", "#1E3A8A", "#F59E0B"]
            if hero_url:
                img_data = requests.get(hero_url, timeout=10).content
                import tempfile
                with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp:
                    tmp.write(img_data)
                    tmp_path = tmp.name
                ct = ColorThief(tmp_path)
                palette = ct.get_palette(color_count=3)
                colors = [f"rgb({r},{g},{b})" for (r,g,b) in palette]
                os.unlink(tmp_path)
            html = f"""<!DOCTYPE html>
<html><head><meta charset="UTF-8"><title>{title}</title>
<style>body{{font-family:system-ui;margin:0;background:{colors[0]};color:white;text-align:center;}}
.hero{{height:100vh;display:flex;flex-direction:column;justify-content:center;}}
button{{background:white;color:{colors[0]};border:none;padding:12px 24px;border-radius:30px;}}</style>
</head><body><div class="hero"><h1>{title}</h1><p>{description[:200]}</p><button>Explore</button></div></body></html>"""
            if proj_dir:
                os.makedirs(proj_dir, exist_ok=True)
                with open(os.path.join(proj_dir, "index.html"), "w") as f:
                    f.write(html)
                if hero_url:
                    img_data = requests.get(hero_url, timeout=10).content
                    with open(os.path.join(proj_dir, "hero.jpg"), "wb") as f:
                        f.write(img_data)
                return {"status": "done", "path": proj_dir}
            return {"html": html}
        except Exception as e:
            return {"error": str(e)}
    # ----- new Claude Code tools -----
    elif tool == "project_index":
        return project_index(args.get("path","."))
    elif tool == "grep_code":
        directory = args.get("directory",".")
        pattern = args.get("pattern","")
        file_filter = args.get("file_filter","*")
        if not pattern: return "❌ missing pattern"
        return grep_code(directory, pattern, file_filter)
    elif tool == "spawn_agent":
        prompt = args.get("prompt","")
        if not prompt: return "❌ missing prompt"
        return spawn_agent(prompt, args.get("max_steps",3))
    elif tool == "build_project":
        build_dir = args.get("path", args.get("project_dir", ""))
        command   = args.get("command", "")
        if not build_dir or not command:
            return "❌ build_project requires path and command"
        log_file = os.path.join(build_dir, "build.log")
        os.makedirs(build_dir, exist_ok=True)
        try:
            with open(log_file, "w") as lf:
                r = subprocess.run(
                    command, shell=True, cwd=build_dir,
                    stdout=lf, stderr=lf, timeout=600
                )
            with open(log_file) as lf:
                output = lf.read()[-3000:]
            status = "✅ Build succeeded" if r.returncode == 0 else f"❌ Build failed (exit {r.returncode})"
            return f"{status}\n{output}"
        except subprocess.TimeoutExpired:
            return "❌ build_project timed out after 600s"
        except Exception as e:
            return f"❌ build_project error: {e}"
    elif tool == "qa_review":
        proj_dir_arg = args.get("path", args.get("project_dir", ""))
        if not proj_dir_arg:
            return "❌ missing path"
        issues = []
        html_path = os.path.join(proj_dir_arg, "index.html")
        css_path = os.path.join(proj_dir_arg, "style.css")
        if os.path.exists(html_path):
            with open(html_path) as f:
                html_content = f.read()
            soup = BeautifulSoup(html_content, "html.parser")
            if not soup.find("title") or not soup.title.string or not soup.title.string.strip():
                issues.append("Missing or empty title tag")
            if not soup.find("meta", attrs={"name": "description"}):
                issues.append("Missing meta description tag")
            if not soup.find("meta", attrs={"name": "viewport"}):
                issues.append("Missing viewport meta tag")
            imgs_missing_alt = [img.get("src","?") for img in soup.find_all("img") if not img.get("alt")]
            if imgs_missing_alt:
                issues.append(f"{len(imgs_missing_alt)} image(s) missing alt text")
            if not soup.find("h1"):
                issues.append("Missing an h1 heading")
        else:
            issues.append("index.html not found")
        if os.path.exists(css_path):
            with open(css_path) as f:
                css_content = f.read()
            if "@media" not in css_content:
                issues.append("No media query found, page may not be responsive")
        else:
            issues.append("style.css not found")
        if not issues:
            return "QA review passed: no issues found"
        return "QA review found issues: " + "; ".join(issues)
    else:
        return f"❌ Unknown tool: {tool}"

@app.post("/v1/build/nextjs")
async def build_nextjs(request: Request):
    body = await request.json()
    prompt = body.get("prompt", "")
    if not prompt:
        return {"ok": False, "error": "prompt required"}

    slug = re.sub(r'[^a-z0-9-]', '-', prompt.lower())[:30]
    ts = datetime.now().strftime("%m%d%H%M")
    proj_name = f"dolor3v-{slug}-{ts}"
    proj_dir = f"{PROJECTS_DIR}/{proj_name}"
    os.makedirs(proj_dir, exist_ok=True)
    log(f"NextJS build started: {proj_dir}")

    # ── Phase 1: Research ──────────────────────────────────────────
    research = {}

    # 1a. Web search for design inspiration
    search_results = Firewall.search(f"{prompt} website design inspiration color palette", 5)
    research["search"] = search_results

    # 1b. Fetch top result for real copy + colors
    top_url = next((r.get("url") for r in search_results if r.get("url")), None)
    fetched_text = ""
    if top_url:
        try:
            fetched_text = Firewall.fetch(top_url)[:3000]
            research["fetched_url"] = top_url
            research["fetched_text"] = fetched_text
        except Exception as e:
            log(f"web_fetch failed: {e}")

    # 1c. Extract color palette from a real image
    palette = ["#1a1a2e", "#16213e", "#e94560"]
    try:
        img_keyword = re.sub(r'[^a-z0-9]', '-', prompt.lower())[:20]
        img_url = f"https://picsum.photos/seed/{img_keyword}/800/600"
        img_resp = requests.get(img_url, timeout=10)
        if img_resp.status_code == 200:
            import tempfile
            with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp:
                tmp.write(img_resp.content)
                tmp_path = tmp.name
            ct = ColorThief(tmp_path)
            raw = ct.get_palette(color_count=3, quality=1)
            palette = [f"#{r:02x}{g:02x}{b:02x}" for r,g,b in raw]
            os.unlink(tmp_path)
            research["palette"] = palette
            research["hero_img"] = img_url
            log(f"Extracted palette: {palette}")
    except Exception as e:
        log(f"Color extraction failed: {e}")
        research["palette"] = palette

    # 1d. Pick font pair from Google Fonts
    font_pairs = [
        ("Playfair Display", "Inter"),
        ("DM Serif Display", "DM Sans"),
        ("Cormorant Garamond", "Nunito Sans"),
        ("Libre Baskerville", "Source Sans Pro"),
        ("Fraunces", "Outfit"),
    ]
    import random
    heading_font, body_font = random.choice(font_pairs)
    research["heading_font"] = heading_font
    research["body_font"] = body_font

    log(f"Research complete: palette={palette}, fonts={heading_font}/{body_font}")

    # ── Phase 2: Scaffold Next.js ──────────────────────────────────
    scaffold_log = f"{proj_dir}/scaffold.log"
    scaffold_cmd = (
        f"cd {PROJECTS_DIR} && "
        f"pnpm create next-app {proj_name} --typescript --tailwind --eslint "
        f"--app --no-src-dir --no-git --yes 2>&1"
    )
    log("Scaffolding Next.js...")
    scaffold_result = await asyncio.to_thread(
        subprocess.run, scaffold_cmd, shell=True,
        capture_output=True, text=True, timeout=600
    )
    with open(scaffold_log, "w") as f:
        f.write(scaffold_result.stdout + scaffold_result.stderr)

    if scaffold_result.returncode != 0:
        return {"ok": False, "error": "Scaffold failed", "log": scaffold_result.stderr[-2000:]}

    # Write next.config.ts for static export
    next_config = (
        "import type { NextConfig } from 'next';\n"
        "const nextConfig: NextConfig = {\n"
        "  output: 'export',\n"
        "  images: { unoptimized: true }\n"
        "};\n"
        "export default nextConfig;\n"
    )
    with open(f"{proj_dir}/next.config.ts", "w") as f:
        f.write(next_config)

    # Install framer-motion
    fm_result = await asyncio.to_thread(
        subprocess.run, f"cd {proj_dir} && pnpm add framer-motion 2>&1",
        shell=True, capture_output=True, text=True, timeout=300
    )
    log(f"framer-motion install: exit {fm_result.returncode}")

    log("Scaffold complete, handing off to LLM for components...")
    return {
        "ok": True,
        "phase": "scaffolded",
        "proj_dir": proj_dir,
        "research": research,
        "message": "Scaffold complete. Call /v1/build/nextjs/complete to write components and build."
    }



@app.post("/v1/build/nextjs/complete")
async def build_nextjs_complete(request: Request):
    body = await request.json()
    prompt = body.get("prompt", "")
    proj_dir = body.get("proj_dir", "")
    research = body.get("research", {})
    if not prompt or not proj_dir:
        return {"ok": False, "error": "prompt and proj_dir required"}

    palette = research.get("palette", ["#1a1a2e","#16213e","#e94560"])
    heading_font = research.get("heading_font", "Playfair Display")
    body_font = research.get("body_font", "Inter")
    hero_img = research.get("hero_img", "https://picsum.photos/seed/hero/1200/800")
    fetched_text = research.get("fetched_text", "")[:1500]
    search_snippets = " ".join([r.get("snippet","") for r in research.get("search",[])])[:1000]

    context = f"""
RESEARCH RESULTS:
Design inspiration snippets: {search_snippets}
Fetched content: {fetched_text}
Color palette (use these exact hex values): {palette[0]} (primary), {palette[1]} (secondary), {palette[2]} (accent)
Heading font: {heading_font} (load from Google Fonts)
Body font: {body_font} (load from Google Fonts)
Hero image URL: {hero_img}
Project directory: {proj_dir}

TASK:
Write a complete, production-ready Next.js {prompt} website.
Use the exact colors, fonts, and image URLs from the research above.
Write ALL of these files using write_file:
1. {proj_dir}/app/globals.css - Import Google Fonts, define CSS variables for the palette
2. {proj_dir}/app/layout.tsx - Root layout with metadata, font imports
3. {proj_dir}/app/page.tsx - Home: hero, features, testimonials (2-3 real quotes), FAQ (3-5 Q&As), CTA
4. {proj_dir}/app/about/page.tsx - About: brand story, team
5. {proj_dir}/app/services/page.tsx - Services: detailed cards
6. {proj_dir}/app/contact/page.tsx - Contact: form, social links
7. {proj_dir}/components/Navbar.tsx - Responsive nav with mobile menu
8. {proj_dir}/components/Footer.tsx - Rich footer with links

Every image must use a real picsum.photos URL.
Use Framer Motion for scroll animations and hover effects.
Make every page fully responsive and accessible.
"""

    system = load_master_prompt() + f"""
You are writing Next.js TypeScript components only.
Use write_file for every file.
Do not scaffold, do not install packages, do not build — just write the component files.
{context}
After writing ALL files respond with exactly: ◆DONE
"""
    msgs = [{"role":"system","content":system},{"role":"user","content":prompt}]
    results = []
    response, provider = "", "groq"
    MAX_STEPS = 20
    parse_fail_count = 0

    for _step in range(MAX_STEPS):
        response, provider = await asyncio.to_thread(llm_route_quality, msgs)
        step_results = []
        decoder = json.JSONDecoder()
        idx = 0
        while idx < len(response):
            brace_idx = response.find("{", idx)
            if brace_idx == -1: break
            try:
                tc, end_idx = decoder.raw_decode(response, brace_idx)
            except json.JSONDecodeError:
                idx = brace_idx + 1
                continue
            idx = end_idx
            if isinstance(tc, dict) and "tool" not in tc:
                preceding = response[max(0,brace_idx-150):brace_idx].lower()
                for tname in TOOL_LIST:
                    if tname.lower() in preceding:
                        tc = {"tool": tname, "args": tc}
                        break
            if isinstance(tc, dict) and "tool" in tc:
                args = tc.get("args", {})
                if tc.get("tool") == "write_file":
                    if not args.get("path") and not args.get("file_path"):
                        args["path"] = os.path.join(proj_dir, "app/page.tsx")
                    content = str(args.get("content",""))
                    img_pattern = re.compile(r'src=["\'](?!https?://)(.*?)["\']')
                    def fix_img(m):
                        kw = re.sub(r'[^a-z0-9]','-',m.group(1).lower())[:20] or "photo"
                        return f'src="https://picsum.photos/seed/{kw}/800/600"'
                    args["content"] = img_pattern.sub(fix_img, content)
                r = await asyncio.to_thread(retry_tool, execute_tool, tc["tool"], args)
                results.append({"tool":tc["tool"],"result":str(r)[:200]})
                step_results.append({"tool":tc["tool"],"result":str(r)[:1500]})

        if "◆DONE" in response or "\u25c6DONE" in response:
            break
        if not step_results:
            parse_fail_count += 1
            if parse_fail_count > 2: break
            msgs.append({"role":"assistant","content":response})
            msgs.append({"role":"user","content":"Could not parse tool call. Use ONLY: {\"tool\":\"write_file\",\"args\":{\"path\":\"...\",\"content\":\"...\"}}. No prose."})
            continue
        parse_fail_count = 0
        msgs.append({"role":"assistant","content":response})
        msgs.append({"role":"user","content":"Tool results:\n"+json.dumps(step_results)[:2000]+"\nContinue writing remaining files, or ◆DONE if all done."})

    # ── Phase 3: Build with auto-fix ──────────────────────────────
    build_result = {"status":"not_run","log":""}
    for attempt in range(3):
        log(f"pnpm build attempt {attempt+1}...")
        br = await asyncio.to_thread(
            subprocess.run,
            f"cd {proj_dir} && pnpm run build 2>&1",
            shell=True, capture_output=True, text=True, timeout=600
        )
        build_log = (br.stdout + br.stderr)[-3000:]
        if br.returncode == 0:
            build_result = {"status":"success","log":build_log}
            log("Build succeeded")
            break
        log(f"Build failed attempt {attempt+1}, searching for fix...")
        err_lines = [l for l in build_log.splitlines() if "error" in l.lower() or "Error" in l][:5]
        err_summary = " ".join(err_lines)[:300]
        fix_search = Firewall.search(f"Next.js TypeScript build error fix: {err_summary}", 3)
        fix_context = " ".join([r.get("snippet","") for r in fix_search])[:1000]
        fix_msgs = [
            {"role":"system","content":"You are a Next.js expert. Fix the build error. Use write_file to overwrite the broken file only. Respond with one write_file tool call then ◆DONE."},
            {"role":"user","content":f"Build error:\n{err_summary}\n\nFix suggestions:\n{fix_context}\n\nProject dir: {proj_dir}"}
        ]
        fix_resp, _ = await asyncio.to_thread(llm_route_quality, fix_msgs)
        fix_decoder = json.JSONDecoder()
        fix_idx = 0
        while fix_idx < len(fix_resp):
            bi = fix_resp.find("{", fix_idx)
            if bi == -1: break
            try:
                ftc, fend = fix_decoder.raw_decode(fix_resp, bi)
            except json.JSONDecodeError:
                fix_idx = bi + 1
                continue
            fix_idx = fend
            if isinstance(ftc, dict) and ftc.get("tool") == "write_file":
                await asyncio.to_thread(retry_tool, execute_tool, "write_file", ftc.get("args",{}))
                results.append({"tool":"write_file","result":"auto-fix applied"})
        build_result = {"status":f"failed_attempt_{attempt+1}","log":build_log}

    # ── Phase 4: Deploy ───────────────────────────────────────────
    gh_url, surge_url = "", ""
    out_dir = f"{proj_dir}/out"
    if build_result["status"] == "success" and os.path.isdir(out_dir):
        slug = re.sub(r'[^a-z0-9-]','-',Path(proj_dir).name.lower())
        if GITHUB_TOKEN:
            try:
                gh_url = await asyncio.to_thread(GitHub.push, proj_dir, slug)
            except Exception as e:
                gh_url = f"❌ {e}"
        try:
            surge_url_r, _ = await asyncio.to_thread(Surge.deploy, out_dir)
            if surge_url_r: surge_url = surge_url_r
        except Exception as e:
            surge_url = f"❌ {e}"
    else:
        surge_url = "❌ Build did not produce out/ folder"

    qa_result = execute_tool("qa_review", {"path": proj_dir})

    return {
        "ok": True,
        "prompt": prompt,
        "project": proj_dir,
        "provider": provider,
        "tools_run": results,
        "build": build_result["status"],
        "build_log": build_result["log"][-500:],
        "github": gh_url,
        "surge": surge_url,
        "qa": qa_result,
        "response": "◆DONE" if build_result["status"]=="success" else "Build failed"
    }


# ========== Routes (all blocking calls offloaded to threads) ==========
@app.get("/health")
async def health():
    kv = Memory.all_kv()
    return {
        "status": "ok",
        "service": "DOLOR3V MCP Gateway v6.0",
        "version": "6.0.0",
        "port": PORT,
        "tools": TOOL_LIST,
        "safe_mode": SAFE_MODE,
        "providers": {
            "ollama": True,
            "groq": bool(GROQ_KEY),
            "openrouter": bool(OPENROUTER_KEY),
            "github": bool(GITHUB_TOKEN),
            "surge": bool(SURGE_TOKEN),
            "hf": bool(HF_TOKEN),
            "tabby": True,
            "event_bus": True
        },
        "memory_keys": len(kv),
        "uptime": time.time()
    }

@app.get("/v1/tools")
async def list_tools():
    return {"tools": TOOL_LIST}

@app.get("/v1/memory")
async def get_memory():
    return Memory.all_kv()

@app.post("/v1/tools/call")
async def call_tool(request: Request):
    body = await request.json()
    tool = body.get("tool")
    args = body.get("arguments", body.get("args", {}))
    t0 = time.time()
    try:
        result = await asyncio.to_thread(retry_tool, execute_tool, tool, args)
        ms = int((time.time()-t0)*1000)
        emit_event("custom", {"tool":tool,"ms":ms}, "mcp")
        return {"result": result, "tool": tool, "ms": ms}
    except Exception as e:
        raise HTTPException(500, detail=str(e))

@app.post("/v1/chat/completions")
async def chat(request: Request):
    body = await request.json()
    messages = body.get("messages", [])
    model = body.get("model", "")
    tools = body.get("tools")
    tool_choice = body.get("tool_choice")
    message, provider = await asyncio.to_thread(llm_route, messages, model, tools, tool_choice)
    finish_reason = "tool_calls" if message.get("tool_calls") else "stop"
    return {
        "id": f"dolor3v-{int(time.time())}",
        "object": "chat.completion",
        "provider": provider,
        "choices": [{"index": 0, "message": message, "finish_reason": finish_reason}]
    }

@app.post("/v1/search")
async def search(request: Request):
    body = await request.json()
    results = await asyncio.to_thread(Firewall.search, body.get("query",""), body.get("max_results",5))
    return {"results": results}

@app.post("/v1/memory")
async def memory_ops(request: Request):
    body = await request.json()
    action = body.get("action","set")
    if action == "set":
        await asyncio.to_thread(Memory.set, body["key"], body["value"])
        return {"ok": True}
    elif action == "get":
        val = await asyncio.to_thread(Memory.get, body["key"])
        return {"value": val}
    elif action == "search":
        r = await asyncio.to_thread(Memory.search, body.get("query",""))
        return {"results": [x["text"][:300] for x in r]}
    elif action == "ingest":
        await asyncio.to_thread(Memory.ingest, body.get("text",""), body.get("source","api"))
        return {"ok": True}
    else:
        raise HTTPException(400, "unknown action")

@app.post("/v1/deploy/github")
async def deploy_github(request: Request):
    body = await request.json()
    result = await asyncio.to_thread(GitHub.push, body.get("project_path",""), body.get("repo_name","dolor3v-project"), body.get("message","DOLOR3V build"))
    return {"result": result}

@app.post("/v1/deploy/surge")
async def deploy_surge(request: Request):
    body = await request.json()
    url, err = await asyncio.to_thread(Surge.deploy, body.get("project_path",""), body.get("subdomain"))
    if url:
        return {"url": url}
    else:
        raise HTTPException(500, detail=err)

@app.post("/v1/deploy/hf")
async def deploy_hf(request: Request):
    body = await request.json()
    result = await asyncio.to_thread(execute_tool, "hf_deploy", body)
    return {"result": result}

@app.post("/v1/deploy/source")
async def deploy_source():
    result = await asyncio.to_thread(GitHub.push_source)
    return {"result": result}

@app.post("/v1/complete")
async def complete(request: Request):
    body = await request.json()
    result = await asyncio.to_thread(tabby_complete, body.get("prefix",""), body.get("suffix",""), body.get("lang","python"))
    return {"completion": result}

@app.post("/v1/firewall")
async def firewall(request: Request):
    body = await request.json()
    ctx = await asyncio.to_thread(Firewall.gather_context, body.get("prompt",""))
    return {"context": ctx}

@app.post("/watchdog")
async def watchdog(request: Request):
    body = await request.json()
    emit_event("watchdog", body, "mcp-watchdog")
    return {"ok": True}

@app.post("/v1/build")
async def build(request: Request):
    body = await request.json()
    prompt = body.get("prompt","")
    slug = re.sub(r'[^a-z0-9-]','-', prompt.lower())[:30]
    ts = datetime.now().strftime("%m%d%H%M")
    proj_dir = f"{PROJECTS_DIR}/{slug}-{ts}"
    os.makedirs(proj_dir, exist_ok=True)

    fw_ctx = await asyncio.to_thread(Firewall.gather_context, prompt)
    system = load_master_prompt() + f"""

You are DOLOR3V autonomous production web builder.

Tools available:
{', '.join(TOOL_LIST)}

Project directory:
{proj_dir}

Firewall context:
{fw_ctx}


STRICT BUILD RULES:

1. You are building a real website.
2. NEVER create:
   - config.json
   - output.txt
   - output.typescript
   - package files
   - random files

3. ONLY create:
   index.html
   style.css
   script.js


4. Before writing files:
   - use web_search once
   - gather design inspiration


5. HTML requirements:
   - semantic HTML5
   - SEO meta tags
   - hero section
   - features
   - testimonials
   - FAQ
   - footer


6. CSS requirements:
   - custom color palette
   - responsive mobile design
   - animations
   - modern UI


7. JS requirements:
   - real interactions
   - no empty files


8. Use write_file for every file.

9. After completion reply ONLY:

◆DONE

"""
    msgs = [{"role": "system", "content": system}, {"role": "user", "content": prompt}]

    results = []
    parse_fail_count = 0
    response, provider = "", "groq"
    MAX_BUILD_STEPS = 20
    for _step in range(MAX_BUILD_STEPS):
        response, provider = await asyncio.to_thread(llm_route_quality, msgs)

        step_results = []
        decoder = json.JSONDecoder()
        idx = 0
        text_len = len(response)
        while idx < text_len:
            brace_idx = response.find("{", idx)
            if brace_idx == -1:
                break
            try:
                tc, end_idx = decoder.raw_decode(response, brace_idx)
            except json.JSONDecodeError:
                idx = brace_idx + 1
                continue
            idx = end_idx
            if isinstance(tc, dict) and "tool" not in tc:
                preceding = response[max(0, brace_idx-150):brace_idx].lower()
                inferred_tool = None
                for tname in TOOL_LIST:
                    if tname.lower() in preceding:
                        inferred_tool = tname
                        break
                if inferred_tool:
                    tc = {"tool": inferred_tool, "args": tc}
            if isinstance(tc, dict) and "tool" in tc:
                args = tc.get("args", {})
                if tc.get("tool") == "write_file" and not args.get("path") and not args.get("file_path"):
                    content_preview = str(args.get("content", ""))
                    if "<!DOCTYPE" in content_preview or "<html" in content_preview:
                        default_name = "index.html"
                    elif content_preview.strip().startswith("function") or "document." in content_preview[:200]:
                        default_name = "script.js"
                    elif "{" in content_preview[:100] and ":" in content_preview[:100] and "<" not in content_preview[:50]:
                        default_name = "style.css"
                    else:
                        default_name = "output.txt"
                    args["path"] = os.path.join(proj_dir, default_name)
                r = await asyncio.to_thread(retry_tool, execute_tool, tc["tool"], args)
                results.append({"tool": tc["tool"], "result": str(r)[:200]})
                step_results.append({"tool": tc["tool"], "result": str(r)[:1500]})

        if not step_results:
            parse_fail_count += 1
            if parse_fail_count > 2:
                break
            msgs.append({"role": "assistant", "content": response})
            msgs.append({"role": "user", "content": "I could not parse a valid tool call from that response. Respond with ONLY one JSON object in the exact shape: {\"tool\": \"<tool_name>\", \"args\": {...}}. No narration, no repeated objects, no other text. Valid tool names: " + ", ".join(TOOL_LIST)})
            continue

        qa_passed = any(
            r.get("tool") == "qa_review" and "passed" in str(r.get("result", "")).lower()
            for r in step_results
        )
        if qa_passed:
            response = "\u25c6DONE"
            break

        msgs.append({"role": "assistant", "content": response})
        msgs.append({"role": "user", "content": "Tool results:\n" + json.dumps(step_results)[:3000] + "\nPerform QA review before completion. Check design quality, responsiveness, completeness, and production readiness. Improve if needed before responding \u25c6DONE."})

    wrote_files = any(r.get("tool") == "write_file" for r in results)
    # If no files were written via tool calls, auto-extract code blocks and save them
    if not wrote_files:
        code_blocks = re.findall(r'```(\w+)?\n(.*?)```', response, re.DOTALL)
        if code_blocks:
            for i, (lang, code) in enumerate(code_blocks):
                ext_map = {
                    'html': 'index.html', 'css': 'style.css', 'js': 'script.js',
                    'javascript': 'script.js', 'python': 'main.py', 'py': 'main.py',
                    'sh': 'setup.sh', 'bash': 'setup.sh', 'json': 'config.json',
                    'yaml': 'config.yaml', 'yml': 'config.yaml', '': 'output.txt'
                }
                ext = lang.strip() if lang else 'html'
                fname = ext_map.get(ext, f'output.{ext}')
                fpath = os.path.join(proj_dir, fname)
                with open(fpath, 'w') as f:
                    f.write(code.strip())
                results.append({"tool": "write_file", "result": f"Saved {fname} ({len(code)} bytes)"})
                log(f"Auto-saved extracted code block: {fname}")
        else:
            html_match = re.search(r'(<!DOCTYPE html>.*?</html>|<html[^>]*>.*?</html>)', response, re.DOTALL | re.IGNORECASE)
            if html_match:
                extracted_html = html_match.group(1)
                fpath = os.path.join(proj_dir, 'index.html')
                with open(fpath, 'w') as f:
                    f.write(extracted_html)
                results.append({"tool": "write_file", "result": f"Saved index.html extracted from response ({len(extracted_html)} bytes)"})
                log("Auto-extracted HTML block from response and saved as index.html")
            else:
                log("No extractable HTML found in malformed response; nothing written")

    gh_url, surge_url = "", ""
    if GITHUB_TOKEN:
        try:
            gh_url = await asyncio.to_thread(GitHub.push, proj_dir, f"dolor3v-{slug}-{ts}")
        except:
            gh_url = "GitHub push failed – check GITHUB_TOKEN"
    try:
        surge_url_r, _ = await asyncio.to_thread(Surge.deploy, proj_dir)
        if surge_url_r: surge_url = surge_url_r
    except:
        pass

    return {
        "ok": True,
        "prompt": prompt,
        "project": proj_dir,
        "provider": provider,
        "tools_run": results,
        "github": gh_url,
        "surge": surge_url,
        "response": response[:500]
    }
AGENT_SYSTEM_PROMPT = (
    "You are DOLOR3V Agent, an autonomous coding/ops assistant with real tool access.\n\n"
    "Available tools: " + ", ".join(TOOL_LIST) + "\n\n"
    "To use a tool, respond with ONLY one JSON object on its own line and nothing else, in this exact shape:\n"
    '{"tool": "<tool_name>", "args": {"...": "..."}}\n\n'
    "After a tool result is returned to you, you may call another tool the same way, "
    "or give your final answer as plain text with no JSON and no markdown fences. "
    "Only call a tool when you actually need it; if you already know the answer, just answer directly.\n\n"
    "🔥 NEW: Use `project_index` to get an overview of the codebase, and `grep_code` to search for patterns. "
    "You can `spawn_agent` to handle subtasks in parallel. "
    "Safety: dangerous shell commands (rm -rf, git push --force, etc.) will be blocked unless DOLOR3V_SAFE_MODE=false."
)

def maybe_summarize(messages, max_chars=MAX_CONTEXT_CHARS):
    """Keep total payload under limit by keeping system prompt + last few messages only."""
    # Quick check: if already small enough, return as-is
    total = sum(len(str(m.get("content",""))) for m in messages)
    if total <= max_chars:
        return messages
    # Always keep system messages (first 2 usually)
    system_msgs = [m for m in messages if m["role"]=="system"][:2]
    # Keep the last 6 non-system messages
    other_msgs = [m for m in messages if m["role"]!="system"]
    keep = other_msgs[-6:] if len(other_msgs) > 6 else other_msgs
    # Insert a summary note
    summary = {"role":"system","content":"[Earlier conversation truncated to fit context window]"}
    return system_msgs + [summary] + keep

def agent_llm(messages):
    return llm_route(messages)

@app.post("/v1/agent")
async def agent(request: Request):
    body = await request.json()
    user_prompt = body.get("prompt", "")
    history = body.get("history", [])
    max_steps = body.get("max_steps", 6)

    # Automatic project context (Claude Code feature)
    proj_index = await asyncio.to_thread(project_index, ".")
    sys_content = AGENT_SYSTEM_PROMPT + "\n\n=== Current project context ===\n" + proj_index

    messages = [{"role":"system","content":sys_content}] + history + [{"role":"user","content":user_prompt}]
    steps = []
    final_text = ""
    provider = "none"

    for _ in range(max_steps):
        messages = maybe_summarize(messages)
        # Try primary LLM call, then fallback if it fails
        reply = None
        provider = "none"
        llm_error = ""
        for attempt in range(2):
            try:
                reply, provider = await asyncio.to_thread(agent_llm, messages)
                break
            except Exception as e:
                llm_error = str(e)
                log(f"Agent LLM attempt {attempt+1} failed: {e}")
                # On first failure, try forcing the next provider by removing Ollama from list
                if attempt == 0:
                    messages.insert(0, {"role":"system","content":"Previous LLM failed. Please use a different provider."})
        if reply is None:
            final_text = f"All LLM providers failed: {llm_error}"
            break

        line = reply.strip()
        if line.startswith("{") and '"tool"' in line:
            try:
                tc = json.loads(line)
                tool_name = tc.get("tool")
                tool_args = tc.get("args", {})
                tool_result = await asyncio.to_thread(retry_tool, execute_tool, tool_name, tool_args)
                steps.append({"tool": tool_name, "args": tool_args, "result": str(tool_result)[:500]})
                messages.append({"role":"assistant","content":line})
                messages.append({"role":"user","content":f"Tool result:\n{tool_result}"})
                continue
            except Exception:
                final_text = reply
                break
        # small delay to avoid Groq rate limits
        await asyncio.sleep(2)
        final_text = reply
        break
    else:
        final_text = "Max steps reached without a final answer."

    emit_event("agent", {"prompt": user_prompt[:100], "steps": len(steps)}, "mcp-agent")

    return {
        "response": final_text,
        "provider": provider,
        "steps": steps
    }

if __name__ == "__main__":
    print(f"""
      DOLOR3V MCP GATEWAY v6.0
  Port     : {PORT}
  Tools    : {len(TOOL_LIST)}
  Safe mode: {SAFE_MODE}
  GitHub   : {"✅" if GITHUB_TOKEN else "❌ set GITHUB_TOKEN"}
  Surge    : {"✅" if SURGE_TOKEN  else "❌ set SURGE_TOKEN"}
  HF       : {"✅" if HF_TOKEN     else "❌ set HF_TOKEN"}
  Groq     : {"✅" if GROQ_KEY     else "❌ set GROQ_API_KEY"}
""")
    uvicorn.run(app, host="0.0.0.0", port=PORT)

# ── Drive-backed storage (added for builder-app / admin-app) ──
from fastapi import UploadFile, File
from googleapiclient.discovery import build
from googleapiclient.http import MediaIoBaseUpload
from google.oauth2 import service_account
import io, json, os

DRIVE_FOLDER_ID = os.environ["DRIVE_FOLDER_ID"]
_creds = service_account.Credentials.from_service_account_info(
    json.loads(os.environ["GOOGLE_SERVICE_ACCOUNT_JSON"]),
    scopes=["https://www.googleapis.com/auth/drive"]
)
drive = build("drive", "v3", credentials=_creds)

@app.post("/v1/projects/{page_id}")
async def save_project(page_id: str, body: dict):
    fname = f"{page_id}.json"
    media = MediaIoBaseUpload(io.BytesIO(json.dumps(body).encode()), mimetype="application/json")
    existing = drive.files().list(q=f"name='{fname}' and '{DRIVE_FOLDER_ID}' in parents").execute()
    if existing["files"]:
        drive.files().update(fileId=existing["files"][0]["id"], media_body=media).execute()
    else:
        drive.files().create(body={"name": fname, "parents": [DRIVE_FOLDER_ID]}, media_body=media).execute()
    return {"status": "ok"}

@app.get("/v1/projects/{page_id}")
async def load_project(page_id: str):
    fname = f"{page_id}.json"
    res = drive.files().list(q=f"name='{fname}' and '{DRIVE_FOLDER_ID}' in parents").execute()
    if not res["files"]:
        return {"data": {}}
    content = drive.files().get_media(fileId=res["files"][0]["id"]).execute()
    return {"data": json.loads(content)}

@app.post("/v1/assets/upload")
async def upload_asset(files: list[UploadFile] = File(...)):
    urls = []
    for f in files:
        media = MediaIoBaseUpload(io.BytesIO(await f.read()), mimetype=f.content_type)
        created = drive.files().create(
            body={"name": f.filename, "parents": [DRIVE_FOLDER_ID]}, media_body=media, fields="id"
        ).execute()
        drive.permissions().create(fileId=created["id"], body={"role": "reader", "type": "anyone"}).execute()
        urls.append(f"https://drive.google.com/uc?id={created['id']}")
    return {"data": [{"src": u} for u in urls]}