"""
signalmesh_gridstack.py — local SignalMesh gridstack for a project's own state.

The mesh, locally: chart the whole project directory ONCE into a nested,
coordinate-addressable gridstack, then every lookup is instant + cheap + synced.

Nested containers (Adam's model):
    kyklos > kyklos_apps > signal_mesh > signalmesh_processes > <grid>
Here the master container is the project itself:
    all4aj > grids{files, symbols, services, config, keywords} > rows × cols

Every cell is addressable by coordinate, e.g.  all4aj.files.12.path
Backed by SQLite + FTS5 so "what knows about X" is sub-millisecond, offline.

CLI:
    python3 signalmesh_gridstack.py chart [root]
    python3 signalmesh_gridstack.py q "bitcoin price"
    python3 signalmesh_gridstack.py addr all4aj.files.12
    python3 signalmesh_gridstack.py sync
    python3 signalmesh_gridstack.py stats
    python3 signalmesh_gridstack.py caps          # index callable capabilities (routes/CLI) + list

Thought-log (chat runs through the mesh — everything is a log):
    python3 signalmesh_gridstack.py chat   "<human input>"   # log human turn → cell
    python3 signalmesh_gridstack.py recall                   # check the log
    python3 signalmesh_gridstack.py think  "<thought>"       # log thought + classify + traces
    python3 signalmesh_gridstack.py trace  "<thought>"       # trace, no log
    python3 signalmesh_gridstack.py classify "<intent>"      # NEW / PARTIAL / EXISTING / SPEC'D

Agentic / coordination (drop-in, works in any charted repo):
    python3 signalmesh_gridstack.py events [since_ts]        # cell+thought event feed (done flags)
    python3 signalmesh_gridstack.py merge <src/> <tgt/>      # symbol collisions + entry points between two subtrees
    python3 signalmesh_gridstack.py brain "<prompt>"         # ask the local native LLM (optional: llama_cpp + gguf)
    # library API for host apps: notify(id,payload) / await_ping(id,timeout) — message-id ping bus
"""
from __future__ import annotations

import hashlib
import json
import os
import re
import sqlite3
import sys
import time
from pathlib import Path

# Root/DB are env-overridable so the gridstack can chart ANY repo (the submission
# model): SIGNALMESH_ROOT points at the target repo, SIGNALMESH_DB at its own DB dir.
ROOT = Path(os.environ.get("SIGNALMESH_ROOT", Path(__file__).parent)).resolve()
DB_DIR = Path(os.environ.get("SIGNALMESH_DB", ROOT / ".signalmesh"))
DB = DB_DIR / "gridstack.db"

CONTAINER = os.environ.get("SIGNALMESH_CONTAINER",
                           re.sub(r"[^a-z0-9]+", "_", ROOT.name.lower()).strip("_") or "project")

# Skip the heavy / non-source stuff so a chart stays fast.
_SKIP_DIRS = {".git", "__pycache__", "node_modules", "_archive", "_unzipped",
              "models", "mavos_teasers", "joomla_videos", "SignalMesh_Videos",
              ".lmstudio", "PBC-models", "tab_frames", "screenshots",
              "dist", "build"}  # build artifacts — never index the packaged bundle
_SKIP_DIR_PREFIXES = (".venv", "venv", "Mavos-replica")
_CODE_EXT = {".py", ".js", ".ts", ".tsx", ".jsx", ".sh", ".sql", ".go", ".rs", ".c", ".cpp"}
_TEXT_EXT = {".md", ".yaml", ".yml", ".json", ".html", ".css", ".txt", ".env", ".toml", ".cfg"}
_MAX_BYTES = 256_000
_SECRET_RE = re.compile(r"(key|token|secret|password|passwd|api[_-]?key)", re.I)
_PY_DEF_RE = re.compile(r"^\s*(?:async\s+)?(def|class)\s+([A-Za-z_]\w*)", re.M)

# Multi-language symbol extraction — keywording must work in ANY language, not just
# Python. Each ext maps to ordered (kind, regex) probes; first capture group = name.
_JS_SYM = [
    ("class",     re.compile(r"^\s*(?:export\s+(?:default\s+)?)?(?:abstract\s+)?class\s+([A-Za-z_$][\w$]*)", re.M)),
    ("function",  re.compile(r"^\s*(?:export\s+(?:default\s+)?)?(?:async\s+)?function\*?\s+([A-Za-z_$][\w$]*)", re.M)),
    ("const",     re.compile(r"^\s*(?:export\s+)?(?:const|let)\s+([A-Za-z_$][\w$]*)\s*(?::[^=]+)?=\s*(?:async\s*)?(?:\([^)]*\)|[A-Za-z_$][\w$]*)\s*=>", re.M)),
    ("component", re.compile(r"^\s*(?:export\s+)?(?:default\s+)?function\s+([A-Z][\w$]*)", re.M)),
    ("hook",      re.compile(r"^\s*(?:export\s+)?const\s+(use[A-Z][\w$]*)\s*=", re.M)),
    ("type",      re.compile(r"^\s*(?:export\s+)?(?:type|interface)\s+([A-Za-z_$][\w$]*)", re.M)),
    ("enum",      re.compile(r"^\s*(?:export\s+)?enum\s+([A-Za-z_$][\w$]*)", re.M)),
]
_GO_SYM = [
    ("func",   re.compile(r"^\s*func\s+(?:\([^)]*\)\s*)?([A-Za-z_]\w*)", re.M)),
    ("type",   re.compile(r"^\s*type\s+([A-Za-z_]\w*)\s+(?:struct|interface)", re.M)),
]
_RS_SYM = [
    ("fn",     re.compile(r"^\s*(?:pub\s+)?(?:async\s+)?fn\s+([A-Za-z_]\w*)", re.M)),
    ("struct", re.compile(r"^\s*(?:pub\s+)?struct\s+([A-Za-z_]\w*)", re.M)),
    ("enum",   re.compile(r"^\s*(?:pub\s+)?enum\s+([A-Za-z_]\w*)", re.M)),
    ("trait",  re.compile(r"^\s*(?:pub\s+)?trait\s+([A-Za-z_]\w*)", re.M)),
]
_C_SYM = [
    ("func",   re.compile(r"^[A-Za-z_][\w\s\*]*?\s+([A-Za-z_]\w*)\s*\([^;{]*\)\s*\{", re.M)),
    ("struct", re.compile(r"^\s*(?:typedef\s+)?struct\s+([A-Za-z_]\w*)", re.M)),
]
_SYM_BY_EXT: dict[str, list] = {
    ".py": [("def", _PY_DEF_RE), ("class", _PY_DEF_RE)],   # py handled specially below
    ".js": _JS_SYM, ".jsx": _JS_SYM, ".ts": _JS_SYM, ".tsx": _JS_SYM, ".mjs": _JS_SYM,
    ".go": _GO_SYM, ".rs": _RS_SYM,
    ".c": _C_SYM, ".cpp": _C_SYM, ".cc": _C_SYM, ".h": _C_SYM, ".hpp": _C_SYM,
}


def _symbols(ext: str, text: str) -> list[tuple[str, str]]:
    """Return ordered (kind, name) symbol pairs for ANY supported language.
    Python keeps its (def|class) regex; everything else uses per-ext probes."""
    if ext == ".py":
        return [(k, n) for k, n in _PY_DEF_RE.findall(text)]
    out, seen = [], set()
    for kind, rx in _SYM_BY_EXT.get(ext, []):
        for n in rx.findall(text):
            name = n if isinstance(n, str) else n[0]
            if name and name not in seen:
                seen.add(name); out.append((kind, name))
    return out


def _connect() -> sqlite3.Connection:
    DB_DIR.mkdir(exist_ok=True)
    con = sqlite3.connect(DB)
    con.row_factory = sqlite3.Row
    return con


def _init(con: sqlite3.Connection) -> None:
    con.executescript("""
    CREATE TABLE IF NOT EXISTS grids (
        name TEXT PRIMARY KEY, container TEXT, parent TEXT, rows INT DEFAULT 0
    );
    CREATE TABLE IF NOT EXISTS cells (
        coord TEXT PRIMARY KEY,   -- container.grid.row
        grid  TEXT, row INT,
        name  TEXT, kind TEXT, value TEXT, mtime REAL, chash TEXT
    );
    CREATE INDEX IF NOT EXISTS cells_name ON cells(grid, name);
    CREATE INDEX IF NOT EXISTS cells_value ON cells(grid, value);
    -- per-cell changelog: every create/modify/delete the grid-god observes
    CREATE TABLE IF NOT EXISTS changelog (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        coord TEXT, ts REAL, change TEXT, detail TEXT
    );
    CREATE INDEX IF NOT EXISTS changelog_coord ON changelog(coord);
    CREATE VIRTUAL TABLE IF NOT EXISTS cell_fts USING fts5(
        coord UNINDEXED, grid, name, kind, value,
        content='cells', content_rowid='rowid'
    );
    CREATE TRIGGER IF NOT EXISTS cells_ai AFTER INSERT ON cells BEGIN
      INSERT INTO cell_fts(rowid,coord,grid,name,kind,value)
      VALUES (new.rowid,new.coord,new.grid,new.name,new.kind,new.value);
    END;
    CREATE TRIGGER IF NOT EXISTS cells_ad AFTER DELETE ON cells BEGIN
      INSERT INTO cell_fts(cell_fts,rowid,coord,grid,name,kind,value)
      VALUES('delete',old.rowid,old.coord,old.grid,old.name,old.kind,old.value);
    END;
    """)
    # migration: add chash to a pre-existing DB
    try:
        con.execute("ALTER TABLE cells ADD COLUMN chash TEXT")
    except sqlite3.OperationalError:
        pass
    con.commit()


def _walk(root: Path):
    # ponytail: per-repo skip, zero global coupling
    _extra_skip = set()
    _skip_file = root / ".signalmesh_skip"
    if _skip_file.exists():
        _extra_skip = {l.strip() for l in _skip_file.read_text().splitlines() if l.strip()}
    for dirpath, dirnames, filenames in os.walk(root):
        dirnames[:] = [d for d in dirnames
                       if d not in _SKIP_DIRS and d not in _extra_skip
                       and not any(d.startswith(p) for p in _SKIP_DIR_PREFIXES)]
        for fn in filenames:
            p = Path(dirpath) / fn
            ext = p.suffix.lower()
            if ext not in _CODE_EXT and ext not in _TEXT_EXT:
                continue
            try:
                if p.stat().st_size > _MAX_BYTES:
                    continue
            except OSError:
                continue
            yield p, ext


def _row(con, grid, row, name, kind, value, mtime):
    coord = f"{CONTAINER}.{grid}.{row}"
    con.execute(
        "INSERT OR REPLACE INTO cells(coord,grid,row,name,kind,value,mtime) VALUES(?,?,?,?,?,?,?)",
        (coord, grid, row, name, kind, value[:4000], mtime))
    return coord


def chart(root: Path = ROOT) -> dict:
    t0 = time.time()
    con = _connect(); _init(con)
    # Re-chart rebuilds the code grids but NEVER the thought-log or the capability
    # registry — the conversation AND what the system can do are mesh state too.
    # cells_ad keeps cell_fts in sync for the rows we drop.
    con.execute("DELETE FROM cells WHERE grid NOT IN ('thoughts','capabilities')")

    counts = {"files": 0, "symbols": 0, "services": 0, "config": 0, "keywords": 0}
    kw_index: dict[str, set] = {}

    for p, ext in _walk(root):
        rel = str(p.relative_to(root))
        try:
            text = p.read_text(errors="ignore")
        except Exception:
            continue
        mtime = p.stat().st_mtime
        lines = text.count("\n") + 1

        # files grid
        fr = counts["files"]
        syms = _symbols(ext, text)
        defs = [f"{k} {n}" for k, n in syms]
        summary = (defs[:8] and " · ".join(defs[:8])) or text.strip().split("\n", 1)[0][:120]
        _row(con, "files", fr, rel, ext.lstrip("."), f"{lines} lines | {summary}", mtime)
        counts["files"] += 1

        # symbols grid (any-language defs/classes/funcs/types)
        for k, n in syms:
            _row(con, "symbols", counts["symbols"], n, k, rel, mtime)
            counts["symbols"] += 1

        # config grid (json/env/yaml keys; mask secrets)
        if ext in (".json", ".env", ".yaml", ".yml", ".toml", ".cfg"):
            for m in re.findall(r'["\']?([A-Za-z_][\w\-]{2,40})["\']?\s*[:=]', text)[:60]:
                masked = "***" if _SECRET_RE.search(m) else m
                _row(con, "config", counts["config"], m, "secret" if masked == "***" else "key",
                     f"{rel}", mtime)
                counts["config"] += 1

        # keyword harvest → coordinate-bound connections to files
        for w in set(re.split(r"[^a-z0-9]+", rel.lower() + " " + summary.lower())):
            if len(w) >= 4 and not w.isdigit():
                kw_index.setdefault(w, set()).add(rel)

    # keywords grid — rows = keyword, value = the file coords it connects across
    for i, (w, files) in enumerate(sorted(kw_index.items(), key=lambda x: -len(x[1]))[:4000]):
        _row(con, "keywords", i, w, "keyword", " ".join(sorted(files)[:20]), time.time())
        counts["keywords"] += 1

    # running services (live ports) — the "listening things"
    services = _scan_services()
    for i, s in enumerate(services):
        _row(con, "services", i, s["name"], "service", json.dumps(s), time.time())
        counts["services"] += 1

    for g, n in counts.items():
        con.execute("INSERT OR REPLACE INTO grids(name,container,parent,rows) VALUES(?,?,?,?)",
                    (g, CONTAINER, "signalmesh_processes", n))
    con.commit(); con.close()
    counts["_seconds"] = round(time.time() - t0, 2)
    return counts


def _scan_services() -> list[dict]:
    import subprocess
    out = []
    try:
        r = subprocess.run(["lsof", "-nP", "-iTCP", "-sTCP:LISTEN"],
                           capture_output=True, text=True, timeout=8).stdout
        for line in r.splitlines()[1:]:
            parts = line.split()
            if len(parts) >= 9:
                out.append({"name": parts[0], "pid": parts[1], "addr": parts[8]})
    except Exception:
        pass
    # dedupe by addr
    seen, uniq = set(), []
    for s in out:
        if s["addr"] not in seen:
            seen.add(s["addr"]); uniq.append(s)
    return uniq[:40]


# ── Grid-god: incremental sync + per-cell changelog (never goes stale) ─────────

def _hash(text: str) -> str:
    return hashlib.sha256(text.encode("utf-8", "ignore")).hexdigest()[:16]


def _log(con, coord: str, change: str, detail: str = "") -> None:
    con.execute("INSERT INTO changelog(coord,ts,change,detail) VALUES(?,?,?,?)",
                (coord, time.time(), change, detail[:200]))


def _index_file(con, rel: str, ext: str, text: str, row: int, chash: str, mtime: float) -> str:
    """(Re)build the files + symbols + config cells for ONE file. Returns its coord."""
    lines = text.count("\n") + 1
    syms = _symbols(ext, text)
    defs = [f"{k} {n}" for k, n in syms]
    summary = (defs[:8] and " · ".join(defs[:8])) or text.strip().split("\n", 1)[0][:120]
    fcoord = f"{CONTAINER}.files.{row}"
    con.execute("DELETE FROM cells WHERE coord=?", (fcoord,))
    con.execute("INSERT INTO cells(coord,grid,row,name,kind,value,mtime,chash) VALUES(?,?,?,?,?,?,?,?)",
                (fcoord, "files", row, rel, ext.lstrip("."), f"{lines} lines | {summary}", mtime, chash))
    # refresh this file's derived cells (keyed by path so re-sync is idempotent)
    con.execute("DELETE FROM cells WHERE grid IN ('symbols','config') AND value=?", (rel,))
    for i, (k, n) in enumerate(syms):
        con.execute("INSERT OR REPLACE INTO cells(coord,grid,row,name,kind,value,mtime,chash) "
                    "VALUES(?,?,?,?,?,?,?,?)",
                    (f"{CONTAINER}.symbols.{rel}#{i}", "symbols", i, n, k, rel, mtime, ""))
    if ext in (".json", ".env", ".yaml", ".yml", ".toml", ".cfg"):
        for i, m in enumerate(re.findall(r'["\']?([A-Za-z_][\w\-]{2,40})["\']?\s*[:=]', text)[:60]):
            masked = "***" if _SECRET_RE.search(m) else m
            con.execute("INSERT OR REPLACE INTO cells(coord,grid,row,name,kind,value,mtime,chash) "
                        "VALUES(?,?,?,?,?,?,?,?)",
                        (f"{CONTAINER}.config.{rel}#{i}", "config", i, m,
                         "secret" if masked == "***" else "key", rel, mtime, ""))
    return fcoord


def sync(root: Path = ROOT) -> dict:
    """Incremental: touch only changed/new/deleted files; log every change per cell.
    mtime fast-path means an unchanged repo costs just a stat() sweep."""
    con = _connect(); _init(con)
    ex = {r["name"]: (r["coord"], r["chash"], r["row"], r["mtime"] or 0)
          for r in con.execute("SELECT name,coord,chash,row,mtime FROM cells WHERE grid='files'")}
    maxrow = con.execute("SELECT COALESCE(MAX(row),-1) m FROM cells WHERE grid='files'").fetchone()["m"]
    on_disk = set()
    ch = {"created": 0, "modified": 0, "deleted": 0, "unchanged": 0}
    dirty: list[str] = []   # files needing re-embed (created+modified)
    gone: list[str] = []    # files removed

    for p, ext in _walk(root):
        rel = str(p.relative_to(root)); on_disk.add(rel)
        try:
            st_mtime = p.stat().st_mtime
        except OSError:
            continue
        if rel in ex and abs(ex[rel][3] - st_mtime) < 1e-6:
            ch["unchanged"] += 1; continue                 # mtime unchanged → no read
        try:
            text = p.read_text(errors="ignore")
        except Exception:
            continue
        h = _hash(text)
        if rel not in ex:
            maxrow += 1; row = maxrow; change = "created"
        elif ex[rel][1] != h:
            row = ex[rel][2]; change = "modified"
        else:
            ch["unchanged"] += 1; continue                 # touched but content identical
        coord = _index_file(con, rel, ext, text, row, h, st_mtime)
        _log(con, coord, change, rel)
        ch[change] += 1; dirty.append(rel)

    for rel, (coord, _h, _r, _m) in ex.items():
        if rel not in on_disk:
            con.execute("DELETE FROM cells WHERE grid='files' AND name=?", (rel,))
            con.execute("DELETE FROM cells WHERE grid IN ('symbols','config') AND value=?", (rel,))
            _log(con, coord, "deleted", rel); ch["deleted"] += 1; gone.append(rel)

    con.commit(); con.close()
    ch["_dirty"], ch["_gone"] = dirty, gone
    return ch


def changelog_for(coord: str = "", limit: int = 20) -> list[dict]:
    con = _connect()
    try:
        if coord:
            rows = con.execute("SELECT coord,ts,change,detail FROM changelog WHERE coord=? "
                               "ORDER BY id DESC LIMIT ?", (coord, limit)).fetchall()
        else:
            rows = con.execute("SELECT coord,ts,change,detail FROM changelog "
                               "ORDER BY id DESC LIMIT ?", (limit,)).fetchall()
        return [dict(r) for r in rows]
    finally:
        con.close()


# A logged thought is a "completion signal" when it declares the work finished.
_DONE_RE = re.compile(
    r"\b(done|complete[d]?|finish(ed)?|fulfilled|succeed(ed)?|success(ful)?|"
    r"ready|shipped|resolved|implemented|passing|verified)\b", re.I)


def is_completion(text: str) -> bool:
    return bool(_DONE_RE.search(text or ""))


def events_since(ts: float = 0.0, limit: int = 200) -> dict:
    """The notify feed: every cell event (file change OR logged thought) after `ts`,
    oldest→newest, so a waiting consumer can poll with a cursor. Each event carries a
    `done` flag when a logged thought declares the task complete — that's the signal
    the user-facing model waits on before checking the work and reporting back."""
    con = _connect()
    try:
        rows = con.execute(
            "SELECT coord,ts,change,detail FROM changelog WHERE ts > ? "
            "ORDER BY id ASC LIMIT ?", (ts, limit)).fetchall()
        tail = con.execute("SELECT MAX(ts) FROM changelog").fetchone()[0] or ts
    finally:
        con.close()
    events, latest, completions = [], tail, []
    for r in rows:
        e = dict(r)
        e["done"] = e["change"] in ("thought", "action", "human") and is_completion(e.get("detail", ""))
        e["kind"] = "thought" if ".thoughts." in e["coord"] else "cell"
        events.append(e)
        if e["done"]:
            completions.append(e)
    return {"events": events, "latest_ts": latest, "count": len(events),
            "completions": completions, "done": bool(completions)}


# ── Merge analysis: mesh-derive integration seams between two stacks ───────────
# Pure-DB (no extra deps). Drop the file into any monorepo and find where two
# subtrees collide / connect: symbol-name collisions + structural entry points.
_MERGE_NOISE = {"__init__", "__repr__", "__str__", "__enter__", "__exit__",
                "main", "setup", "teardown", "run", "get", "post", "cleanup"}


def _symbols_under(con, prefix: str) -> dict:
    out: dict = {}
    for name, value, row in con.execute(
            "SELECT name,value,row FROM cells WHERE grid='symbols' AND value LIKE ?",
            (prefix + "%",)):
        if not name or len(name) < 4 or name in _MERGE_NOISE or name.startswith("test_"):
            continue
        out.setdefault(name, []).append((value, row or 0))
    return out


def merge_analyze(source_prefix: str, target_prefix: str, limit: int = 60) -> dict:
    """Symbol collisions + entry points between two path prefixes in the charted repo."""
    con = _connect()
    try:
        src = _symbols_under(con, source_prefix)
        tgt = _symbols_under(con, target_prefix)
    finally:
        con.close()
    collisions = [{"symbol": n, "source": src[n][:4], "target": tgt[n][:4]}
                  for n in sorted(set(src) & set(tgt))]
    anchors = ("create_app", "create_provider", "registry", "Provider",
               "router", "include_router", "FastAPI", "build_")
    entry = []
    for n in sorted(set(src) | set(tgt)):
        if any(a in n for a in anchors):
            side = "both" if n in src and n in tgt else ("source" if n in src else "target")
            files = [f for f, _ in (src.get(n, []) + tgt.get(n, []))[:3]]
            entry.append({"symbol": n, "side": side, "files": files})
    return {"source_prefix": source_prefix, "target_prefix": target_prefix,
            "source_symbols": len(src), "target_symbols": len(tgt),
            "collisions": collisions[:limit], "collision_count": len(collisions),
            "entry_points": entry[:limit]}


# ── Message-id ping bus: agent coordination through the mesh (thread-based) ────
# A host app subscribes a chat/message id; a worker pings it on completion. No
# asyncio dependency, so it works embedded or standalone.
import threading as _threading
_SUBS: dict = {}
_PEND: dict = {}
_NLOCK = _threading.Lock()


def notify(message_id: str, payload: dict = None) -> dict:
    """Ping every waiter on `message_id` (buffers if none subscribed yet)."""
    msg = {"message_id": message_id, "ts": time.time(), **(payload or {})}
    with _NLOCK:
        waiters = list(_SUBS.get(message_id, []))
        if not waiters:
            _PEND[message_id] = msg
        for ev, box in waiters:
            box.append(msg)
            ev.set()
    return {"published": True, "subscribers": len(waiters), "message_id": message_id}


def await_ping(message_id: str, timeout: float = 25.0) -> dict:
    """Block until a ping for `message_id` (or timeout → {pending:true})."""
    with _NLOCK:
        if message_id in _PEND:
            return _PEND.pop(message_id)
        ev = _threading.Event(); box: list = []
        _SUBS.setdefault(message_id, []).append((ev, box))
    got = ev.wait(timeout)
    with _NLOCK:
        lst = _SUBS.get(message_id, [])
        lst[:] = [(e, b) for (e, b) in lst if e is not ev]
        if not lst:
            _SUBS.pop(message_id, None)
    return box[0] if (got and box) else {"pending": True, "message_id": message_id}


# ── Native brain (optional): a local, offline LLM the mesh can think with ──────
# Zero hard dependency. If llama_cpp + a gguf are available, the mesh gains a
# native reasoning engine (for `brain`/`ask`, fallback, and intent backing).
# Model: $QWEN_MODEL_PATH → ~/.cache/open-codex → HF download. Degrades cleanly.
_BRAIN = None
_BRAIN_FILE = os.environ.get("MESH_BRAIN_FILE", "Qwen3-4B-Q4_K_M.gguf")
_BRAIN_REPO = os.environ.get("MESH_BRAIN_REPO", "Qwen/Qwen3-4B-GGUF")
_THINK_RE = re.compile(r"<think>.*?</think>\s*", re.DOTALL)


def _brain_model_path():
    p = os.environ.get("QWEN_MODEL_PATH") or os.environ.get("MESH_BRAIN_PATH")
    if p and os.path.exists(p):
        return p
    cache = os.path.expanduser("~/.cache/open-codex")
    local = os.path.join(cache, _BRAIN_FILE)
    if os.path.exists(local):
        return local
    try:
        from huggingface_hub import hf_hub_download
        return hf_hub_download(repo_id=_BRAIN_REPO, filename=_BRAIN_FILE, local_dir=cache)
    except Exception:
        return None


def brain_available() -> bool:
    try:
        import llama_cpp  # noqa: F401
    except Exception:
        return False
    return _brain_model_path() is not None


def _strip_think(t: str) -> str:
    t = _THINK_RE.sub("", t)
    if "<think>" in t and "</think>" not in t:
        t = t.split("<think>", 1)[0]
    return t.strip()


def _chatml(messages: list) -> str:
    s = "".join(f"<|im_start|>{m['role']}\n{m['content']}<|im_end|>\n" for m in messages)
    return s + "<|im_start|>assistant\n"


def _brain():
    global _BRAIN
    if _BRAIN is None:
        from llama_cpp import Llama
        mp = _brain_model_path()
        if not mp:
            raise RuntimeError("no gguf model — set QWEN_MODEL_PATH or install huggingface_hub")
        _BRAIN = Llama(model_path=mp, n_ctx=4096, n_gpu_layers=-1, verbose=False)
    return _BRAIN


def brain_ask(prompt: str, system: str = "", max_tokens: int = 512, think: bool = False) -> str:
    """Ask the local native brain; map-reduce-condenses oversized prompts to fit ctx."""
    llm = _brain()
    def ntok(t):
        try:
            return len(llm.tokenize(t.encode("utf-8"), add_bos=False, special=False))
        except Exception:
            return max(1, len(t) // 4)
    budget = 4096 - max_tokens - 128
    if ntok(prompt) > budget:
        win = max(2000, budget * 3)
        parts = []
        for i in range(0, len(prompt), win):
            p = _chatml([{"role": "system", "content": "Condense; keep key facts, names, code identifiers. /no_think"},
                         {"role": "user", "content": prompt[i:i + win]}])
            r = llm(prompt=p, max_tokens=max(96, budget // 6), temperature=0.1, stream=False)
            parts.append(_strip_think(r["choices"][0]["text"]))
        prompt = "\n".join(parts)
        
    # NATIVE INTELLIGENCE AMPLIFICATION
    if not system:
        system = (
            "You are an elite Senior Staff Engineer embedded in the SignalMesh gridstack. "
            "You provide highly accurate, strictly formatted technical answers based ONLY on the provided context coordinates. "
            "RULES:\n"
            "1. NEVER use conversational filler (e.g. 'Here is the code', 'I hope this helps', 'As an AI').\n"
            "2. Be extremely terse. Use markdown heavily.\n"
            "3. Ground all claims in the exact [coord] references provided.\n"
            "4. If you don't know, output: 'ERR: insufficient_mesh_context'."
        )
        
    sysc = (system + (" /no_think" if not think else "")).strip()
    
    # Inject few-shot example for structural grounding
    msgs = [{"role": "system", "content": sysc}]
    msgs.append({"role": "user", "content": "[coord: dummy.file.1] def process(): pass\nQuestion: What does this file do?"})
    msgs.append({"role": "assistant", "content": "The coordinate `[dummy.file.1]` contains a `process()` function."})
    msgs.append({"role": "user", "content": prompt})
    
    # Lower temperature for deterministic, hallucination-free generation
    out = llm(prompt=_chatml(msgs), max_tokens=max_tokens, temperature=0.05, stream=False)
    return _strip_think(out["choices"][0]["text"])


def grid_god(interval: float = 4.0, root: Path = ROOT, semantic: bool = True) -> None:
    """Always-on warden: re-sync changed files every `interval`s so the gridstack
    never goes stale; every change is logged per-cell AND the semantic index is
    incrementally re-embedded for changed files (so `tune` stays fresh too)."""
    print(f"[grid-god] watching {root} every {interval}s — ctrl-C to stop", flush=True)
    while True:
        try:
            ch = sync(root)
            if ch["created"] + ch["modified"] + ch["deleted"]:
                msg = f"+{ch['created']} ~{ch['modified']} -{ch['deleted']}"
                if semantic and (ch["_dirty"] or ch["_gone"]) and VEC_NPY.exists():
                    try:
                        es = embed_sync(ch["_dirty"], ch["_gone"])
                        if es.get("refreshed") or es.get("removed_files"):
                            msg += f" | re-embedded {es.get('refreshed', 0)}"
                    except Exception as e:
                        msg += f" | embed-skip ({str(e)[:40]})"
                print(f"[grid-god] {time.strftime('%H:%M:%S')}  {msg}", flush=True)
        except Exception as e:
            print(f"[grid-god] sync error: {e}", flush=True)
        time.sleep(interval)


def q(query: str, limit: int = 12) -> list[dict]:
    con = _connect()
    try:
        fts = " OR ".join(re.findall(r"[A-Za-z0-9]+", query)) or query
        rows = con.execute(
            "SELECT coord,grid,name,kind,value FROM cell_fts WHERE cell_fts MATCH ? "
            "ORDER BY rank LIMIT ?", (fts, limit)).fetchall()
        return [dict(r) for r in rows]
    finally:
        con.close()


# ── Semantic layer — local embeddings (nomic via LM Studio), no egress ─────────

VEC_NPY = DB_DIR / "chunk_vecs.npy"
VEC_IDS = DB_DIR / "chunk_meta.json"   # [{coord,file,snippet}] aligned to the matrix rows
_LM = os.environ.get("LMSTUDIO_HOST", "http://localhost:1234").rstrip("/")
_EMBED_MODEL = os.environ.get("MAVOS_EMBED_MODEL", "text-embedding-nomic-embed-text-v1.5")
# Vendored/third-party trees to skip on the first (fast) semantic index.
_VENDOR = ("apax-main/", "ClawTeam-main/", "AGI_SoulMatrix_Clean/", "turboLazrDough/",
           "5ble/", "money4band/", "beacon/", "OpenMontage/", "infini-dset/",
           "video_explainer-main/", "skip-main/", "2ppc-main/", "seo/",
           "Mavos-replica", "TalkingHead-main/")


def _embed(texts: list[str]) -> list[list[float]]:
    import urllib.request
    body = json.dumps({"model": _EMBED_MODEL, "input": texts}).encode()
    req = urllib.request.Request(_LM + "/v1/embeddings", data=body,
                                 headers={"Content-Type": "application/json"})
    with urllib.request.urlopen(req, timeout=120) as r:
        return [d["embedding"] for d in json.load(r)["data"]]


def _chunk(text: str, size: int = None, overlap: int = 120, cap: int = None) -> list[str]:
    # Env-tunable: MAVOS_CHUNK_SIZE / MAVOS_CHUNK_CAP. Default = 1 rich head chunk
    # per file (fast first index); raise cap for deeper coverage.
    size = size or int(os.environ.get("MAVOS_CHUNK_SIZE", "1600"))
    cap = cap if cap is not None else int(os.environ.get("MAVOS_CHUNK_CAP", "1"))
    text = text.strip()
    if not text:
        return []
    out, i = [], 0
    while i < len(text) and len(out) < cap:
        out.append(text[i:i + size])
        i += size - overlap
    return out


def embed_index(include_vendor: bool = False, batch: int = 64) -> dict:
    """Embed real file CONTENT chunks locally (nomic + task prefixes). 1-time, no egress."""
    import numpy as np
    con = _connect()
    try:
        rows = con.execute("SELECT name FROM cells WHERE grid='files'").fetchall()
    finally:
        con.close()
    files = [r["name"] for r in rows
             if include_vendor or not any(r["name"].startswith(v) for v in _VENDOR)]

    t0 = time.time()
    meta: list[dict] = []          # {coord_index_back to file}
    docs: list[str] = []
    for rel in files:
        try:
            content = (ROOT / rel).read_text(errors="ignore")
        except Exception:
            continue
        for ci, ch in enumerate(_chunk(content)):
            meta.append({"file": rel, "snippet": re.sub(r"\s+", " ", ch[:160]).strip()})
            docs.append("search_document: " + ch)

    DB_DIR.mkdir(exist_ok=True)
    vecs: list = []
    for i in range(0, len(docs), batch):
        vecs += _embed(docs[i:i + batch])
    import numpy as np
    mat = np.asarray(vecs, dtype="float32")
    mat /= (np.linalg.norm(mat, axis=1, keepdims=True) + 1e-9)
    np.save(VEC_NPY, mat)
    VEC_IDS.write_text(json.dumps(meta))
    return {"files": len(files), "chunks": len(meta),
            "dim": int(mat.shape[1]) if len(meta) else 0,
            "seconds": round(time.time() - t0, 1), "vendor_included": include_vendor}


def embed_sync(dirty: list[str], gone: list[str], batch: int = 64) -> dict:
    """Incrementally refresh the semantic index: drop changed/removed files' chunks
    and re-embed only the changed ones. Keeps `tune` never-stale, cheaply."""
    import numpy as np
    if not VEC_NPY.exists():
        return {"skipped": "no semantic index yet"}
    remove = {f for f in (set(dirty) | set(gone))
              if not any(f.startswith(v) for v in _VENDOR)}
    refresh = [f for f in dirty
               if not any(f.startswith(v) for v in _VENDOR) and (ROOT / f).exists()]
    if not remove and not refresh:
        return {"refreshed": 0, "removed": 0}

    mat = np.load(VEC_NPY)
    meta = json.loads(VEC_IDS.read_text())
    keep = [i for i, m in enumerate(meta) if m["file"] not in remove]
    if len(keep) != len(meta):
        mat = mat[keep]
        meta = [meta[i] for i in keep]

    new_docs, new_meta = [], []
    for rel in refresh:
        try:
            content = (ROOT / rel).read_text(errors="ignore")
        except Exception:
            continue
        for ch in _chunk(content):
            new_meta.append({"file": rel, "snippet": re.sub(r"\s+", " ", ch[:160]).strip()})
            new_docs.append("search_document: " + ch)
    if new_docs:
        vecs = []
        for i in range(0, len(new_docs), batch):
            vecs += _embed(new_docs[i:i + batch])
        nv = np.asarray(vecs, dtype="float32")
        nv /= (np.linalg.norm(nv, axis=1, keepdims=True) + 1e-9)
        mat = np.vstack([mat, nv]) if mat.size else nv
        meta += new_meta

    np.save(VEC_NPY, mat)
    VEC_IDS.write_text(json.dumps(meta))
    return {"refreshed": len(refresh), "removed_files": len(remove), "chunks": len(meta)}


def tune_semantic(query: str, k: int = 8) -> list[dict]:
    import numpy as np
    if not VEC_NPY.exists():
        return []
    mat = np.load(VEC_NPY)
    meta = json.loads(VEC_IDS.read_text())
    qv = np.asarray(_embed(["search_query: " + query])[0], dtype="float32")
    qv /= (np.linalg.norm(qv) + 1e-9)
    sims = mat @ qv
    order = np.argsort(-sims)
    out, seen = [], set()
    for idx in order:                       # best chunk per file
        m = meta[int(idx)]
        if m["file"] in seen:
            continue
        seen.add(m["file"])
        out.append({"file": m["file"], "similarity": round(float(sims[idx]), 3),
                    "snippet": m["snippet"]})
        if len(out) >= k:
            break
    return out


def tune(query: str, k: int = 8) -> dict:
    """
    Tune into the project's OWN local mesh. Each keyword either hits a known
    project concept directly, or self-heals: reverse-engineers to the nearest
    known words the project actually contains and bridges to their cells.
    Fully local — this is the project being its own SignalMesh.
    """
    from difflib import SequenceMatcher
    con = _connect()
    try:
        vocab = {r["name"]: r["value"]
                 for r in con.execute("SELECT name,value FROM cells WHERE grid='keywords'")}
    finally:
        con.close()

    tokens = [t for t in re.split(r"[^a-z0-9]+", query.lower()) if len(t) >= 2]
    bridges, hit_files = [], {}

    for tok in tokens:
        if tok in vocab:                                   # direct hit
            bridges.append({"keyword": tok, "bridged_to": tok, "confidence": 1.0})
            for f in vocab[tok].split():
                hit_files[f] = hit_files.get(f, 0) + 1
            continue
        # miss → reverse-engineer to nearest known project words (containment + ratio)
        scored = []
        for w in vocab:
            contain = 1.0 if (tok in w or w in tok) else 0.0
            sim = max(SequenceMatcher(None, tok, w).ratio(), contain)
            if sim >= 0.5:
                scored.append((w, sim))
        scored.sort(key=lambda x: -x[1])
        for w, sc in scored[:3]:
            bridges.append({"keyword": tok, "bridged_to": w, "confidence": round(sc, 3)})
            for f in vocab[w].split():
                hit_files[f] = hit_files.get(f, 0) + sc

    ranked = sorted(hit_files.items(), key=lambda x: -x[1])[:k]
    semantic = []
    try:
        semantic = tune_semantic(query, k)   # meaning-based; bridges infinite terms
    except Exception:
        pass
    return {
        "query": query,
        "bridges": bridges,                  # lexical self-healing trail (typos→words)
        "lexical": [{"file": f, "score": round(s, 2)} for f, s in ranked],
        "semantic": semantic,                # meaning-based bridge (any term → concept)
        "signals_matched": len(ranked) + len(semantic),
    }


def addr(coord: str) -> dict | None:
    con = _connect()
    try:
        r = con.execute("SELECT * FROM cells WHERE coord=?", (coord,)).fetchone()
        return dict(r) if r else None
    finally:
        con.close()


def stats() -> dict:
    con = _connect()
    try:
        g = {r["name"]: r["rows"] for r in con.execute("SELECT name,rows FROM grids")}
        return {"container": CONTAINER, "grids": g,
                "total_cells": con.execute("SELECT COUNT(*) c FROM cells").fetchone()["c"]}
    finally:
        con.close()


# ── Thought-log: the mesh reframes chat handling itself ────────────────────────
# Everything is a log. Human input is logged as its OWN cell; the LLM never
# "receives" the chat, it coordinate-checks the log, thinks, and logs ONLY the
# thought. A thought lands in cell_fts the instant it's written, so the mesh
# keyword-traces it for free and hands back the filtered slice to act from. The
# final logged thought IS the reply. Coordinates flow (tens of tokens), not file
# bodies (tens of thousands) — that's the whole saving.

def _next_row(con, grid: str) -> int:
    return con.execute("SELECT COALESCE(MAX(row),-1)+1 r FROM cells WHERE grid=?",
                       (grid,)).fetchone()["r"]


def log_turn(role: str, text: str) -> str:
    """Append one turn (human|thought|action) to the thoughts grid as its own cell."""
    con = _connect(); _init(con)
    try:
        row = _next_row(con, "thoughts")
        coord = f"{CONTAINER}.thoughts.{row}"
        con.execute("INSERT OR REPLACE INTO cells(coord,grid,row,name,kind,value,mtime,chash) "
                    "VALUES(?,?,?,?,?,?,?,?)",
                    (coord, "thoughts", row, role, role, text[:4000], time.time(), ""))
        con.execute("INSERT OR REPLACE INTO grids(name,container,parent,rows) VALUES(?,?,?,?)",
                    ("thoughts", CONTAINER, "signalmesh_processes", row + 1))
        _log(con, coord, role, text[:120])
        con.commit()
        return coord
    finally:
        con.close()


def recall(limit: int = 12, role: str = "") -> list[dict]:
    """Check the log instead of receiving the chat. Recent turns, oldest→newest."""
    con = _connect(); _init(con)
    try:
        sql = ("SELECT coord,row,kind,value,mtime FROM cells WHERE grid='thoughts'"
               + (" AND kind=?" if role else "") + " ORDER BY row DESC LIMIT ?")
        args = (role, limit) if role else (limit,)
        return [dict(r) for r in reversed(con.execute(sql, args).fetchall())]
    finally:
        con.close()


def trace(text: str, k: int = 8) -> dict:
    """Keyword-and-trace a thought through the mesh: the thought IS the query.
    Returns the filtered coordinate slice to act from — never whole files."""
    return {"fts": q(text, k), "tune": tune(text, k)}


# Executable / "acting" file types — moving parts the project can actually run.
_EXEC_EXT = {".py", ".js", ".ts", ".tsx", ".jsx", ".sh", ".go", ".rs", ".c", ".cpp", ".sql"}
_DOC_EXT = {".md", ".txt", ".rst", ".adoc"}
_STOP = {"with", "that", "this", "from", "have", "into", "will", "your", "when", "then",
         "than", "them", "they", "what", "which", "about", "would", "could", "should",
         "there", "here", "make", "want", "need", "using", "able", "also", "just"}


def _cell_kind(h: dict) -> str:
    """Classify a traced cell: 'concrete' = a real moving part (symbol/def/class, live
    service, config wiring, executable file); 'doc' = a spec/prose file (.md/.txt) that
    describes but doesn't run; 'soft' = self-traces (thoughts), keywords, everything else."""
    g = h["grid"]
    if g in ("symbols", "services", "config"):
        return "concrete"
    if g == "files":
        ext = Path(h["name"]).suffix.lower()
        if ext in _EXEC_EXT:
            return "concrete"
        if ext in _DOC_EXT:
            return "doc"
    return "soft"


def classify(text: str, exclude_coord: str = "") -> dict:
    """Decide if an intent is NEW / PARTIAL / EXISTING / SPEC'D — not by raw hit-count
    (ORed tokens collide with generic words across unrelated subprojects), but by whether
    the intent's distinctive terms CO-LOCATE in concrete cells. Terms that ground nowhere
    concrete are the parts that don't exist yet. SPEC'D = lives in doc bodies (semantic,
    lexically verified) but no executable cells — written down, not built."""
    toks = list(dict.fromkeys(t for t in re.split(r"[^a-z0-9]+", text.lower())
                              if len(t) >= 4 and not t.isdigit() and t not in _STOP))[:16]
    file_tokens: dict[str, set] = {}     # concrete file → which intent terms co-locate
    doc_tokens: dict[str, set] = {}      # spec/doc file → which intent terms co-locate
    tok_hits: dict[str, int] = {}        # term → # concrete cells it grounds in
    symbol_terms: set = set()            # terms that hit a DEDICATED def/class (strong signal)
    for t in toks:
        n = 0
        for h in q(t, 30):
            if h["coord"] == exclude_coord:
                continue
            kind = _cell_kind(h)
            f = h["name"] if h["grid"] == "files" else h["value"]
            if kind == "concrete":
                file_tokens.setdefault(f, set()).add(t); n += 1
                if h["grid"] == "symbols":
                    symbol_terms.add(t)
            elif kind == "doc":
                doc_tokens.setdefault(f, set()).add(t)
        tok_hits[t] = n

    coverage = max((len(v) for v in file_tokens.values()), default=0)
    doc_coverage = max((len(v) for v in doc_tokens.values()), default=0)
    clusters = sorted((f for f, v in file_tokens.items() if len(v) >= 2),
                      key=lambda f: -len(file_tokens[f]))
    doc_clusters = sorted((f for f, v in doc_tokens.items() if len(v) >= 2),
                          key=lambda f: -len(doc_tokens[f]))
    grounded = [t for t in toks if tok_hits.get(t)]
    documented = [t for t in toks if any(t in v for v in doc_tokens.values())]
    ungrounded = [t for t in toks if not tok_hits.get(t)]
    need = min(3, max(2, (len(toks) + 1) // 2))

    # FTS only sees paths/summaries, so .md/.txt BODIES (where heavy pasted specs live)
    # are invisible. tune_semantic reads embedded content chunks. And since the embedding
    # floor is high in a big repo (a novel query still finds a ~0.6 "nearest" doc), a
    # semantic doc hit only counts if its chunk actually SHARES the intent's vocabulary.
    sem = []
    try:
        sem = tune_semantic(text, 8)
    except Exception:
        pass
    SEM = 0.5
    spec_files = [s["file"] for s in sem
                  if Path(s["file"]).suffix.lower() in _DOC_EXT and s["similarity"] >= SEM
                  and any(t in (s.get("snippet") or "").lower() for t in toks)]

    # Two independent signals that a capability is already built:
    #  • co-location depth  — several intent terms in ONE file (tightly-coupled feature)
    #  • symbol breadth     — several intent terms each with a DEDICATED def/class, even
    #                         across different modules (well-separated feature). Without
    #                         this, good architecture (concerns split across files) reads
    #                         as "not here" — the false-negative that hid the licensing/
    #                         stripe/checkout code, which lives in 3 separate modules.
    strong = len(symbol_terms)
    if (coverage >= need and len(clusters) >= 2) or strong >= 3:
        v, note = "EXISTING", ("already in the stack — terms co-locate, or several map to dedicated "
                               "symbols across modules. Act from these coordinates; don't rebuild.")
    elif coverage >= 2 or strong >= 2:
        v, note = "PARTIAL", ("a seed exists — some terms co-locate or map to real symbols, but "
                              "thinly. Finish or reframe the existing pieces to fit the new intent.")
    elif doc_coverage >= 2 or spec_files:
        v, note = ("SPEC'D", "the concept lives in docs/specs (by name or content) but has NO "
                             "executable cells — written down, not built. Implement it from the spec.")
    else:
        v, note = "NEW", ("only isolated/generic words match — the concept as a unit isn't in the "
                          "project (or traces only to itself/docs). Forge it as a new "
                          "feature/process/function/visual.")
    return {"verdict": v, "note": note, "coverage": coverage, "doc_coverage": doc_coverage,
            "symbol_terms": sorted(symbol_terms),
            "grounded_terms": grounded, "documented_terms": documented, "ungrounded_terms": ungrounded,
            "clusters": clusters[:6], "doc_clusters": doc_clusters[:6],
            "spec_files": sorted(set(spec_files))[:6], "semantic_available": bool(sem)}


def think(text: str, k: int = 8) -> dict:
    """The whole inner loop in one move: log the thought (its own FTS-indexed cell),
    classify it (NEW/PARTIAL/EXISTING/SPEC'D), and hand back the mesh-filtered traces."""
    coord = log_turn("thought", text)
    return {"coord": coord, "verdict": classify(text, exclude_coord=coord), **trace(text, k)}


def chat(text: str) -> dict:
    """Human input → its own log cell. The LLM does NOT receive this directly; it
    recall()s / tunes in for updates. Returns the coord to coordinate-check."""
    coord = log_turn("human", text)
    return {"coord": coord, "next": "recall → think → act → think; final thought = reply"}


# ── Capability registry: the mesh indexes what the system CAN DO, not just code ──
# Code tells you what EXISTS; capabilities tell you what's CALLABLE. MCP tools,
# agent personas, HTTP routes and CLI commands become first-class, coordinate-
# addressable cells, so `q`/`tune` surface a runnable capability the same way they
# surface a symbol. This is the "index of everything, including capabilities" —
# an agent asks the mesh "what can act on stripe?" and gets the tool, not a guess.

# HTTP route decorators across frameworks (FastAPI/Flask/Express/Starlette). First
# group = verb, second = path. High-signal, low-false-positive — these are real
# externally-callable capabilities, not just any function.
_ROUTE_RE = re.compile(
    r"""@?\s*(?:app|router|api|bp|blueprint|[a-z_]+)\s*\.\s*"""
    r"""(get|post|put|patch|delete|websocket|ws|route)\s*\(\s*["'`]([^"'`]+)["'`]""",
    re.I)
# CLI commands: click decorators + argparse subparsers.
_CMD_RE = re.compile(r"""@\s*[a-z_]+\.command\s*\(\s*(?:name\s*=\s*)?["']([^"']+)["']""", re.I)

# Next.js App Router route handlers are externally callable too, but unlike
# decorator frameworks the URL lives in the file path and the HTTP verb lives in
# an exported function/const. Harvest both forms so TypeScript apps participate in
# the same capability grid as Python/Express services.
_NEXT_ROUTE_HANDLER_RE = re.compile(
    r"\bexport\s+(?:(?:async\s+)?function\s+|const\s+)"
    r"(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS)\b",
    re.I,
)
_NEXT_ROUTE_REEXPORT_RE = re.compile(r"\bexport\s*\{([^}]+)\}", re.S)


def _nextjs_route_path(rel: str) -> str:
    """Infer the public URL for a Next.js App Router route module."""
    parts = Path(rel).parts
    if len(parts) < 2 or not parts[-1].startswith("route."):
        return ""
    if parts[0] == "app":
        route_parts = parts[1:-1]
    elif len(parts) >= 3 and parts[:2] == ("src", "app"):
        route_parts = parts[2:-1]
    else:
        return ""

    public = [
        part for part in route_parts
        if not (part.startswith("(") and part.endswith(")")) and not part.startswith("@")
    ]
    return "/" + "/".join(public) if public else "/"


def _nextjs_route_caps(rel: str, text: str) -> list[tuple[str, str]]:
    path = _nextjs_route_path(rel)
    if not path:
        return []

    verbs = {m.upper() for m in _NEXT_ROUTE_HANDLER_RE.findall(text)}
    for block in _NEXT_ROUTE_REEXPORT_RE.findall(text):
        for piece in block.split(","):
            alias = re.search(
                r"(?:^|\s+as\s+)(GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS)\s*$",
                piece.strip(),
                re.I,
            )
            if alias:
                verbs.add(alias.group(1).upper())
    return [(verb, path) for verb in sorted(verbs)]


def _harvest_route_caps(root: Path) -> list[dict]:
    """Scan the repo for externally-callable capabilities (HTTP routes, CLI commands)
    and return them as capability items. Portable: works on any meshed repo with no
    runtime — a bare web service still gets a capability grid describing its surface."""
    items: list[dict] = []
    seen: set = set()
    for p, ext in _walk(root):
        if ext not in _CODE_EXT:
            continue
        try:
            text = p.read_text(errors="ignore")
        except Exception:
            continue
        rel = str(p.relative_to(root))
        for verb, route_path in _nextjs_route_caps(rel, text):
            key = (verb, route_path)
            if key not in seen:
                seen.add(key)
                items.append({"name": f"{verb} {route_path}", "kind": "route",
                              "value": f"{verb} {route_path} - Next.js App Router route in {rel}",
                              "source": rel})
        for verb, path in _ROUTE_RE.findall(text):
            key = (verb.upper(), path)
            if path.startswith("/") and key not in seen:
                seen.add(key)
                items.append({"name": f"{verb.upper()} {path}", "kind": "route",
                              "value": f"{verb.upper()} {path} — HTTP route in {rel}",
                              "source": rel})
        for name in _CMD_RE.findall(text):
            key = ("CMD", name)
            if key not in seen:
                seen.add(key)
                items.append({"name": name, "kind": "command",
                              "value": f"CLI command '{name}' in {rel}", "source": rel})
    return items


def index_capabilities(items: list[dict], replace: bool = True) -> dict:
    """Ingest external capability nodes into the mesh as coordinate-addressable cells.
    Each item: {name, kind, value|description, source?}. `value` carries the FTS
    payload (name + description + tags) so q/tune match on intent. With replace=True
    the whole capabilities grid is rebuilt from `items` — callers assemble the full
    set (routes + MCP tools + agent personas) in one shot, keeping a single source of
    truth. Survives code re-charts (chart preserves the capabilities grid)."""
    con = _connect(); _init(con)
    try:
        if replace:
            con.execute("DELETE FROM cells WHERE grid='capabilities'")
        n = 0
        for it in items:
            name = str(it.get("name", ""))[:200]
            kind = str(it.get("kind", "capability"))[:40]
            value = str(it.get("value") or it.get("description") or name)[:4000]
            coord = f"{CONTAINER}.capabilities.{n}"
            con.execute(
                "INSERT OR REPLACE INTO cells(coord,grid,row,name,kind,value,mtime,chash) "
                "VALUES(?,?,?,?,?,?,?,?)",
                (coord, "capabilities", n, name, kind, value, time.time(), ""))
            n += 1
        con.execute("INSERT OR REPLACE INTO grids(name,container,parent,rows) VALUES(?,?,?,?)",
                    ("capabilities", CONTAINER, "signalmesh_processes", n))
        con.commit()
        return {"capabilities": n}
    finally:
        con.close()


def capabilities(kind: str = "", limit: int = 200) -> list[dict]:
    """List capability cells, optionally filtered by kind (route|command|mcp_tool|agent)."""
    con = _connect(); _init(con)
    try:
        sql = "SELECT coord,row,name,kind,value FROM cells WHERE grid='capabilities'"
        args: tuple = ()
        if kind:
            sql += " AND kind=?"; args = (kind,)
        sql += " ORDER BY row LIMIT ?"; args = args + (limit,)
        return [dict(r) for r in con.execute(sql, args).fetchall()]
    finally:
        con.close()


# ── Universal target ingest ─────────────────────────────────────────────────────
# The one thing the mesh couldn't do: chart an *external* app it's handed so the
# existing probe→heal→drive loop (setup_facets / Gepetto / launch_healer) has
# coordinates to act on. Hand it a URL, a .app bundle, a binary, or a code dir;
# it introspects the functional I/O surface + missing-value gaps into the grid.
_TARGET_UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
              "(KHTML, like Gecko) Chrome/124.0 Safari/537.36")  # real UA: urllib default trips CF 1010
_URL_MAX = 512_000


def _introspect_url(url: str) -> tuple[str, list[dict], list[str]]:
    """GET the URL once and extract its functional I/O flows: forms (with their
    inputs), same-origin navigable routes, script/asset refs — and the required
    inputs with no default become the 'missing values' gaps."""
    import urllib.request, urllib.parse
    from html.parser import HTMLParser

    class _F(HTMLParser):
        def __init__(self):
            super().__init__()
            self.forms: list[dict] = []; self.links: set = set(); self.apis: set = set()
            self._cur = None
        def handle_starttag(self, tag, attrs):
            a = dict(attrs)
            if tag == "form":
                self._cur = {"action": a.get("action", ""),
                             "method": (a.get("method") or "GET").upper(), "inputs": []}
                self.forms.append(self._cur)
            elif tag in ("input", "select", "textarea") and self._cur is not None:
                if a.get("name"):
                    self._cur["inputs"].append(
                        {"name": a["name"], "required": "required" in a, "value": a.get("value", "")})
            elif tag == "a" and a.get("href"):
                self.links.add(a["href"])
            elif tag == "script" and a.get("src"):
                self.apis.add(a["src"])
        def handle_endtag(self, tag):
            if tag == "form":
                self._cur = None

    items, gaps, status, html = [], [], 0, ""
    try:
        req = urllib.request.Request(url, headers={"User-Agent": _TARGET_UA})
        with urllib.request.urlopen(req, timeout=8) as r:
            status = getattr(r, "status", 0) or r.getcode()
            html = r.read(_URL_MAX).decode("utf-8", "ignore")
    except Exception as e:
        gaps.append(f"unreachable: {e}")
    items.append({"name": f"GET {url}", "kind": "health",
                  "value": f"root reachability GET {url} -> HTTP {status}"})
    fp = _F()
    try:
        fp.feed(html)
    except Exception:
        pass
    origin = urllib.parse.urlsplit(url).netloc
    for f in fp.forms:
        action = urllib.parse.urljoin(url, f["action"] or url)
        items.append({"name": f"{f['method']} {action}", "kind": "io_flow",
                      "value": f"{f['method']} {action} — form; inputs: "
                               + ", ".join(i["name"] for i in f["inputs"])})
        for i in f["inputs"]:
            if i["required"] or not i["value"]:
                gaps.append(f"{action} needs '{i['name']}'")
    for href in sorted(fp.links):
        full = urllib.parse.urljoin(url, href)
        if urllib.parse.urlsplit(full).netloc == origin:
            items.append({"name": full, "kind": "nav_flow", "value": f"navigable route {full}"})
    for src in sorted(fp.apis):
        items.append({"name": src, "kind": "asset",
                      "value": f"script/asset {urllib.parse.urljoin(url, src)}"})
    return "url", items, gaps


def _introspect_app(p: Path) -> tuple[str, list[dict], list[str]]:
    """Read a macOS .app bundle's Info.plist: bundle id, executable (the launch
    flow), and registered URL schemes (its callable I/O)."""
    import plistlib
    items, gaps, meta = [], [], {}
    try:
        meta = plistlib.loads((p / "Contents" / "Info.plist").read_bytes())
    except Exception as e:
        gaps.append(f"Info.plist unreadable: {e}")
    bid = meta.get("CFBundleIdentifier", ""); exe = meta.get("CFBundleExecutable", "")
    items.append({"name": bid or p.name, "kind": "app",
                  "value": f".app {p.name} id={bid} exe={exe} "
                           f"min={meta.get('LSMinimumSystemVersion', '')}"})
    if exe:
        exe_path = p / "Contents" / "MacOS" / exe
        ok = exe_path.exists()
        items.append({"name": f"launch {exe}", "kind": "launch",
                      "value": f"launch flow: {exe_path} (present={ok})"})
        if not ok:
            gaps.append(f"executable missing: {exe_path}")
    for t in meta.get("CFBundleURLTypes", []) or []:
        for s in t.get("CFBundleURLSchemes", []) or []:
            items.append({"name": f"{s}://", "kind": "io_flow",
                          "value": f"URL scheme {s}:// handled by {bid or p.name}"})
    return "app", items, gaps


def _introspect_binary(p: Path) -> tuple[str, list[dict], list[str]]:
    """Probe an executable's callable surface via its own --help, parsing the
    subcommand table. The launch/verify flow the heal loop drives."""
    import subprocess
    items = [{"name": p.name, "kind": "binary",
              "value": f"executable {p} (size={p.stat().st_size})"}]
    gaps, help_txt = [], ""
    for flag in ("--help", "-h", "help"):
        try:
            r = subprocess.run([str(p), flag], capture_output=True, text=True, timeout=8)
            help_txt = r.stdout or r.stderr
            if help_txt.strip():
                break
        except Exception as e:
            gaps.append(f"{flag} failed: {e}")
    for m in re.finditer(r"^\s{2,}([a-z][a-z0-9:_-]{1,30})\s{2,}\S", help_txt, re.M):
        items.append({"name": m.group(1), "kind": "command",
                      "value": f"subcommand '{m.group(1)}' of {p.name}"})
    if not help_txt.strip():
        gaps.append(f"no help surface — {p.name} may need args or is not a CLI")
    return "binary", items, gaps


def _introspect_target(arg: str) -> tuple[str, list[dict], list[str]]:
    """Classify a target and route to the right introspector. Returns
    (kind, flow_items, gap_notes)."""
    if arg.lower().startswith(("http://", "https://")):
        return _introspect_url(arg)
    p = Path(arg).expanduser()
    if p.suffix == ".app" or (p.is_dir() and (p / "Contents" / "Info.plist").exists()):
        return _introspect_app(p)
    if p.is_dir():
        return "dir", _harvest_route_caps(p), []  # rung 2: reuse the existing harvester
    if p.exists():
        return _introspect_binary(p)
    return "unknown", [], [f"target not found: {arg}"]


def mesh_target(arg: str, limit: int = 300) -> dict:
    """Ingest an arbitrary external app (URL | .app | binary | code dir) into the
    grid as `targets` cells so q/tune/addr find it and the existing heal loop can
    reconcile/launch/drive it. Idempotent per-target. Leaves a thought breadcrumb
    the autopilot/Gepetto read."""
    kind, items, gaps = _introspect_target(arg)
    marker = f"[target={arg}]"
    con = _connect(); _init(con)
    try:
        con.execute("DELETE FROM cells WHERE grid='targets' AND instr(value, ?)>0", (marker,))
        base = con.execute(
            "SELECT COALESCE(MAX(row),-1)+1 FROM cells WHERE grid='targets'").fetchone()[0]
        n = 0
        for it in items[:limit]:
            row = base + n
            con.execute(
                "INSERT OR REPLACE INTO cells(coord,grid,row,name,kind,value,mtime,chash) "
                "VALUES(?,?,?,?,?,?,?,?)",
                (f"{CONTAINER}.targets.{row}", "targets", row, str(it["name"])[:200],
                 str(it["kind"])[:40], f"{it['value']} {marker}"[:4000], time.time(), ""))
            n += 1
        total = con.execute("SELECT COUNT(*) FROM cells WHERE grid='targets'").fetchone()[0]
        con.execute("INSERT OR REPLACE INTO grids(name,container,parent,rows) VALUES(?,?,?,?)",
                    ("targets", CONTAINER, "signalmesh_processes", total))
        con.commit()
    finally:
        con.close()
    think(f"[TARGET INGEST] {kind}:{arg} → {n} flow(s), {len(gaps)} gap(s)"
          + (": " + "; ".join(gaps[:6]) if gaps else ""))
    return {"target": arg, "kind": kind, "flows": n, "gaps": gaps}


def targets(limit: int = 300) -> list[dict]:
    """List ingested target flows."""
    con = _connect(); _init(con)
    try:
        return [dict(r) for r in con.execute(
            "SELECT coord,name,kind,value FROM cells WHERE grid='targets' "
            "ORDER BY row LIMIT ?", (limit,)).fetchall()]
    finally:
        con.close()


def gen_agents_md(out: str | Path = None) -> str:
    """Generate AGENTS.md straight FROM the grid (shape, live services, entry points)
    so any repo dropped in self-describes. Tells a CLI agent to use the gridstack for
    instant project cognition instead of crawling files."""
    s = stats()
    con = _connect()
    try:
        services = [json.loads(r["value"]) for r in
                    con.execute("SELECT value FROM cells WHERE grid='services'")]
        entry = con.execute(
            "SELECT name,value FROM cells WHERE grid='files' AND ("
            "name LIKE '%main.py' OR name LIKE '%launch%' OR name LIKE '%/app.py' "
            "OR name LIKE 'README%' OR name LIKE '%cli.py' OR name LIKE '%index.html') "
            "LIMIT 25").fetchall()
        caps = con.execute(
            "SELECT name,kind FROM cells WHERE grid='capabilities' ORDER BY kind,row LIMIT 40").fetchall()
    finally:
        con.close()

    L = [f"# AGENTS.md — {CONTAINER} cognition via local SignalMesh gridstack", "",
         "This repo is indexed as its own local SignalMesh. **Do not crawl files by hand** —",
         "query the gridstack for instant, offline, always-synced project knowledge.",
         "It is kept live by the grid-god (incremental sync + per-cell changelogs).",
         "Everything is local; nothing egresses (anything that ever leaves goes through SEC-Ω).",
         "",
         "Engine not in the repo? `pipx install signalmesh-meshd` (verify: `pip show signalmesh-meshd`",
         "or `pipx list`) — or drop `signalmesh_gridstack.py` in and run `chart`.",
         "", "## Query the grid", "```bash",
         'python3 signalmesh_gridstack.py q    "<keyword>"   # instant lookup: files/symbols/config',
         'python3 signalmesh_gridstack.py tune "<topic>"     # find by MEANING (semantic + fuzzy)',
         "python3 signalmesh_gridstack.py stats              # whole-project shape",
         "python3 signalmesh_gridstack.py log [coord]        # per-cell change history",
         "python3 signalmesh_gridstack.py addr <coord>       # exact cell by coordinate",
         "python3 signalmesh_gridstack.py sync               # incremental re-index after edits",
         "```", "",
         "## Chat runs through the mesh (thought-log protocol)",
         "Do not treat the conversation as a stream you re-receive each turn. It is mesh",
         "state. Every turn is a cell in the `thoughts` grid, FTS-indexed the instant it",
         "is written — so a thought keyword-traces itself for free.", "", "```bash",
         'python3 signalmesh_gridstack.py chat   "<human input>"  # log human turn → its own cell',
         "python3 signalmesh_gridstack.py recall                  # check the log (don't re-receive)",
         'python3 signalmesh_gridstack.py think  "<your thought>" # log thought + classify + traces',
         'python3 signalmesh_gridstack.py classify "<intent>"     # NEW/PARTIAL/EXISTING/SPEC\'D?',
         "```", "",
         "**The loop:** human input is logged → you `recall` (coordinate-check, don't ingest the",
         "whole chat) → you `think` (logs ONLY the thought, mesh hands back the filtered slice) →",
         "you act from those coordinates → you `think` again. Your final logged thought IS the reply.",
         "Carry coordinates (~tens of tokens), never file bodies (~tens of thousands).", "",
         "**Is this intent already in the project?** `think`/`classify` reads the *type and",
         "diversity* of cells the thought traces to and returns a verdict:",
         "- **EXISTING** — distinctive terms co-locate across many concrete cells. Act from coords.",
         "- **PARTIAL** — a few concrete cells, low diversity. Finish/reframe the existing seed.",
         "- **SPEC'D** — terms co-locate in docs/specs (.md/.txt) but NO executable cells exist.",
         "  Written down, not built — implement it from the spec.",
         "- **NEW** — only isolated/generic words match, or traces only to itself. Forge it fresh.", "",
         f"## Shape ({s['total_cells']} cells)"]
    L += [f"- **{g}**: {n}" for g, n in s["grids"].items()]
    if caps:
        L += ["", "## Capabilities (what this repo can DO — not just what exists)",
              "The mesh indexes callable capabilities — HTTP routes, CLI commands, MCP tools",
              "and agent personas — as coordinate-addressable cells. `q`/`tune` return a",
              "runnable capability the same way they return a symbol.", "", "```bash",
              'python3 signalmesh_gridstack.py caps                # index + list callable surface',
              'python3 signalmesh_gridstack.py caps --kind=mcp_tool # filter (route|command|mcp_tool|agent)',
              'python3 signalmesh_gridstack.py events [since_ts]   # mesh event feed (bus coordination)',
              'python3 signalmesh_gridstack.py brain "<question>"  # local Qwen answer grounded in the grid',
              "```", ""]
        by_kind: dict[str, int] = {}
        for c in caps:
            by_kind[c["kind"]] = by_kind.get(c["kind"], 0) + 1
        L += [f"- **{k}**: {n}" for k, n in sorted(by_kind.items())]
        L += [f"  - `{c['name']}`" for c in caps[:12]]
    if services:
        L += ["", "## Live services (listeners)"]
        L += [f"- {sv.get('name')} @ {sv.get('addr')}" for sv in services[:20]]
    if entry:
        L += ["", "## Entry points"]
        L += [f"- `{e['name']}` — {str(e['value'])[:70]}" for e in entry]
    L += ["", "## Rule", "Local only. Secrets are masked at chart time; any egress is SEC-Ω guarded.", ""]
    md = "\n".join(L)

    out = Path(out) if out else (DB_DIR / "AGENTS.md")
    out.parent.mkdir(parents=True, exist_ok=True)
    out.write_text(md)
    return str(out)


if __name__ == "__main__":
    cmd = sys.argv[1] if len(sys.argv) > 1 else "stats"
    if cmd == "chart":
        root = Path(sys.argv[2]) if len(sys.argv) > 2 else ROOT
        print(json.dumps(chart(root), indent=2))
    elif cmd == "q":
        for r in q(" ".join(sys.argv[2:])):
            print(f"  [{r['coord']}] ({r['grid']}/{r['kind']}) {r['name']}: {r['value'][:90]}")
    elif cmd == "tune":
        print(json.dumps(tune(" ".join(sys.argv[2:])), indent=2))
    elif cmd == "embed":
        print(json.dumps(embed_index(include_vendor="--all" in sys.argv), indent=2))
    elif cmd == "sync":
        print(json.dumps(sync(), indent=2))
    elif cmd in ("grid-god", "gridgod"):
        iv = float(sys.argv[2]) if len(sys.argv) > 2 else 4.0
        grid_god(interval=iv)
    elif cmd == "log":
        coord = sys.argv[2] if len(sys.argv) > 2 else ""
        for r in changelog_for(coord):
            print(f"  {time.strftime('%H:%M:%S', time.localtime(r['ts']))}  "
                  f"{r['change']:8} {r['coord']}  {r['detail']}")
    elif cmd in ("agents-md", "agentsmd"):
        path = sys.argv[2] if len(sys.argv) > 2 else None
        print("wrote", gen_agents_md(path))
    elif cmd == "addr":
        print(json.dumps(addr(sys.argv[2]), indent=2))
    elif cmd == "chat":
        print(json.dumps(chat(" ".join(sys.argv[2:])), indent=2))
    elif cmd == "think":
        print(json.dumps(think(" ".join(sys.argv[2:])), indent=2))
    elif cmd == "trace":
        print(json.dumps(trace(" ".join(sys.argv[2:])), indent=2))
    elif cmd in ("classify", "new?"):
        print(json.dumps(classify(" ".join(sys.argv[2:])), indent=2))
    elif cmd in ("recall", "tune-in", "tunein"):
        lim = int(sys.argv[2]) if len(sys.argv) > 2 and sys.argv[2].isdigit() else 12
        for r in recall(lim):
            print(f"  [{r['coord']}] {r['kind']:7} {str(r['value'])[:100]}")
    elif cmd in ("caps", "capabilities"):
        # Harvest this repo's callable surface (routes + CLI commands) into the
        # capabilities grid, then list it. OCODX merges MCP tools + agents on top.
        if "--list" not in sys.argv:
            res = index_capabilities(_harvest_route_caps(ROOT))
            print(json.dumps(res, indent=2))
        kind = next((a.split("=", 1)[1] for a in sys.argv if a.startswith("--kind=")), "")
        for c in capabilities(kind):
            print(f"  [{c['coord']}] {c['kind']:9} {c['name']}")
    elif cmd in ("target", "mesh-target", "ingest-target"):
        if "--list" in sys.argv:
            for t in targets():
                print(f"  [{t['coord']}] {t['kind']:9} {t['name']}")
        elif len(sys.argv) < 3:
            print("usage: target <url|/path/to/App.app|/path/to/binary|/code/dir>  [--list]")
        else:
            print(json.dumps(mesh_target(sys.argv[2]), indent=2))
    elif cmd in ("events", "feed"):
        since = float(sys.argv[2]) if len(sys.argv) > 2 else 0.0
        print(json.dumps(events_since(since), indent=2))
    elif cmd == "merge":
        if len(sys.argv) < 4:
            print("usage: merge <source_prefix> <target_prefix>")
        else:
            print(json.dumps(merge_analyze(sys.argv[2], sys.argv[3]), indent=2))
    elif cmd in ("brain", "ask"):
        if not brain_available():
            print("native brain unavailable — needs llama_cpp + a gguf "
                  "(set QWEN_MODEL_PATH, or install huggingface_hub to auto-download).")
        else:
            print(brain_ask(" ".join(sys.argv[2:])))
    elif cmd == "ingest-migration":
        from src.open_codex.mesh_migration import MigrationWorkspace, DataIngester
        if len(sys.argv) < 3:
            print("usage: ingest-migration <source_dir>")
        else:
            w = MigrationWorkspace()
            d = DataIngester(w)
            d.ingest_directory(sys.argv[2])
    elif cmd == "infer-schema":
        from src.open_codex.mesh_migration import MigrationWorkspace, SchemaInferrer
        w = MigrationWorkspace()
        s = SchemaInferrer(w)
        s.infer()
    elif cmd == "translate-ids":
        from src.open_codex.mesh_migration import MigrationWorkspace, SchemaInferrer, TranslationMatrix
        w = MigrationWorkspace()
        s = SchemaInferrer(w)
        graph = s.infer()
        t = TranslationMatrix(w, graph)
        t.translate()
    elif cmd == "export-migration":
        from src.open_codex.mesh_migration import MigrationWorkspace, SchemaInferrer, ExportEngine
        if len(sys.argv) < 4:
            print("usage: export-migration <dialect> <output_file>")
        else:
            w = MigrationWorkspace()
            s = SchemaInferrer(w)
            graph = s.infer()
            e = ExportEngine(w, graph)
            e.export(sys.argv[2], sys.argv[3])
    elif cmd == "rebrand-project":
        from src.open_codex.mesh_rebrand import MeshRebrander
        if len(sys.argv) < 4:
            print("usage: rebrand-project <old_term> <new_term>")
        else:
            r = MeshRebrander(ROOT)
            r.rebrand(sys.argv[2], sys.argv[3])
    elif cmd == "stats":
        print(json.dumps(stats(), indent=2))
    else:
        print(__doc__)