Loom / brain.py
deploy-space action
deploy 6158a00 (c)
b972454
Raw
History Blame Contribute Delete
27.4 kB
"""``.brain/`` folder I/O for the Conscious multi-agent system (Tier 3, Phase 1).
Pure filesystem + git operations. No HTTP, no DB. The brain is a human-readable,
git-versioned folder persisted in the user's workspace repo. The DB
(``conscious_db.py``) mirrors state for fast queries; this module is the
canonical writer for the on-disk artifact.
Layout (see TIER3_PLAN.md §6.2)::
<workspace>/.brain/
├── CONSCIOUS.md # regenerated summary
├── goal.md # the goal, verbatim
├── plan.json # task DAG mirror
├── config.json # conscious-level config
├── agents/<id>.md # one per agent
├── agents/<id>.log # append-only activity log
├── blackboard/<section>.jsonl
├── blackboard/custom/<section>.jsonl
├── blackboard/events.jsonl
├── drawer/<invoke-id>/{request.json,result.md,files/,status.json}
├── proposals/queue.jsonl
└── skills/ # conscious skill doc (seeded at creation)
All writes are serialized per-brain-path via ``BRAIN_LOCKS`` so concurrent
appends never corrupt a ``.jsonl`` line.
"""
from __future__ import annotations
import json
import os
import re
import shutil
import threading
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
# Built-in blackboard sections. Custom sections land in blackboard/custom/.
BUILTIN_SECTIONS = {"goal", "plan", "decision", "finding", "question", "artifact", "event"}
# Per-brain-path lock registry — serializes .jsonl appends.
BRAIN_LOCKS: dict[str, threading.Lock] = {}
_BRAIN_LOCKS_GUARD = threading.Lock()
def _lock_for(brain_path: Path) -> threading.Lock:
key = str(brain_path.resolve())
with _BRAIN_LOCKS_GUARD:
lk = BRAIN_LOCKS.get(key)
if lk is None:
lk = threading.Lock()
BRAIN_LOCKS[key] = lk
return lk
def _iso_now() -> str:
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def _compact_json(obj: Any) -> str:
return json.dumps(obj, separators=(",", ":"), ensure_ascii=False)
def _pretty_json(obj: Any) -> str:
return json.dumps(obj, indent=2, ensure_ascii=False)
# ---------------------------------------------------------------------------
# init / structure
# ---------------------------------------------------------------------------
def init_brain(workspace_path: Path, conscious: dict) -> Path:
"""Create the ``.brain/`` folder with the full structure.
Idempotent: a no-op if the folder exists, but always refreshes
``CONSCIOUS.md`` and ``config.json`` from the current conscious dict.
Returns the ``.brain/`` path.
"""
brain = workspace_path / ".brain"
(brain / "agents").mkdir(parents=True, exist_ok=True)
(brain / "blackboard" / "custom").mkdir(parents=True, exist_ok=True)
(brain / "drawer").mkdir(parents=True, exist_ok=True)
(brain / "proposals").mkdir(parents=True, exist_ok=True)
(brain / "skills").mkdir(parents=True, exist_ok=True)
# goal.md (verbatim)
(brain / "goal.md").write_text(conscious.get("goal", "") or "", encoding="utf-8")
# config.json
cfg = {
"conscious_id": conscious["id"],
"cost_ceiling_usd": conscious.get("cost_ceiling_usd", 0),
"brain_commit_policy": conscious.get("brain_commit_policy", "on"),
"graphiti_enabled": bool(conscious.get("graphiti_enabled", 0)),
"max_agents": conscious.get("max_agents", 8),
"created_at": conscious.get("created_at") or _iso_now(),
"updated_at": _iso_now(),
}
(brain / "config.json").write_text(_pretty_json(cfg), encoding="utf-8")
# empty .jsonl files (touch)
for sec in BUILTIN_SECTIONS:
p = brain / "blackboard" / f"{sec}.jsonl"
if not p.exists():
p.touch()
# proposals queue + events are part of BUILTIN_SECTIONS via "event" section,
# but proposals/queue.jsonl is a separate file.
pq = brain / "proposals" / "queue.jsonl"
if not pq.exists():
pq.touch()
# plan.json (empty default)
pj = brain / "plan.json"
if not pj.exists() or pj.stat().st_size == 0:
pj.write_text(_pretty_json({
"conscious_id": conscious["id"],
"version": 0,
"tasks": [],
"updated_at": _iso_now(),
}), encoding="utf-8")
# seed the conscious SKILL.md (best-effort; skills source may not exist yet)
_seed_skill(brain)
# CONSCIOUS.md (refreshed every call)
regenerate_conscious_md(brain, conscious, [], [], [], [])
return brain
def _seed_skill(brain: Path) -> None:
"""Copy the conscious SKILL.md into .brain/skills/ if the source exists."""
here = Path(__file__).resolve().parent
src = here / "agent_skills" / "conscious" / "SKILL.md"
if not src.is_file():
return
dest = brain / "skills" / "conscious" / "SKILL.md"
try:
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dest)
except Exception:
pass
# ---------------------------------------------------------------------------
# read snapshot
# ---------------------------------------------------------------------------
def read_brain(brain_path: Path) -> dict:
"""Return a structured snapshot of the brain (used by conscious_context)."""
bp = brain_path
out: dict[str, Any] = {"brain_path": str(bp)}
if (bp / "goal.md").is_file():
out["goal"] = (bp / "goal.md").read_text(encoding="utf-8")
else:
out["goal"] = ""
if (bp / "config.json").is_file():
try:
out["config"] = json.loads((bp / "config.json").read_text(encoding="utf-8"))
except Exception:
out["config"] = {}
else:
out["config"] = {}
out["blackboard"] = read_blackboard(bp)
out["events"] = read_events(bp)
out["proposals"] = read_proposals(bp)
out["agents"] = read_agents(bp)
return out
def read_blackboard(brain_path: Path, section: str | None = None) -> dict[str, list[dict]]:
"""Read blackboard sections. Returns {section: [entries...]}.
If ``section`` is given, returns only that section. Entries are returned
in append order; readers wanting "latest version per key" take the max-version
line per key themselves.
"""
out: dict[str, list[dict]] = {}
bb = brain_path / "blackboard"
if not bb.is_dir():
return out
sections = [section] if section else list(BUILTIN_SECTIONS)
for sec in sections:
p = bb / f"{sec}.jsonl"
entries: list[dict] = []
if p.is_file():
for line in p.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line:
continue
try:
entries.append(json.loads(line))
except Exception:
continue
if entries:
out[sec] = entries
# custom sections
custom_dir = bb / "custom"
if custom_dir.is_dir():
for p in custom_dir.glob("*.jsonl"):
entries = []
for line in p.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line:
continue
try:
entries.append(json.loads(line))
except Exception:
continue
if entries:
out[p.stem] = entries
return out
def read_events(brain_path: Path, since: int = 0) -> list[dict]:
"""Read events with id > since. Events use a monotonic id field."""
p = brain_path / "blackboard" / "events.jsonl"
if not p.is_file():
return []
out: list[dict] = []
for line in p.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
if obj.get("id", 0) > since:
out.append(obj)
except Exception:
continue
return out
def read_proposals(brain_path: Path) -> list[dict]:
p = brain_path / "proposals" / "queue.jsonl"
if not p.is_file():
return []
out: list[dict] = []
for line in p.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line:
continue
try:
out.append(json.loads(line))
except Exception:
continue
return out
def read_agents(brain_path: Path) -> list[str]:
"""Return list of agent IDs that have a profile file."""
d = brain_path / "agents"
if not d.is_dir():
return []
return [p.stem for p in d.glob("*.md")]
# ---------------------------------------------------------------------------
# append operations (all serialized via per-brain lock)
# ---------------------------------------------------------------------------
def _blackboard_file(brain_path: Path, section: str) -> Path:
if section in BUILTIN_SECTIONS:
return brain_path / "blackboard" / f"{section}.jsonl"
(brain_path / "blackboard" / "custom").mkdir(parents=True, exist_ok=True)
return brain_path / "blackboard" / "custom" / f"{section}.jsonl"
def append_blackboard(brain_path: Path, section: str, key: str, value: str,
author: str, committed_by: str,
proposal_id: str | None, version: int,
entry_id: str | None = None) -> str:
"""Append a blackboard entry to ``blackboard/<section>.jsonl``.
Returns the entry id. Atomic per-brain lock.
"""
eid = entry_id or os.urandom(8).hex()
obj = {
"id": eid,
"section": section,
"key": key,
"value": value,
"author_agent_id": author,
"committed_by_agent_id": committed_by,
"proposal_id": proposal_id,
"version": version,
"created_at": _iso_now(),
}
lk = _lock_for(brain_path)
with lk:
p = _blackboard_file(brain_path, section)
p.parent.mkdir(parents=True, exist_ok=True)
with p.open("a", encoding="utf-8") as f:
f.write(_compact_json(obj) + "\n")
return eid
def append_event(brain_path: Path, key: str, value: str,
author: str | None = None, entry_id: str | None = None) -> int:
"""Append to events.jsonl; returns the new event id (monotonic per brain).
Phase 3: rotates events.jsonl when it exceeds ``MAX_EVENTS_LINES`` lines.
The rotated file is moved to ``events.<timestamp>.jsonl`` (archived) and a
fresh ``events.jsonl`` starts. Event ids continue monotonically across
rotations (the max id is preserved). Archived files are kept indefinitely
(they're human-readable history); a future sweeper can prune by age.
Phase 5 optimization: caches the max_id per brain path in a module-level
dict so we don't re-read + re-parse the whole file on every append. The
cache is invalidated on rotation. Falls back to a full scan if the cache
is cold.
"""
lk = _lock_for(brain_path)
with lk:
p = brain_path / "blackboard" / "events.jsonl"
p.parent.mkdir(parents=True, exist_ok=True)
# Phase 5 optimization: use cached max_id if available (audit M3).
# Only do the full scan on first touch or after a rotation.
cache_key = str(brain_path.resolve())
max_id = _event_id_cache.get(cache_key, -1)
lines: list[str] = []
if max_id < 0:
# cold cache — scan the file once
if p.is_file():
raw = p.read_text(encoding="utf-8").splitlines()
for line in raw:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
if isinstance(obj.get("id"), int) and obj["id"] > max_id:
max_id = obj["id"]
except Exception:
continue
lines.append(line)
else:
max_id = 0
new_id = max_id + 1
obj = {
"id": new_id,
"key": key,
"value": value,
"author_agent_id": author,
"created_at": _iso_now(),
}
if entry_id:
obj["entry_id"] = entry_id
new_line = _compact_json(obj)
# rotation: if over the cap, archive + start fresh
if max_id > 0 and max_id % MAX_EVENTS_LINES == 0 and max_id > 0:
archive = brain_path / "blackboard" / f"events.{_iso_now().replace(':', '').replace('-', '')}.jsonl"
try:
if p.is_file():
p.rename(archive)
except Exception:
pass # archive failure must never block the append
# append a single line (O(1), not rewrite)
with p.open("a", encoding="utf-8") as f:
f.write(new_line + "\n")
# update the cache
_event_id_cache[cache_key] = new_id
return new_id
# Phase 5 optimization: cache max event id per brain path (audit M3).
# Keyed by resolved brain_path string. Invalidated on rotation.
_event_id_cache: dict[str, int] = {}
# Phase 3 — events.jsonl rotation cap. When events.jsonl exceeds this many
# lines, the older events are archived to events.<timestamp>.jsonl and a fresh
# events.jsonl starts. Ids continue monotonically across rotations.
MAX_EVENTS_LINES = int(os.environ.get("CONSCIOUS_MAX_EVENTS_LINES", "10000"))
def append_proposal(brain_path: Path, proposal: dict) -> None:
"""Append a proposal to proposals/queue.jsonl."""
lk = _lock_for(brain_path)
with lk:
p = brain_path / "proposals" / "queue.jsonl"
p.parent.mkdir(parents=True, exist_ok=True)
with p.open("a", encoding="utf-8") as f:
f.write(_compact_json(proposal) + "\n")
# ---------------------------------------------------------------------------
# agent profiles
# ---------------------------------------------------------------------------
def write_agent_profile(brain_path: Path, agent: dict) -> None:
"""Write/overwrite agents/<id>.md."""
p = brain_path / "agents" / f"{agent['id']}.md"
p.parent.mkdir(parents=True, exist_ok=True)
subs = agent.get("subscribed_events") or []
if isinstance(subs, str):
try:
subs = json.loads(subs)
except Exception:
subs = []
body = [
f"# Agent: {agent['id']}",
"",
f"- **Role:** {agent.get('role', '')}",
f"- **Model:** {agent.get('model', '')}",
f"- **Tier:** {agent.get('tier', '')}",
f"- **Status:** {agent.get('status', 'idle')}",
f"- **Worktree:** {agent.get('worktree_path') or '(Phase 2)'}",
f"- **Branch:** {agent.get('branch') or '(Phase 2)'}",
f"- **Parent:** {agent.get('parent_agent_id') or 'null'}",
f"- **Orchestrator:** {'yes' if agent.get('is_orchestrator') else 'no'}",
f"- **Subscribed events:** {json.dumps(subs)}",
f"- **Created:** {agent.get('created_at', '')}",
"",
"## Recent activity (last 20)",
"",
]
log_path = brain_path / "agents" / f"{agent['id']}.log"
if log_path.is_file():
lines = log_path.read_text(encoding="utf-8").splitlines()[-20:]
body.extend(lines)
p.write_text("\n".join(body), encoding="utf-8")
def append_agent_log(brain_path: Path, agent_id: str, entry: str) -> None:
"""Append a line to agents/<id>.log."""
p = brain_path / "agents" / f"{agent_id}.log"
p.parent.mkdir(parents=True, exist_ok=True)
lk = _lock_for(brain_path)
with lk:
with p.open("a", encoding="utf-8") as f:
f.write(f"[{_iso_now()}] {entry}\n")
# ---------------------------------------------------------------------------
# drawer
# ---------------------------------------------------------------------------
def create_drawer_entry(brain_path: Path, invoke_id: str, request: dict) -> Path:
"""Create drawer/<invoke-id>/ with request.json, empty result.md, files/, status.json."""
d = brain_path / "drawer" / invoke_id
(d / "files").mkdir(parents=True, exist_ok=True)
(d / "request.json").write_text(_pretty_json(request), encoding="utf-8")
(d / "result.md").write_text("(pending)", encoding="utf-8")
status = {
"status": "pending",
"started_at": _iso_now(),
"completed_at": None,
"error": None,
}
(d / "status.json").write_text(_pretty_json(status), encoding="utf-8")
return d
def complete_drawer_entry(brain_path: Path, invoke_id: str, result: str,
status: str, error: str | None = None) -> None:
"""Write result.md and update status.json for a drawer entry."""
d = brain_path / "drawer" / invoke_id
if not d.is_dir():
return
(d / "result.md").write_text(result if result else "(no output)", encoding="utf-8")
obj = {
"status": status,
"started_at": _iso_now(),
"completed_at": _iso_now(),
"error": error,
}
(d / "status.json").write_text(_pretty_json(obj), encoding="utf-8")
def read_drawer(brain_path: Path, invoke_id: str) -> dict | None:
"""Read a single drawer entry; returns None if missing."""
d = brain_path / "drawer" / invoke_id
if not d.is_dir():
return None
out: dict[str, Any] = {"invoke_id": invoke_id}
req_p = d / "request.json"
if req_p.is_file():
try:
out["request"] = json.loads(req_p.read_text(encoding="utf-8"))
except Exception:
out["request"] = {}
res_p = d / "result.md"
if res_p.is_file():
out["result"] = res_p.read_text(encoding="utf-8")
else:
out["result"] = ""
st_p = d / "status.json"
if st_p.is_file():
try:
out["status"] = json.loads(st_p.read_text(encoding="utf-8"))
except Exception:
out["status"] = {}
files_dir = d / "files"
out["files"] = [p.name for p in files_dir.iterdir()] if files_dir.is_dir() else []
return out
def list_drawer(brain_path: Path, limit: int = 20) -> list[dict]:
"""List recent drawer entries (sorted by mtime desc). Each entry contains
request summary + status, not the full result text."""
d = brain_path / "drawer"
if not d.is_dir():
return []
items: list[dict] = []
for sub in d.iterdir():
if not sub.is_dir():
continue
full = read_drawer(brain_path, sub.name)
if not full:
continue
req = full.get("request", {}) or {}
result_text = full.get("result", "") or ""
st = full.get("status", {}) or {}
items.append({
"invoke_id": sub.name,
"from": req.get("from_agent_id"),
"to": req.get("to_agent_id"),
"kind": req.get("kind"),
"task": req.get("task", ""),
"status": st.get("status", "pending"),
"summary": result_text[:200],
"created_at": st.get("started_at"),
})
items.sort(key=lambda x: x.get("created_at") or "", reverse=True)
return items[:limit]
# ---------------------------------------------------------------------------
# regenerated views
# ---------------------------------------------------------------------------
def regenerate_conscious_md(brain_path: Path, conscious: dict,
agents: list[dict], recent_decisions: list[dict],
open_questions: list[dict], open_tasks: list[dict]) -> None:
"""Rewrite CONSCIOUS.md from current state."""
orch = next((a for a in agents if a.get("is_orchestrator")), None)
lines = [
f"# Conscious: {conscious.get('title', 'Untitled')}",
"",
f"- **Goal:** see goal.md",
f"- **Orchestrator:** {orch['id'] if orch else '(none)'} ({orch['model'] if orch else ''})",
f"- **Agents:** {len(agents)} active",
f"- **Created:** {conscious.get('created_at', '')}",
f"- **Cost ceiling:** ${conscious.get('cost_ceiling_usd', 0):.2f} (0 = infinite)",
f"- **Brain commit policy:** {conscious.get('brain_commit_policy', 'on')}",
"",
"## Agents",
]
for a in agents:
lines.append(f"- {a['id']}{a.get('role', '')}{a.get('model', '')}{a.get('status', 'idle')}")
lines += ["", "## Recent decisions (last 10)"]
if recent_decisions:
for d in recent_decisions[-10:]:
lines.append(f"- [{d.get('created_at', '')}] {d.get('author_agent_id', '?')}: {d.get('key', '')}{str(d.get('value', ''))[:120]}")
else:
lines.append("- (none)")
lines += ["", "## Open questions"]
if open_questions:
for q in open_questions[-10:]:
lines.append(f"- [{q.get('created_at', '')}] {q.get('author_agent_id', '?')}: {str(q.get('value', ''))[:200]}")
else:
lines.append("- (none)")
lines += ["", "## Open tasks"]
if open_tasks:
for t in open_tasks[-10:]:
lines.append(f"- [{t['id']}] {t.get('title', '')} — assignee: {t.get('assignee_agent_id') or '(unassigned)'} — status: {t.get('status', 'pending')}")
else:
lines.append("- (none)")
(brain_path / "CONSCIOUS.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
def regenerate_plan_json(brain_path: Path, conscious_id: str, tasks: list[dict]) -> None:
"""Rewrite plan.json from the task list."""
# compute version = max version seen + 1, or len(tasks) as a monotonic proxy
version = 0
for t in tasks:
v = t.get("version", 0) or 0
if isinstance(v, int) and v > version:
version = v
version = max(version, len(tasks))
obj = {
"conscious_id": conscious_id,
"version": version + 1,
"tasks": [
{
"id": t["id"],
"title": t.get("title", ""),
"assignee_agent_id": t.get("assignee_agent_id"),
"status": t.get("status", "pending"),
"depends_on": _parse_json_list(t.get("depends_on", "[]")),
"cost_ceiling_usd": t.get("cost_ceiling_usd"),
"created_at": t.get("created_at"),
"updated_at": t.get("updated_at"),
}
for t in tasks
],
"updated_at": _iso_now(),
}
(brain_path / "plan.json").write_text(_pretty_json(obj), encoding="utf-8")
def _parse_json_list(raw: Any) -> list:
if isinstance(raw, list):
return raw
if isinstance(raw, str):
try:
v = json.loads(raw)
return v if isinstance(v, list) else []
except Exception:
return []
return []
# ---------------------------------------------------------------------------
# git commit policy
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Phase 5 security: pre-commit secret scan for .brain/ (TIER3_PLAN §17)
# ---------------------------------------------------------------------------
# Patterns that indicate a leaked credential. If any file in .brain/ matches,
# the commit + HF upload is blocked and a `brain.scan.blocked` event is appended.
_SECRET_PATTERNS = [
re.compile(r"ghp_[A-Za-z0-9]{36}"), # GitHub PAT (classic)
re.compile(r"github_pat_[A-Za-z0-9_]{82}"), # GitHub PAT (fine-grained)
re.compile(r"hf_[A-Za-z0-9]{34,}"), # HF token
re.compile(r"AKIA[0-9A-Z]{16}"), # AWS access key
re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH |PGP )?PRIVATE KEY-----"), # private key
re.compile(r"xox[bpoa]-[A-Za-z0-9-]{10,}"), # Slack token
re.compile(r"sk-[A-Za-z0-9]{20,}"), # OpenAI / generic API key
]
def scan_for_secrets(brain_path: Path) -> list[dict]:
"""Walk .brain/ and scan every file for known secret patterns.
Returns a list of ``{file, pattern, line}`` dicts for each match.
Empty list = clean. The scan reads files as text (utf-8, errors='replace');
binary files are skipped. Does NOT recurse into ``.worktrees/``.
"""
findings: list[dict] = []
if not brain_path.is_dir():
return findings
for p in brain_path.rglob("*"):
if not p.is_file():
continue
# skip the worktrees dir (it's outside .brain/ but just in case)
if ".worktrees" in p.parts:
continue
try:
text = p.read_text(encoding="utf-8", errors="replace")
except Exception:
continue
for i, line in enumerate(text.splitlines(), 1):
for pat in _SECRET_PATTERNS:
if pat.search(line):
findings.append({
"file": str(p.relative_to(brain_path)),
"pattern": pat.pattern[:40],
"line": i,
})
break # one finding per line is enough
return findings
def commit_brain(brain_path: Path, workspace_path: Path,
event_type: str, summary: str, policy: str) -> bool:
"""If policy == 'on': scan for secrets, then stage .brain/, commit locally.
Phase 5 security: scans .brain/ for known secret patterns BEFORE staging.
If secrets are found, the commit is REFUSED and a ``brain.scan.blocked``
event is appended so the orchestrator sees it. Returns True if committed,
False if blocked or skipped.
The (token, repo_url) args from the Phase 1 signature are DROPPED (audit H3:
they were unused + a latent footgun). Pushes go through the caller's
``push_to_remote(user_id, workspace_id, branch)`` helper, never here.
Phase 5 robustness: failures are logged via ``oplog.log_event`` instead of
silently swallowed (audit L1).
"""
if policy != "on":
return False
# Security: scan for secrets before committing (TIER3_PLAN §17)
findings = scan_for_secrets(brain_path)
if findings:
# block the commit + log
try:
from oplog import log_event
log_event("brain_scan_blocked", brain_path=str(brain_path),
findings_count=len(findings),
files=[f["file"] for f in findings[:10]])
except Exception:
pass
return False
try:
import subprocess
msg = f"[conscious] {event_type}: {summary}"[:200]
subprocess.run(["git", "-C", str(workspace_path), "add", ".brain/"],
capture_output=True, text=True, timeout=30)
subprocess.run(["git", "-C", str(workspace_path), "commit", "-m", msg],
capture_output=True, text=True, timeout=30)
return True
except Exception as exc:
# log the failure instead of silently swallowing (audit L1)
try:
from oplog import log_event
log_event("brain_commit_failed", error=type(exc).__name__)
except Exception:
pass
return False
def brain_path_for_workspace(workspace_path: Path | str) -> Path:
"""Resolve the .brain/ path for a workspace."""
return Path(workspace_path) / ".brain"