"""Multi-agent design loop on top of the existing chat_cad modeler. Three roles: 1. Planner decomposes a natural-language brief into modeling milestones. 2. Modeler runs CadQuery ops via the existing tool-use loop. 3. Critic renders the scene to a PNG, looks at it with Claude vision, says either 'done' or 'revise' with concrete feedback. The orchestrator iterates Modeler -> Critic until the critic accepts or max_iters is hit. Each iteration's feedback flows into the next modeler call. This file deliberately reuses TOOLS / SYSTEM_PROMPT from llm.py for the modeler role rather than duplicating them. """ from __future__ import annotations import base64 import io import json import os import struct from dataclasses import dataclass, field from typing import Any import numpy as np # Headless backend (no Tk / Qt). Must be set before importing pyplot. import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt from mpl_toolkits.mplot3d.art3d import Poly3DCollection from cad_engine import CadEngine from llm import TOOLS, SYSTEM_PROMPT, run_claude # ---------------- visual rendering ---------------- # def _read_stl_triangles(path: str) -> np.ndarray: """Return an (N, 3, 3) array of triangle vertices. Handles binary STL. For empty/ASCII files, returns an empty array. """ with open(path, "rb") as f: head = f.read(80) if not head: return np.zeros((0, 3, 3)) n_bytes = f.read(4) if len(n_bytes) < 4: return np.zeros((0, 3, 3)) n_tri = struct.unpack(" str: """Render the current parts in `engine` to a PNG suitable for Claude vision. Each part is rendered in its manifest colour with black edges, on a soft iso view that approximates the in-browser viewport. """ manifest = engine.manifest() fig = plt.figure(figsize=(size[0] / 100, size[1] / 100), dpi=100) ax = fig.add_subplot(111, projection="3d") ax.set_proj_type("ortho") ax.set_facecolor("#1a1f26") fig.patch.set_facecolor("#171b22") all_pts = [] for p in manifest: name = p["name"] try: stl_path = engine.export_part_stl(name) except Exception: continue tris = _read_stl_triangles(stl_path) if len(tris) == 0: continue all_pts.append(tris.reshape(-1, 3)) coll = Poly3DCollection(tris, facecolor=p["color"], edgecolor="#0a0d12", linewidth=0.4, alpha=0.95) ax.add_collection3d(coll) if all_pts: pts = np.vstack(all_pts) mn, mx = pts.min(0), pts.max(0) ctr = (mn + mx) / 2 rng = (mx - mn).max() * 0.6 or 10.0 ax.set_xlim(ctr[0] - rng, ctr[0] + rng) ax.set_ylim(ctr[1] - rng, ctr[1] + rng) ax.set_zlim(ctr[2] - rng, ctr[2] + rng) else: ax.text(0.5, 0.5, 0.5, "(empty scene)", color="#8a8f99") ax.view_init(elev=25, azim=-55) # match in-browser default-ish for axis in (ax.xaxis, ax.yaxis, ax.zaxis): axis.pane.fill = False axis.pane.set_edgecolor("#2a2f37") ax.tick_params(colors="#6a7080", labelsize=7) ax.set_xlabel("X", color="#6a7080"); ax.set_ylabel("Y", color="#6a7080") ax.set_zlabel("Z", color="#6a7080") fig.tight_layout(pad=0.5) fig.savefig(output_path, dpi=100, facecolor=fig.get_facecolor()) plt.close(fig) return output_path def _png_to_b64(path: str) -> str: with open(path, "rb") as f: return base64.b64encode(f.read()).decode("ascii") # ---------------- planner ---------------- # PLANNER_SYSTEM = """You are the planning component of a multi-agent CAD design system. The user gives a natural-language brief for a mechanical part. Decompose the brief into 2-6 sequential MILESTONES. Each milestone should be one coherent piece of geometry: a base body, a feature like a slot or boss, a pattern, a fillet pass, etc. Order them so each builds on the previous. For each milestone produce: - name: short snake_case identifier - intent: one sentence describing what should exist in the scene after this step - success_criteria: one sentence on what a reviewer would look for to call it done Respond as STRICT JSON only, no prose, no markdown. Schema: {"plan":[{"name":"...","intent":"...","success_criteria":"..."},...]}""" def plan_design(client, model: str, brief: str, knowledge_block: str = "") -> list[dict]: system = PLANNER_SYSTEM if knowledge_block: system = PLANNER_SYSTEM + "\n\n" + knowledge_block resp = client.messages.create( model=model, max_tokens=1024, system=system, messages=[{"role": "user", "content": f"Brief:\n{brief}"}], ) text = "".join(b.text for b in resp.content if getattr(b, "type", None) == "text") # tolerate the model accidentally wrapping in ```json text = text.strip().lstrip("`").rstrip("`") if text.startswith("json"): text = text[4:].lstrip() try: return json.loads(text)["plan"] except Exception as e: raise RuntimeError(f"planner returned non-JSON: {e}\n---\n{text[:500]}") # ---------------- visual critic ---------------- # CRITIC_SYSTEM = """You are a senior mechanical engineer reviewing a CAD design in progress. You will be shown a rendered image of the current scene plus the original design brief and a description of what was just attempted. Your job is to decide whether the current scene satisfies what was attempted AND whether the overall brief is now complete. Be specific and terse. Do not hallucinate features you cannot see in the image. Respond as STRICT JSON only: { "verdict": "done" | "revise" | "continue", "feedback": "one or two sentences", "specific_changes": ["concrete change 1", "concrete change 2"] } verdict meanings: - "done" : the brief is fully satisfied; the design is finished. - "continue" : the last step looks OK; proceed to the next milestone. - "revise" : the last step is wrong or incomplete; redo it per specific_changes.""" # ---------------- DFM critic ---------------- # DFM_SYSTEM = """You are a senior design-for-manufacturing engineer reviewing parts in progress. You will be given numeric geometric measurements (bbox, volume, surface area, estimated wall thickness, aspect ratio) of the parts currently in the scene. Decide whether the current scene has serious manufacturing problems for common processes (CNC, 3D print FDM, sheet metal, casting). Be specific and quantitative. Do NOT speculate beyond what the numbers tell you. Respond as STRICT JSON only: { "verdict": "pass" | "warn" | "fail", "feedback": "one or two sentences", "issues": ["concrete issue 1 with the part name and number", "..."] } pass : looks manufacturable; nothing to fix. warn : minor concerns; design can ship but flag these. fail : a part as designed is unmanufacturable (wall too thin, sliver geometry, etc.).""" def _compute_dfm_findings(engine, names: list[str]) -> list[dict]: """Per-part geometric measurements relevant to manufacturability.""" findings: list[dict] = [] for name in names: if name not in engine.parts: continue try: shape = engine.parts[name].val() vol = float(shape.Volume()) try: surf = float(shape.Area()) except Exception: surf = 0.0 bb = shape.BoundingBox() dims = sorted([bb.xlen, bb.ylen, bb.zlen]) min_dim = dims[0]; max_dim = dims[2] aspect = (max_dim / min_dim) if min_dim > 1e-6 else float("inf") wall_est = (2.0 * vol / surf) if surf > 1e-6 else 0.0 warnings = [] if wall_est < 1.5: warnings.append(f"est. wall {wall_est:.2f} mm < 1.5 mm — fragile for most processes") if min_dim < 0.5: warnings.append(f"smallest bbox dim {min_dim:.2f} mm — likely sliver geometry") if aspect > 30: warnings.append(f"aspect ratio {aspect:.1f}:1 — long and thin, may warp on FDM") findings.append({ "part": name, "volume_mm3": round(vol, 2), "surface_mm2": round(surf, 2), "bbox_mm": [round(bb.xlen, 2), round(bb.ylen, 2), round(bb.zlen, 2)], "min_dim_mm": round(min_dim, 2), "max_dim_mm": round(max_dim, 2), "aspect_ratio": round(aspect, 1), "est_wall_thickness_mm": round(wall_est, 2), "warnings": warnings, }) except Exception as e: findings.append({"part": name, "error": str(e)}) return findings def critique_dfm(client, model: str, brief: str, engine) -> dict: """Geometric findings + LLM interpretation = DFM verdict.""" names = list(engine.parts.keys()) if not names: return {"verdict": "pass", "feedback": "(empty scene)", "issues": []} findings = _compute_dfm_findings(engine, names) user_msg = (f"BRIEF:\n{brief}\n\nGEOMETRIC FINDINGS:\n" f"{json.dumps(findings, indent=2)}\n\n" "Assess manufacturability.") resp = client.messages.create( model=model, max_tokens=600, system=DFM_SYSTEM, messages=[{"role": "user", "content": user_msg}]) text = "".join(b.text for b in resp.content if getattr(b, "type", None) == "text") text = text.strip().lstrip("`").rstrip("`") if text.startswith("json"): text = text[4:].lstrip() try: out = json.loads(text) out["findings"] = findings return out except Exception: return {"verdict": "warn", "feedback": text[:200], "issues": [], "findings": findings} # ---------------- Standards critic (RAG-driven) ---------------- # STANDARDS_SYSTEM = """You are an organizational-standards critic. You will be shown (1) the brief, (2) excerpts from the user's knowledge base describing their personal/team standards (e.g. 'always use M4 bolts', 'wall thickness >=3 mm for FDM'), and (3) what's currently in the scene. Decide whether the design respects the user's standards. Ignore items in the notes that have nothing to do with this brief. Be specific. Respond as STRICT JSON only: { "verdict": "pass" | "warn" | "fail", "feedback": "one or two sentences", "issues": ["concrete violation 1, citing the relevant note", "..."] }""" def critique_standards(client, model: str, brief: str, engine) -> dict: """RAG over knowledge base + LLM compliance judgment.""" if not hasattr(engine, "knowledge"): return {"verdict": "pass", "feedback": "(no knowledge base)", "issues": []} notes = engine.knowledge.search(brief, k=10) if engine.knowledge.notes else [] if not notes: return {"verdict": "pass", "feedback": "(no relevant notes in knowledge base)", "issues": []} notes_text = "\n".join(f"- {n['text']}" for n in notes) scene = engine.list_parts() user_msg = (f"BRIEF:\n{brief}\n\n" f"USER'S NOTES / STANDARDS:\n{notes_text}\n\n" f"CURRENT SCENE:\n{scene}\n\n" "Assess standards compliance.") resp = client.messages.create( model=model, max_tokens=600, system=STANDARDS_SYSTEM, messages=[{"role": "user", "content": user_msg}]) text = "".join(b.text for b in resp.content if getattr(b, "type", None) == "text") text = text.strip().lstrip("`").rstrip("`") if text.startswith("json"): text = text[4:].lstrip() try: return json.loads(text) except Exception: return {"verdict": "warn", "feedback": text[:200], "issues": []} VERIFY_SYSTEM = """You are a CAD verification agent. The user describes what they intended to build. A rendered image of the current scene is provided. Compare the image against the description. Decide whether the scene matches what was asked for and how closely. Be honest and specific — if a key feature is missing or wrong, say so. Output JSON exactly: { "match_score": 0-100 (integer; how closely the scene matches the intent), "verdict": "match" | "partial" | "mismatch", "what_matches": ["specific things the scene does have"], "what_is_missing_or_wrong": ["specific issues; empty if perfect"], "suggested_commands": ["zero or more chat_cad commands that would fix the gap, e.g. 'fillet base 2' or 'hole base 4'"] } No prose. JSON only. Use short, concrete strings. """ def verify_intent(client, model: str, user_description: str, image_path: str, parts_summary: str) -> dict: """Standalone verification: 'does the current scene match what I asked for?' — usable in any mode, not just Design Agent. """ img_b64 = _png_to_b64(image_path) msg = { "role": "user", "content": [ {"type": "image", "source": { "type": "base64", "media_type": "image/png", "data": img_b64}}, {"type": "text", "text": f"USER ASKED FOR:\n{user_description}\n\n" f"PARTS IN THE SCENE:\n{parts_summary}\n\n" "Verify the rendered scene against the user's intent."}, ], } resp = client.messages.create( model=model, max_tokens=700, system=VERIFY_SYSTEM, messages=[msg]) text = "".join(b.text for b in resp.content if getattr(b, "type", None) == "text") text = text.strip().lstrip("`").rstrip("`") if text.startswith("json"): text = text[4:].lstrip() try: return json.loads(text) except Exception as e: return {"match_score": 50, "verdict": "partial", "what_matches": [], "what_is_missing_or_wrong": [f"(verifier returned non-JSON: {e})"], "suggested_commands": []} def critique_visual(client, model: str, brief: str, milestone: dict, modeler_summary: str, image_path: str) -> dict: img_b64 = _png_to_b64(image_path) msg = { "role": "user", "content": [ {"type": "image", "source": { "type": "base64", "media_type": "image/png", "data": img_b64}}, {"type": "text", "text": f"BRIEF:\n{brief}\n\n" f"MILESTONE just attempted:\n" f" name: {milestone['name']}\n" f" intent: {milestone['intent']}\n" f" success_criteria: {milestone['success_criteria']}\n\n" f"MODELER summary:\n{modeler_summary}\n\n" "Assess the rendered scene."}, ], } resp = client.messages.create( model=model, max_tokens=512, system=CRITIC_SYSTEM, messages=[msg]) text = "".join(b.text for b in resp.content if getattr(b, "type", None) == "text") text = text.strip().lstrip("`").rstrip("`") if text.startswith("json"): text = text[4:].lstrip() try: return json.loads(text) except Exception as e: return {"verdict": "continue", "feedback": f"(critic returned non-JSON: {e})", "specific_changes": []} # ---------------- orchestrator ---------------- # @dataclass class AgentEvent: role: str # "planner" | "modeler" | "critic" | "system" kind: str # "log" | "milestone" | "tool" | "image" | "verdict" text: str data: dict = field(default_factory=dict) def to_json(self) -> dict: return {"role": self.role, "kind": self.kind, "text": self.text, "data": self.data} def design_loop(client, model: str, engine: CadEngine, brief: str, max_revises_per_milestone: int = 2) -> list[dict]: """Run the full planner -> [modeler -> critic]* loop. Returns the event log as plain dicts so it can be JSON-serialised directly to the browser. """ log: list[AgentEvent] = [] log.append(AgentEvent("system", "log", f"brief: {brief}")) # 0. retrieve from knowledge base (TF-IDF over user notes + past designs) knowledge_block = "" if hasattr(engine, "knowledge"): try: knowledge_block = engine.knowledge.context_block(brief, k=5) if knowledge_block: log.append(AgentEvent("system", "log", "retrieved knowledge notes: " + str(knowledge_block.count("\n- ")))) except Exception as e: log.append(AgentEvent("system", "log", f"knowledge lookup failed: {e}")) # 1. plan try: plan = plan_design(client, model, brief, knowledge_block) except Exception as e: log.append(AgentEvent("planner", "log", f"plan failed: {e}")) return [e.to_json() for e in log] log.append(AgentEvent("planner", "milestone", f"plan has {len(plan)} milestone(s)", data={"plan": plan})) modeler_history: list[dict] = [] last_render = os.path.join(engine.output_dir, "agent_view.png") for m_idx, milestone in enumerate(plan): log.append(AgentEvent("planner", "milestone", f"milestone {m_idx + 1}/{len(plan)}: {milestone['name']}", data=milestone)) for revise_idx in range(max_revises_per_milestone + 1): # 2. modeler instr = (f"Implement this milestone using the CAD tools:\n" f" name: {milestone['name']}\n" f" intent: {milestone['intent']}\n" f" success_criteria: {milestone['success_criteria']}\n") if revise_idx > 0: last_critic = log[-1].data instr += (f"\nPREVIOUS ATTEMPT WAS REJECTED. Critic feedback:\n" f" {last_critic.get('feedback', '')}\n" f" specific changes:\n" + "\n".join(f" - {c}" for c in last_critic.get( "specific_changes", []))) try: reply, ops = run_claude(client, model, modeler_history, engine, instr) except Exception as e: log.append(AgentEvent("modeler", "log", f"modeler call failed: {e}")) return [e.to_json() for e in log] log.append(AgentEvent("modeler", "tool", reply or "(no reply)", data={"ops": ops})) # 3. render try: render_scene_png(engine, last_render) except Exception as e: log.append(AgentEvent("system", "log", f"render failed: {e}")) break # 4. critic try: critique = critique_visual(client, model, brief, milestone, reply or "", last_render) except Exception as e: log.append(AgentEvent("critic", "log", f"critic call failed: {e}")) break log.append(AgentEvent("critic", "verdict", f"{critique.get('verdict','?')}: " f"{critique.get('feedback','')}", data=critique)) verdict = critique.get("verdict", "continue") if verdict == "done": log.append(AgentEvent("system", "log", "visual critic marked design complete")) _run_final_critics(client, model, engine, brief, log) _autosave_design(engine, brief, log) return [e.to_json() for e in log] if verdict == "continue": break # verdict == "revise" -> loop again within this milestone log.append(AgentEvent("system", "log", "all milestones complete (or revise cap hit)")) _run_final_critics(client, model, engine, brief, log) _autosave_design(engine, brief, log) return [e.to_json() for e in log] def _run_final_critics(client, model: str, engine: CadEngine, brief: str, log: list[AgentEvent]) -> None: """Run DFM + standards critics on the finished scene and append findings to the log. These are advisory — they don't loop back to the modeler in v1. """ # DFM try: dfm = critique_dfm(client, model, brief, engine) v = dfm.get("verdict", "?") fb = dfm.get("feedback", "") log.append(AgentEvent("critic", "verdict", f"DFM {v}: {fb}", data=dfm)) except Exception as e: log.append(AgentEvent("critic", "log", f"DFM critic failed: {e}")) # Standards (only if we have a non-empty knowledge base) try: std = critique_standards(client, model, brief, engine) v = std.get("verdict", "?") fb = std.get("feedback", "") log.append(AgentEvent("critic", "verdict", f"Standards {v}: {fb}", data=std)) except Exception as e: log.append(AgentEvent("critic", "log", f"Standards critic failed: {e}")) def _autosave_design(engine: CadEngine, brief: str, log: list[AgentEvent]) -> None: """Add a knowledge note summarising this design so future briefs can retrieve it. """ if not hasattr(engine, "knowledge"): return try: parts_list = list(engine.parts.keys()) if not parts_list: return # collect material info if known mats = {} for p in parts_list: try: mats[p] = engine.materials.material_of(p) except Exception: pass summary = ( f"BRIEF: {brief}\n" f"PARTS: {', '.join(parts_list)}\n" f"MATERIALS: {mats if mats else '(none assigned)'}\n" ) engine.knowledge.add(summary, tags=["design"], source="agent") except Exception: pass