Spaces:
Build error
Build error
File size: 57,726 Bytes
dbf9d04 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 |
import os
import re
import json
import base64
import mimetypes
import argparse
import pathlib
import io
import requests
from typing import List, Dict, Any, Optional, TypedDict, Literal, Tuple
from dataclasses import dataclass, field
from io import BytesIO
from PIL import Image
from uuid import uuid4
# --- LangGraph / LangChain ---
from langgraph.graph import StateGraph, END
from langgraph.checkpoint.memory import MemorySaver
# --- OpenAI / Azure ---
from openai import AzureOpenAI, OpenAI
from dotenv import load_dotenv
# --- HF Transformers & Diffusers (Local VLM) ---
import torch
from transformers import AutoModelForVision2Seq, AutoProcessor, AutoTokenizer, pipeline, BitsAndBytesConfig
import transformers.utils as _hf_utils
if not hasattr(_hf_utils, "FLAX_WEIGHTS_NAME"):
_hf_utils.FLAX_WEIGHTS_NAME = "flax_model.msgpack"
from diffusers import DiffusionPipeline
try:
import transformers.models.auto.video_processing_auto as _video_auto
if getattr(_video_auto, "VIDEO_PROCESSOR_MAPPING_NAMES", None) is None:
_video_auto.VIDEO_PROCESSOR_MAPPING_NAMES = {}
except Exception as _video_err:
print(f"Warning: unable to patch video processor registry: {_video_err}")
# --- Playwright (for screenshots) ---
from playwright.sync_api import sync_playwright
# --- Load Environment Variables ---
load_dotenv()
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ## SECTION 1: UNIFIED MODEL MANAGER (Singleton)
#
# Manages loading all models (Azure, Local VLM, Generator)
# to ensure they are only loaded into memory once.
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# --- Configs from all files ---
QWEN_VL_MODEL_NAME = os.getenv("QWEN_VL_MODEL", "Qwen/Qwen2.5-VL-7B-Instruct")
SD_GENERATOR_MODEL = os.getenv("SD_GENERATOR_MODEL", "segmind/tiny-sd")
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
DTYPE = torch.bfloat16 if torch.cuda.is_available() else torch.float32
class ModelManager:
"""Manages loading all models and API clients at startup."""
_instance = None
def __new__(cls, *args, **kwargs):
if not cls._instance:
cls._instance = super(ModelManager, cls).__new__(cls)
return cls._instance
def __init__(self):
if not hasattr(self, 'vlm_model'): # Initialize only once
print("Initializing and loading all models and clients...")
# 1. Configure Azure Client
self.AZURE_ENDPOINT = os.getenv("ENDPOINT_URL", "")
self.AZURE_API_KEY = os.getenv("AZURE_OPENAI_API_KEY", "")
if not self.AZURE_API_KEY or not self.AZURE_ENDPOINT:
print("Warning: AZURE_OPENAI_API_KEY or ENDPOINT_URL not set.")
else:
self.azure_client = AzureOpenAI(
azure_endpoint=self.AZURE_ENDPOINT,
api_key=self.AZURE_API_KEY,
api_version="2024-10-21"
)
print("AzureOpenAI client loaded.")
# 2. Configure OpenAI Client (for edit_node_tool)
try:
self.OPENAI_API_KEY = os.environ["OPENAI_API_KEY"]
self.openai_client = OpenAI(api_key=self.OPENAI_API_KEY)
print("OpenAI client loaded.")
except KeyError:
print("Warning: OPENAI_API_KEY not set. GPT editor tool will not work.")
self.openai_client = None
# 3. Configure and load the Local VLM (Qwen)
print(f"Loading local VLM: {QWEN_VL_MODEL_NAME}...")
quantization_config = BitsAndBytesConfig(
load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_quant_type="nf4"
)
self.vlm_processor = AutoProcessor.from_pretrained(QWEN_VL_MODEL_NAME, trust_remote_code=True)
self.vlm_model = AutoModelForVision2Seq.from_pretrained(
QWEN_VL_MODEL_NAME,
torch_dtype=DTYPE,
device_map="auto",
quantization_config=quantization_config,
trust_remote_code=True
)
print("Local VLM (Qwen) loaded.")
# 4. Configure and load the Generator
print(f"Loading image generator: {SD_GENERATOR_MODEL}...")
self.generator_pipe = DiffusionPipeline.from_pretrained(
SD_GENERATOR_MODEL, torch_dtype=DTYPE
)
self.generator_pipe.enable_model_cpu_offload()
print("Generator loaded.")
print("All models and clients loaded and ready.")
def get_azure_client(self) -> AzureOpenAI:
if not hasattr(self, 'azure_client'):
raise RuntimeError("Azure client not initialized. Set AZURE_OPENAI_API_KEY and ENDPOINT_URL.")
return self.azure_client
def get_openai_client(self) -> OpenAI:
if not hasattr(self, 'openai_client') or self.openai_client is None:
raise RuntimeError("OpenAI client not initialized. Set OPENAI_API_KEY.")
return self.openai_client
# --- VLM Chat (in asset tool) ---
def chat_vlm(self, messages, temperature=0.2, max_new_tokens=2048):
gen_kwargs = {"do_sample": temperature > 0, "max_new_tokens": max_new_tokens}
if temperature > 0:
gen_kwargs["temperature"] = temperature
inputs = self.vlm_processor.apply_chat_template(
messages, add_generation_prompt=True, tokenize=True, return_tensors="pt", return_dict=True
).to(self.vlm_model.device)
with torch.no_grad():
output_ids = self.vlm_model.generate(**inputs, **gen_kwargs)
gen_only = [out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs["input_ids"], output_ids)]
return self.vlm_processor.batch_decode(gen_only, skip_special_tokens=True, clean_up_tokenization_spaces=True)[0]
def chat_llm(self, prompt: str):
messages = [{"role": "user", "content": [{"type": "text", "text": prompt}]}]
return self.chat_vlm(messages, temperature=0.1, max_new_tokens=1024)
# --- Generator (from asset_tool) ---
def generate_image(self, prompt: str) -> Image.Image:
print(f"Generating image with prompt: '{prompt}'")
return self.generator_pipe(prompt, num_inference_steps=4, guidance_scale=0.0).images[0]
# --- Azure Chat (from agent_azure_vlm) ---
def chat_complete_azure(self, deployment: str, messages: List[Dict[str, Any]],
temperature: float, max_tokens: int) -> str:
client = self.get_azure_client()
resp = client.chat.completions.create(
model=deployment,
messages=messages,
temperature=temperature,
max_tokens=max_tokens,
)
return (resp.choices[0].message.content or "").strip()
# --- Initialize models ONCE ---
models = ModelManager()
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ## SECTION 2: ASSET-FINDING TOOL
#
# This is the self-contained graph for finding/generating assets.
# It will be used as a tool by the "Brain" AND by the Azure pipeline.
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
### --- Utilities from asset_tool.py ---
def load_image(path: str) -> Image.Image:
return Image.open(path).convert("RGB")
def b64img(pil_img: Image.Image) -> str:
buf = io.BytesIO()
pil_img.save(buf, format="PNG")
return base64.b64encode(buf.getvalue()).decode("utf-8")
### --- State from asset_tool.py ---
class AssetGraphState(TypedDict):
"""State for the asset-finding subgraph."""
instructions: str
bounding_box: Tuple[int, int]
search_query: str
found_image_url: Optional[str]
final_asset_path: Optional[str]
### --- Nodes from asset_tool.py ---
def asset_prepare_search_query_node(state: AssetGraphState) -> dict:
print("---(Asset Tool) NODE: Prepare Search Query---")
prompt = f"""You are an expert at refining search queries. Extract only the essential visual keywords.
**CRITICAL INSTRUCTIONS:**
- DO NOT include words related to licensing.
- DO NOT include quotation marks.
User's request: "{state['instructions']}"
Respond with ONLY the refined search query."""
raw_query = models.chat_llm(prompt)
search_query = raw_query.strip().replace('"', '')
print(f"Prepared search query: '{search_query}'")
return {"search_query": search_query}
def asset_generate_image_node(state: AssetGraphState) -> dict:
print("---(Asset Tool) NODE: Generate Image---")
prompt = state["instructions"]
generated_image = models.generate_image(prompt)
output_dir = pathlib.Path("Outputs/Assets")
output_dir.mkdir(parents=True, exist_ok=True)
filename = f"generated_{uuid4()}.png"
full_save_path = output_dir / filename
generated_image.save(full_save_path)
print(f"Image generated and saved to {full_save_path}")
html_path = pathlib.Path("Assets") / filename
final_asset_path = str(html_path.as_posix())
return {"final_asset_path": final_asset_path}
def asset_download_and_resize_node(state: AssetGraphState) -> dict:
print("---(Asset Tool) NODE: Download and Resize---")
image_url = state.get("found_image_url")
try:
response = requests.get(image_url, timeout=10)
response.raise_for_status()
img = Image.open(BytesIO(response.content))
img.thumbnail(state['bounding_box'])
output_dir = pathlib.Path("Outputs/Assets")
output_dir.mkdir(parents=True, exist_ok=True)
filename = f"asset_{uuid4()}.png"
full_save_path = output_dir / filename
img.save(full_save_path)
print(f"Image saved and resized to {full_save_path}")
html_path = pathlib.Path("Assets") / filename
final_asset_path = str(html_path.as_posix())
return {"final_asset_path": final_asset_path}
except Exception as e:
print(f"Error processing image: {e}")
return {"final_asset_path": None}
def asset_route_after_search(state: AssetGraphState) -> str:
if state.get("found_image_url"):
return "download_and_resize"
else:
print("Search failed. Routing to generate a new image.")
return "generate_image"
def asset_pexels_search_node(state: AssetGraphState) -> dict:
print("---(Asset Tool) TOOL: Searching Pexels---")
api_key = os.getenv("PEXELS_API_KEY")
search_query = state.get("search_query")
if not api_key:
print("Warning: PEXELS_API_KEY not set. Skipping search.")
return {"found_image_url": None}
if not search_query:
return {"found_image_url": None}
headers = {"Authorization": api_key}
params = {"query": search_query, "per_page": 1}
try:
response = requests.get("https://api.pexels.com/v1/search", headers=headers, params=params, timeout=10)
response.raise_for_status()
if response.json().get('photos'):
image_url = response.json()['photos'][0]['src']['original']
print(f"Found a candidate image: {image_url}")
return {"found_image_url": image_url}
except requests.exceptions.RequestException as e:
print(f"Pexels API Error: {e}")
return {"found_image_url": None}
### --- Graph Builder from asset_tool.py ---
def build_asset_graph():
workflow = StateGraph(AssetGraphState)
workflow.add_node("prepare_search_query", asset_prepare_search_query_node)
workflow.add_node("pexels_search", asset_pexels_search_node)
workflow.add_node("generate_image", asset_generate_image_node)
workflow.add_node("download_and_resize", asset_download_and_resize_node)
workflow.set_entry_point("prepare_search_query")
workflow.add_edge("prepare_search_query", "pexels_search")
workflow.add_conditional_edges("pexels_search", asset_route_after_search)
workflow.add_edge("generate_image", END)
workflow.add_edge("download_and_resize", END)
return workflow.compile()
# --- Compile the graph ---
asset_agent_app = build_asset_graph()
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ## SECTION 3: CODE EDITOR TOOL
#
# This is the self-contained graph for editing HTML.
# It will be used as a tool by the "Brain".
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
class CodeEditorState(TypedDict):
html_code: str
user_request: str
model_choice: Literal["gpt-4o-mini-2", "qwen-local"]
messages: list[str]
EDITOR_SYSTEM_PROMPT = """
You are an expert senior web developer specializing in HTML, CSS, and JavaScript.
Your task is to take an existing HTML file, a user's request for changes, and to output the *new, complete, and updated HTML file*.
**CRITICAL RULES:**
1. **Output ONLY the Code:** Your entire response MUST be *only* the raw, updated HTML code.
2. **No Conversation:** Do NOT include "Here is the updated code:", "I have made the following changes:", or any other explanatory text, comments, or markdown formatting.
3. **Return the Full File:** Always return the complete HTML file, from `<!DOCTYPE html>` to `</html>`, incorporating the requested changes. Do not return just a snippet.
"""
def _clean_llm_output(code: str) -> str:
"""Removes common markdown formatting."""
code = code.strip()
if code.startswith("```html"):
code = code[7:]
if code.endswith("```"):
code = code[:-3]
return code.strip()
def _call_gpt_editor(html_code: str, user_request: str, model: str) -> str:
"""Uses OpenAI (GPT) model."""
user_prompt = f"**User Request:**\n{user_request}\n\n**Original HTML Code:**\n```html\n{html_code}\n```\n\n**Your updated HTML Code:**"
try:
client = models.get_openai_client()
response = client.chat.completions.create(
model=model,
messages=[
{"role": "system", "content": EDITOR_SYSTEM_PROMPT},
{"role": "user", "content": user_prompt}
],
temperature=0.0,
max_tokens=8192,
)
edited_code = response.choices[0].message.content
return _clean_llm_output(edited_code)
except Exception as e:
print(f"Error calling OpenAI API: {e}")
return f"\n{html_code}"
def _call_qwen_editor(html_code: str, user_request: str) -> str:
"""Uses Local Qwen VLM."""
user_prompt = f"**User Request:**\n{user_request}\n\n**Original HTML Code:**\n```html\n{html_code}\n```\n\n**Your updated HTML Code:**"
messages = [
{"role": "system", "content": [{"type": "text", "text": EDITOR_SYSTEM_PROMPT}]},
{"role": "user", "content": [{"type": "text", "text": user_prompt}]}
]
try:
edited_code = models.chat_vlm(messages, temperature=0.0, max_new_tokens=8192)
return _clean_llm_output(edited_code)
except Exception as e:
print(f"Error calling local Qwen VLM: {e}")
return f"\n{html_code}"
def node_edit_code(state: CodeEditorState) -> dict:
print("---(Edit Tool) NODE: Edit Code---")
html_code, user_request, model_choice = state['html_code'], state['user_request'], state['model_choice']
messages = state.get('messages', [])
if not user_request:
return {"messages": messages + ["No user request provided. Skipping edit."]}
try:
if "gpt" in model_choice.lower():
new_html_code = _call_gpt_editor(html_code, user_request, model_choice)
else:
new_html_code = _call_qwen_editor(html_code, user_request)
msg = f"Code edit complete using {model_choice}."
print(msg)
return {"html_code": new_html_code, "user_request": "", "messages": messages + [msg]}
except Exception as e:
error_msg = f"Error in code editing node: {e}"
print(error_msg)
return {"html_code": html_code, "messages": messages + [error_msg]}
def build_edit_graph():
workflow = StateGraph(CodeEditorState)
workflow.add_node("edit_code", node_edit_code)
workflow.set_entry_point("edit_code")
workflow.add_edge("edit_code", END)
return workflow.compile(checkpointer=MemorySaver())
# --- Compile the graph ---
edit_agent_app = build_edit_graph()
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ## SECTION 4: AZURE VLM PIPELINE (RE-ORDERED)
#
# This pipeline is reordered to be much faster.
# 1. CodeGen runs FIRST, creating placeholders.
# 2. A fast regex parser finds the placeholders.
# 3. Asset search runs.
# 4. A patcher node inserts the asset paths.
# 5. Scoring & Refinement run as normal.
#
# This completely removes the slow local VLM call from this graph.
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
## --- Helpers ---
_SCORE_KEYS = ["aesthetics","completeness","layout_fidelity","text_legibility","visual_balance"]
def _section(text: str, name: str) -> str:
pat = rf"{name}:\s*\n(.*?)(?=\n[A-Z_]+:\s*\n|\Z)"
m = re.search(pat, text, flags=re.S)
return m.group(1).strip() if m else ""
def _score_val(block: str, key: str, default: int = 0) -> int:
m = re.search(rf"{key}\s*:\s*(-?\d+)", block, flags=re.I)
try:
return int(m.group(1)) if m else default
except:
return default
def encode_image_to_data_url(path: str) -> str:
mime = mimetypes.guess_type(path)[0] or "image/png"
with open(path, "rb") as f:
b64 = base64.b64encode(f.read()).decode("utf-8")
return f"data:{mime};base64,{b64}"
def extract_html(text: str) -> str:
m = re.search(r"```html(.*?)```", text, flags=re.S|re.I)
if m:
return m.group(1).strip()
i = text.lower().find("<html")
return text[i:].strip() if i != -1 else text.strip()
### --- Prompts ---
RELDESC_SYSTEM = "You are a meticulous UI analyst who describes layouts as a single dense paragraph of relative relationships."
RELDESC_PROMPT = """
From the provided wireframe image, produce ONE detailed paragraph (no bullets, no lists, no headings, no JSON, no code)
that states the RELATIVE layout and styling of the page so a code generator can rebuild it.
Requirements for the paragraph:
- Mention the overall background color and the dominant text color (infer reasonable #HEX).
- Describe the NAV BAR first: its position (top row), left/center/right alignment of brand/logo and items, the item order,
which (if any) is emphasized/active, approximate pill/underline treatment, and colors for default vs active.
- Describe the CONTENT in reading order by rows:
* For each row, say HOW MANY items appear side-by-side and their approximate relative widths (e.g., equal thirds, 1/3–2/3).
* For “cards” or boxes: state the title text (or short descriptor), the presence of body text, any CTA (button/link) labels,
button shape (rounded/square), fill/outline, and inferred #HEX colors for surface, border, and CTA.
* Call out approximate spacing (e.g., tight/medium/roomy), gutters/gaps (px if you can), and typical radii (px).
- Describe the FOOTER last: alignment (center/left/right), text size relative to body, and background/text colors.
- Include one sentence on typography: font family category (system sans/serif), approximate base size (px), and headings hierarchy.
- Keep everything in ONE paragraph. Do not use line breaks except normal wrapping.
Return ONLY that single paragraph.
"""
BRIEF_SYSTEM = "You are a senior product designer who converts wireframes into precise UI design briefs."
BRIEF_PROMPT = """
Using the RELATIVE LAYL'OUT DESCRIPTION (authoritative for relative structure) and the wireframe image,
write a **UI DESIGN BRIEF** using EXACTLY these section headings, in this order. Be concise but specific. Infer reasonable hex colors.
If there is any conflict, prefer the wireframe image but keep structure consistent with the relative description.
### UI_SUMMARY
One paragraph that states page purpose and the major regions.
### COLOR_PALETTE
List 6–10 colors as `name: #HEX` including background, surface/card, text, muted text, primary,
secondary/accent, link, button-default, button-active.
### TYPOGRAPHY
Font family (system stack), base font-size (px), title sizes (h1/h2/h3 in px), and weight rules.
Line-heights.
### LAYOUT
Container max-width (px), global padding (px), section gaps (px), and the overall structure
(header/nav, content rows/columns, footer). State **how many items appear side-by-side** in each row
and at which breakpoint they stack.
### NAVBAR
Exact order of items, which one is ACTIVE, and pill styling (padding, radius, default vs active
background/text colors).
### CARDS
For each card in the content row: title text, body summary, CTA label and style (button/link),
card padding, radius, shadow, spacing between title/body/cta.
### RESPONSIVE_RULES
Breakpoints (sm/md/lg in px) and what changes at each (column stack rules, spacing adjustments).
### SPACING_AND_BORDERS
Numbers (px) for margins, gaps, radii used across elements.
Output ONLY the brief text with those headings (no code fences, no JSON).
"""
# *** NEW UPDATED CODE_PROMPT ***
CODE_SYSTEM = "You are a meticulous frontend engineer who writes clean, modern, responsive HTML+CSS."
CODE_PROMPT = """
Using the following **RELATIVE LAYOUT DESCRIPTION** and **UI DESIGN BRIEF**, generate a SINGLE, self-contained HTML document:
Requirements:
- Semantic tags: header/nav/main/section/footer.
- One <style> block; no external CSS/JS.
- Define CSS variables from the palette and use them consistently.
- Implement the layout: container max-width, gaps, grid columns, and stacking rules per breakpoints.
- **CRITICAL ASSET RULE**: If you need an image (logo, hero, card image, etc.), you MUST use a placeholder in this **exact** format:
<img src="placeholder" data-asset-id="a-unique-id-for-this-image" data-asset-description="a detailed description for an image search engine">
(Example: <img src="placeholder" data-asset-id="hero-image" data-asset-description="photo of a modern office building">)
- **DO NOT** use the `ASSET_PATHS` variable, it will be empty.
- Output ONLY valid HTML starting with <html> and ending with </html>.
"""
SCORING_RUBRIC = r"""
You are an experienced front-end engineer. Compare two images: (A) the original wireframe, and (B) the generated HTML rendering,
and read the HTML/CSS code used for (B).
Return a PLAIN-TEXT report with the following sections EXACTLY in this order
(no JSON, no code fences around the whole report):
SCORES:
aesthetics: <0-10>
completeness: <0-10>
layout_fidelity: <0-10> # be harsh; row alignment, relative sizes and positions must match A
text_legibility: <0-10>
visual_balance: <0-10>
aggregate: <float> # mean of the five scores
ISSUES_TOP3:
- <short, specific issue 1>
- <issue 2>
- <issue 3>
LAYOUT_DIFFS:
- component: <nav|grid|card[1]|card[2]|footer>
a_bbox_pct: [x,y,w,h] # approx percentages (0–100) of page width/height in A
b_bbox_pct: [x,y,w,h] # same for B
fix: <one sentence with exact px/cols/gaps>
CSS_PATCH:
```css
/* <= 40 lines, use existing selectors where possible; use px and hex colors */
.selector { property: value; }
/* ... */
```
HTML_EDITS:
- <one edit per line; selector + action, e.g., "add-class .card --class=wide":
- <allowed actions: move-before, move-after, insert-before, insert-after, set-attr, replace-text, add-class, remove-class>
REGENERATE_PROMPT:
<1–4 lines with exact grid, gaps (px), radii (px), hex colors, and font sizes to rebuild if needed>
FEEDBACK:
<one dense paragraph prioritizing layout_fidelity with exact px/cols/gaps/hex values>
"""
REFINE_SYSTEM = "You are a senior frontend engineer who strictly applies critique to improve HTML/CSS while matching the wireframe."
REFINE_PROMPT = """
You are given:
1) (A) the original wireframe image
2) The CURRENT HTML (single-file) that produced (B) the rendering
3) A critique ("feedback") produced by a rubric-based comparison of A vs B
Task:
- Produce a NEW single-file HTML that addresses EVERY feedback point while staying faithful to A.
- Fix layout fidelity (columns, spacing, alignment), completeness (ensure all components in A exist),
typography/contrast for legibility, and overall aesthetics and balance.
- Keep it self-contained (inline <style>; no external CSS/JS).
- Output ONLY valid HTML starting with <html> and ending with </html>.
"""
@dataclass
class CodeRefineState:
# CLI inputs
image_path: str
out_rel_desc: str
out_brief: str
out_html: str
vision_deployment: str
text_deployment: str
reldesc_tokens: int
brief_tokens: int
code_tokens: int
judge_tokens: int
temp: float
refine_max_iters: int
refine_threshold: int
shot_width: int
shot_height: int
# Runtime state
image_data_url: Optional[str] = None
rel_desc: Optional[str] = None
brief: Optional[str] = None
html: Optional[str] = None
current_iteration: int = 0
scores: Optional[Dict[str, Any]] = None
stop_refinement: bool = False
asset_plan: List[Dict[str, Any]] = field(default_factory=list)
asset_paths: Dict[str, str] = field(default_factory=dict)
messages: List[str] = field(default_factory=list)
def parse_text_report(report: str) -> Dict[str, Any]:
sb = _section(report, "SCORES")
scores = {k: _score_val(sb, k, 0) for k in _SCORE_KEYS}
m_agg = re.search(r"aggregate\s*:\s*([0-9]+(?:\.[0-9]+)?)", sb, flags=re.I)
aggregate = float(m_agg.group(1)) if m_agg else sum(scores.values())/5.0
css_patch = ""
css_match = re.search(r"CSS_PATCH:\s*```css\s+(.*?)\s+```", report, flags=re.S|re.I)
if css_match:
css_patch = css_match.group(1).strip()
html_edits = _section(report, "HTML_EDITS")
regenerate_prompt = _section(report, "REGENERATE_PROMPT")
feedback = _section(report, "FEEDBACK")
issues = _section(report, "ISSUES_TOP3")
layout_diffs = _section(report, "LAYOUT_DIFFS")
return {
"scores": scores, "aggregate": aggregate, "css_patch": css_patch, "html_edits": html_edits,
"regenerate_prompt": regenerate_prompt, "feedback": feedback, "issues_top3": issues,
"layout_diffs": layout_diffs, "raw": report,
}
def refine_with_feedback(vision_deployment: str, wireframe_image: str, current_html: str, feedback: str,
css_patch: str = "", html_edits: str = "", regenerate_prompt: str = "",
temperature: float = 0.12, max_tokens: int = 2200) -> str:
data_a = encode_image_to_data_url(wireframe_image)
refine_instructions = f"{REFINE_PROMPT.strip()}\n\nAPPLY THESE PATCHES EXACTLY:..." # (rest of prompt)
messages = [
{"role": "system", "content": REFINE_SYSTEM},
{"role": "user", "content": [
{"type": "image_url", "image_url": {"url": data_a}},
{"type": "text", "text": refine_instructions + "\n\nCURRENT_HTML:\n```html\n" + current_html + "\n```"}
]},
]
out = models.chat_complete_azure(vision_deployment, messages, temperature, max_tokens)
html = extract_html(out)
if "<html" not in html.lower():
html = f"<!DOCTYPE html>\n<html>\n<head><meta charset='utf-8'><title>Refined</title></head>\n<body>\n{html}\n</body>\n</html>"
return html
def node_stage0(state: CodeRefineState) -> CodeRefineState:
state.image_data_url = encode_image_to_data_url(state.image_path)
messages = [
{"role": "system", "content": RELDESC_SYSTEM},
{"role": "user", "content": [
{"type":"image_url", "image_url":{"url":state.image_data_url}},
{"type":"text", "text": RELDESC_PROMPT.strip()},
]},
]
state.rel_desc = models.chat_complete_azure(state.vision_deployment, messages, state.temp, state.reldesc_tokens)
pathlib.Path(state.out_rel_desc).parent.mkdir(parents=True, exist_ok=True)
with open(state.out_rel_desc, "w", encoding="utf-8") as f: f.write(state.rel_desc.strip())
state.messages.append("Stage0: Generated relative layout description.")
return state
def node_stage1(state: CodeRefineState) -> CodeRefineState:
messages = [
{"role": "system", "content": BRIEF_SYSTEM},
{"role": "user", "content": [
{"type":"image_url", "image_url":{"url":state.image_data_url}},
{"type":"text", "text": BRIEF_PROMPT.strip() + "\n\nRELATIVE LAYOUT DESCRIPTION:\n" + state.rel_desc.strip()},
]},
]
state.brief = models.chat_complete_azure(state.vision_deployment, messages, state.temp, state.brief_tokens)
pathlib.Path(state.out_brief).parent.mkdir(parents=True, exist_ok=True)
with open(state.out_brief, "w", encoding="utf-8") as f: f.write(state.brief)
state.messages.append("Stage1: Generated UI design brief.")
return state
def node_stage2(state: CodeRefineState) -> CodeRefineState:
messages = [
{"role": "system", "content": CODE_SYSTEM},
{"role": "user", "content": [
{"type":"text", "text": CODE_PROMPT.strip()},
{"type":"text", "text": "RELATIVE LAYOUT DESCRIPTION:\n" + state.rel_desc.strip()},
{"type":"text", "text": "UI DESIGN BRIEF:\n" + state.brief.strip()},
{"type":"text", "text": "ASSET_PATHS:\n{}"}, # Send empty
]},
]
raw = models.chat_complete_azure(state.text_deployment, messages, state.temp, state.code_tokens)
state.html = extract_html(raw)
# Note: We don't save the HTML yet, as it's not patched.
state.messages.append("Stage2: Generated HTML (with placeholders).")
return state
def node_plan_assets_from_html(state: CodeRefineState) -> CodeRefineState:
"""
Parses the generated HTML for placeholders and builds the asset_plan.
This replaces the slow VLM planner.
"""
print("---(Azure VLM) NODE: Planning assets from HTML placeholders---")
# Regex to find: <img ... data-asset-id="..." data-asset-description="...">
placeholder_regex = r'<img[^>]+data-asset-id="([^"]+)"[^>]+data-asset-description="([^"]+)"'
matches = re.findall(placeholder_regex, state.html, re.I)
asset_plan = []
for component_id, description in matches:
asset_plan.append({
"component_id": component_id,
"description": description,
"bounding_box": {"width": 512, "height": 512} # Use a default size
})
state.asset_plan = asset_plan
state.messages.append(f"Stage2.5: Planned {len(asset_plan)} assets from HTML.")
print(f"Asset plan: {asset_plan}")
return state
def node_stage1_find_assets(state: CodeRefineState) -> CodeRefineState:
print("---(Azure VLM) NODE: Finding Assets---")
if not state.asset_plan:
state.messages.append("Stage2.6: No assets to find.")
return state
current_asset_paths = {}
for asset_request in state.asset_plan:
component_id = asset_request.get('component_id')
desc = asset_request.get('description')
bbox = asset_request.get('bounding_box', {})
if not all([component_id, desc, bbox]): continue
print(f"-> Finding asset for '{component_id}': {desc}")
try:
width = int(bbox.get('width', 512))
height = int(bbox.get('height', 512))
except (ValueError, TypeError):
width, height = 512, 512
# Call the asset_agent_app
result = asset_agent_app.invoke({"instructions": desc, "bounding_box": (width, height)})
if final_path := result.get("final_asset_path"):
current_asset_paths[component_id] = final_path
msg = f"Asset resolved for {component_id}: {final_path}"
state.messages.append(msg); print(f" ✅ {msg}")
else:
msg = f"Asset process failed for {component_id}."
state.messages.append(msg); print(f" ❌ {msg}")
state.asset_paths = current_asset_paths
return state
# Patch HTML with found assets
def node_patch_html_with_assets(state: CodeRefineState) -> CodeRefineState:
"""
Replaces the placeholders in the HTML with the paths from asset_paths.
"""
print("---(Azure VLM) NODE: Patching HTML with assets---")
if not state.asset_paths:
state.messages.append("Stage2.7: No assets to patch.")
# Save the un-patched HTML
pathlib.Path(state.out_html).parent.mkdir(parents=True, exist_ok=True)
with open(state.out_html, "w", encoding="utf-8") as f: f.write(state.html)
state.messages.append(f"Saved un-patched HTML -> {state.out_html}")
return state
html = state.html
for component_id, new_path in state.asset_paths.items():
# Find the placeholder tag using the component_id and replace its src
# This regex finds the <img ... data-asset-id="component_id" ...>
# and replaces the src="placeholder" part.
patch_regex = rf'(<img[^>]+data-asset-id="{re.escape(component_id)}"[^>]+)src="placeholder"'
replace_with = rf'\1src="{new_path}"'
new_html, count = re.subn(patch_regex, replace_with, html, flags=re.I)
if count > 0:
html = new_html
state.messages.append(f"Patched {component_id} -> {new_path}")
else:
state.messages.append(f"Warning: Could not find placeholder for {component_id} to patch.")
state.html = html
# Save the *patched* HTML
pathlib.Path(state.out_html).parent.mkdir(parents=True, exist_ok=True)
with open(state.out_html, "w", encoding="utf-8") as f: f.write(state.html)
state.messages.append(f"Stage2.7: Saved patched HTML -> {state.out_html}")
return state
def node_stage3_score(state: CodeRefineState) -> CodeRefineState:
html_path = pathlib.Path(state.out_html)
shot_png_path = html_path.with_name(html_path.stem + f"_iter{state.current_iteration}.png")
with sync_playwright() as p:
browser = p.chromium.launch()
ctx = browser.new_context(viewport={"width": state.shot_width, "height": state.shot_height}, device_scale_factor=2.0)
page = ctx.new_page()
page.goto(pathlib.Path(state.out_html).resolve().as_uri())
page.wait_for_load_state("networkidle")
page.screenshot(path=shot_png_path, full_page=True)
ctx.close()
browser.close()
data_a = encode_image_to_data_url(state.image_path)
data_b = encode_image_to_data_url(shot_png_path)
messages = [
{"role": "system", "content": "Return the specified PLAIN-TEXT report exactly as instructed."},
{"role": "user", "content": [
{"type": "text", "text": SCORING_RUBRIC.strip()},
{"type": "image_url", "image_url":{"url": data_a}},
{"type": "image_url", "image_url":{"url": data_b}},
{"type": "text", "text": "HTML/CSS code used to produce image (B):\n" + state.html}
]},
]
resp = models.chat_complete_azure(state.vision_deployment, messages, 0.0, state.judge_tokens)
state.scores = parse_text_report(resp)
state.messages.append(f"Stage3: Scoring done (Iter {state.current_iteration}).")
min_score = min(int(state.scores["scores"][k]) for k in _SCORE_KEYS)
if min_score >= state.refine_threshold:
state.stop_refinement = True
return state
def node_refine_loop(state: CodeRefineState) -> CodeRefineState:
if state.stop_refinement or state.current_iteration >= state.refine_max_iters:
state.messages.append("Refinement loop ended.")
return state
state.current_iteration += 1
state.html = refine_with_feedback(
vision_deployment=state.vision_deployment,
wireframe_image=state.image_path,
current_html=state.html,
feedback=state.scores.get("feedback",""),
css_patch=state.scores.get("css_patch",""),
html_edits=state.scores.get("html_edits",""),
regenerate_prompt=state.scores.get("regenerate_prompt",""),
temperature=state.temp,
max_tokens=state.code_tokens
)
versioned_path = pathlib.Path(state.out_html).with_name(pathlib.Path(state.out_html).stem + f"_v{state.current_iteration}" + pathlib.Path(state.out_html).suffix)
with open(versioned_path, "w", encoding="utf-8") as f: f.write(state.html)
state.out_html = str(versioned_path) # Update state to use new path for next scoring
state.messages.append(f"Saved refined HTML v{state.current_iteration} -> {versioned_path}")
return state
def decide_next(state: CodeRefineState) -> str:
if not state.stop_refinement and state.current_iteration < state.refine_max_iters:
return "refine_loop"
return "end"
def decide_if_assets_found(state: CodeRefineState) -> str:
"""Checks the asset_plan to decide the next step."""
if state.asset_plan:
return "find_assets"
else:
# No assets found, skip finding and patching
return "patch_html"
def build_azure_vlm_graph():
workflow = StateGraph(CodeRefineState)
workflow.add_node("stage0", node_stage0)
workflow.add_node("stage1", node_stage1)
workflow.add_node("stage2", node_stage2)
workflow.add_node("plan_assets_from_html", node_plan_assets_from_html)
workflow.add_node("stage1_find_assets", node_stage1_find_assets)
workflow.add_node("patch_html", node_patch_html_with_assets)
workflow.add_node("stage3_score", node_stage3_score)
workflow.add_node("refine_loop", node_refine_loop)
workflow.set_entry_point("stage0")
workflow.add_edge("stage0", "stage1")
workflow.add_edge("stage1", "stage2")
workflow.add_edge("stage2", "plan_assets_from_html")
workflow.add_conditional_edges(
"plan_assets_from_html",
decide_if_assets_found,
{
"find_assets": "stage1_find_assets",
"patch_html": "patch_html" # Skip to patching (which will also skip)
}
)
workflow.add_edge("stage1_find_assets", "patch_html")
workflow.add_edge("patch_html", "stage3_score")
# Original refinement loop
workflow.add_edge("stage3_score", "refine_loop")
workflow.add_conditional_edges("refine_loop", decide_next, {"refine_loop": "stage3_score", "end": END})
return workflow.compile(checkpointer=MemorySaver())
# --- Compile the graph ---
azure_vlm_app = build_azure_vlm_graph()
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ## SECTION 5: MAIN "BRAIN" AGENT (Command Center)
#
# This new agent uses the local Qwen VLM to decide which
# pipeline to run using standard conditional routing.
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
class BrainState(TypedDict):
messages: List[Dict[str, Any]]
cli_args: argparse.Namespace
# New fields for routing
next_task: Optional[str] = None
task_args: Optional[Dict[str, Any]] = None
task_result: Optional[str] = None
# --- Pipeline Functions ---
def helper_run_azure_vlm_pipeline(image_path: str) -> str:
"""
Use this tool to generate a new HTML webpage from a wireframe image.
This runs the full Azure VLM pipeline, including asset finding and refinement.
Args:
image_path (str): The file path to the input wireframe image.
"""
print(f"--- BRAIN: Invoking Azure VLM Pipeline for {image_path} ---")
try:
# --- Define defaults INSIDE the function ---
default_out_html = "Outputs/default_vlm_output.html"
default_out_brief = "Outputs/default_vlm_brief.txt"
default_out_reldesc = "Outputs/default_vlm_reldesc.txt"
# --- Hardcode model choices ---
vision_deployment = "gpt-4.1-mini" # Using default
text_deployment = "gpt-4.1-mini" # Using default
pathlib.Path(default_out_html).parent.mkdir(parents=True, exist_ok=True)
state = CodeRefineState(
image_path=image_path,
out_rel_desc=default_out_reldesc,
out_brief=default_out_brief,
out_html=default_out_html,
vision_deployment=vision_deployment, # Hardcoded
text_deployment=text_deployment, # Hardcoded
# Hardcode pipeline defaults
reldesc_tokens=700,
brief_tokens=1100,
code_tokens=2200,
judge_tokens=900,
temp=0.12,
refine_max_iters=3,
refine_threshold=8,
shot_width=1536,
shot_height=900
)
run_id = f"wireframe-{uuid4()}"
config = {"configurable": {"thread_id": run_id}}
result = azure_vlm_app.invoke(state, config=config)
final_path = result.get('out_html', default_out_html)
return json.dumps({
"status": "success",
"message": "Azure VLM pipeline completed.",
"final_html_path": final_path,
"messages": result.get("messages", [])
})
except Exception as e:
print(f"Error in Azure VLM helper: {e}")
return json.dumps({"status": "error", "message": str(e)})
def helper_run_code_editor(html_path: str, edit_request: str) -> str:
"""
Use this tool to edit an existing HTML file based on a user's text request.
Args:
html_path (str): The file path to the HTML file you want to edit.
edit_request (str): The user's instruction (e.g., "Make the h1 tag blue").
"""
print(f"--- BRAIN: Invoking Code Editor for {html_path} ---")
try:
# --- Hardcode model choice ---
model_choice = "qwen-local"
p = pathlib.Path(html_path)
output_path = str(p.parent / f"{p.stem}_edited.html")
with open(html_path, "r", encoding="utf-8") as f:
original_html = f.read()
initial_state = {
"html_code": original_html,
"user_request": edit_request,
"model_choice": model_choice, # Hardcoded
"messages": []
}
config = {"configurable": {"thread_id": f"edit-thread-{uuid4()}"}}
final_state = edit_agent_app.invoke(initial_state, config=config)
new_html_code = final_state['html_code']
pathlib.Path(output_path).parent.mkdir(parents=True, exist_ok=True)
with open(output_path, "w", encoding="utf-8") as f:
f.write(new_html_code)
return json.dumps({
"status": "success",
"message": "Code editing complete.",
"final_html_path": output_path,
"messages": final_state.get("messages", [])
})
except Exception as e:
print(f"Error in Code Editor helper: {e}")
return json.dumps({"status": "error", "message": str(e)})
def helper_run_asset_search(description: str, width: int = 512, height: int = 512) -> str:
"""
Use this tool to find or generate a single image asset.
... (docstring args) ...
"""
print(f"--- BRAIN: Invoking Asset Search for '{description}' ---")
try:
result = asset_agent_app.invoke({"instructions": description, "bounding_box": (width, height)})
if final_path := result.get("final_asset_path"):
return json.dumps({
"status": "success",
"message": "Asset found/generated.",
"asset_path": final_path
})
else:
return json.dumps({"status": "error", "message": "Asset process failed."})
except Exception as e:
print(f"Error in Asset Search helper: {e}")
return json.dumps({"status": "error", "message": str(e)})
# --- List of all helper functions for the Brain ---
helper_functions = [
helper_run_azure_vlm_pipeline,
helper_run_code_editor,
helper_run_asset_search,
]
# --- Brain Agent Definition (Router) ---
class QwenRouterAgent:
def __init__(self, model_manager, functions, system_prompt=""):
self.model = model_manager
self.functions = {f.__name__: f for f in functions}
self.system_prompt = system_prompt
def __call__(self, state: BrainState):
messages = state['messages']
qwen_messages = []
if self.system_prompt:
qwen_messages.append({"role": "system", "content": [{"type": "text", "text": self.system_prompt}]})
for msg in messages:
qwen_messages.append({
"role": msg['role'],
"content": [{"type": "text", "text": msg['content']}]
})
last_user_message = messages[-1]['content']
# *** UPDATED PROMPT ***
router_prompt = f"""
You are a "command center" agent. Your job is to route a user's request to the correct function
by providing a single, valid JSON object.
**Function Schemas:**
1. **Generate a new page from an image:**
{{
"function_name": "helper_run_azure_vlm_pipeline",
"function_args": {{
"image_path": "<string: path to the input image>"
}}
}}
2. **Edit an existing HTML file:**
{{
"function_name": "helper_run_code_editor",
"function_args": {{
"html_path": "<string: path to the HTML file to edit>",
"edit_request": "<string: the user's full instruction for changes>"
}}
}}
3. **Find or generate a single image asset:**
{{
"function_name": "helper_run_asset_search",
"function_args": {{
"description": "<string: a detailed description of the image>",
"width": "<int, default: 512>",
"height": "<int, default: 512>"
}}
}}
4. **No function needed:**
{{
"function_name": "end",
"function_args": {{}}
}}
**Instructions:**
1. Analyze the "User Request" and "Context".
2. Choose the *one* function from the schemas above that best matches the request.
3. **CRITICAL RULE: `helper_run_azure_vlm_pipeline` *already includes* asset searching. If the user asks to "generate a page" or "build a wireframe", you MUST choose *only* that function. DO NOT choose `helper_run_asset_search` separately.**
4. Populate the `function_args` with the *exact argument names* from the schema.
5. Respond with ONLY the valid JSON object for your chosen function.
---
**User Request:** "{last_user_message}"
**Context:**
"""
cli_args = state['cli_args']
if cli_args.image:
router_prompt += f"- An image path was provided: {cli_args.image}\n"
if cli_args.html:
router_prompt += f"- An HTML path was provided: {cli_args.html}\n"
router_prompt += "\n**Your JSON Response:**"
print("--- BRAIN: Routing prompt ---")
vlm_response = self.model.chat_llm(router_prompt)
print(f"--- BRAIN: VLM Response ---\n{vlm_response}\n-------------------------")
try:
call_json = json.loads(vlm_response[vlm_response.find("{"):vlm_response.rfind("}")+1])
func_name = call_json.get("function_name")
func_args = call_json.get("function_args", {})
if func_name == "end" or func_name not in self.functions:
print("--- BRAIN: No operation selected. Ending task. ---")
return {"next_task": "end", "task_result": "No operation selected."}
# We just fill in any missing values from the CLI context.
if func_name == "helper_run_azure_vlm_pipeline":
func_args['image_path'] = func_args.get('image_path', cli_args.image)
elif func_name == "helper_run_code_editor":
func_args['html_path'] = func_args.get('html_path', cli_args.html)
func_args['edit_request'] = func_args.get('edit_request', last_user_message)
elif func_name == "helper_run_asset_search":
func_args['description'] = func_args.get('description', last_user_message)
func_args['width'] = func_args.get('width', 512)
func_args['height'] = func_args.get('height', 512)
return {
"next_task": func_name,
"task_args": func_args
}
except Exception as e:
print(f"--- BRAIN: Error parsing VLM response. Ending task. --- \n{e}")
return {"next_task": "end", "task_result": f"Error parsing VLM response: {e}"}
# --- Graph Nodes ---
def node_run_vlm_pipeline(state: BrainState) -> dict:
print("---(Brain Graph) NODE: node_run_vlm_pipeline ---")
args = state['task_args']
result = helper_run_azure_vlm_pipeline(**args)
return {"task_result": result, "messages": state['messages'] + [{"role": "assistant", "content": result}]}
def node_run_code_editor(state: BrainState) -> dict:
print("---(Brain Graph) NODE: node_run_code_editor ---")
args = state['task_args']
result = helper_run_code_editor(**args)
return {"task_result": result, "messages": state['messages'] + [{"role": "assistant", "content": result}]}
def node_run_asset_search(state: BrainState) -> dict:
print("---(Brain Graph) NODE: node_run_asset_search ---")
args = state['task_args']
result = helper_run_asset_search(**args)
return {"task_result": result, "messages": state['messages'] + [{"role": "assistant", "content": result}]}
# --- Router Function ---
def brain_router(state: BrainState) -> str:
"""Routes to the correct node based on the 'next_task' state."""
print(f"---(Brain Graph) ROUTER: Next task is '{state['next_task']}' ---")
if state['next_task'] == "helper_run_azure_vlm_pipeline":
return "run_vlm"
elif state['next_task'] == "helper_run_code_editor":
return "run_edit"
elif state['next_task'] == "helper_run_asset_search":
return "run_asset"
else:
return "end"
# --- Graph Builder ---
def build_brain_graph():
brain_agent_node = QwenRouterAgent(models, helper_functions)
workflow = StateGraph(BrainState)
# Add nodes
workflow.add_node("agent", brain_agent_node)
workflow.add_node("run_vlm", node_run_vlm_pipeline)
workflow.add_node("run_edit", node_run_code_editor)
workflow.add_node("run_asset", node_run_asset_search)
workflow.set_entry_point("agent")
# Add conditional router
workflow.add_conditional_edges(
"agent",
brain_router,
{
"run_vlm": "run_vlm",
"run_edit": "run_edit",
"run_asset": "run_asset",
"end": END
}
)
# Add final edges
workflow.add_edge("run_vlm", END)
workflow.add_edge("run_edit", END)
workflow.add_edge("run_asset", END)
return workflow.compile()
# --- Compile the brain graph ---
brain_app = build_brain_graph()
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ## SECTION 6: CLI RUNNER (MAIN) - SIMPLIFIED FOR TESTING
#
# This entrypoint is modified to run a full test suite.
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
def run_test(test_name: str, initial_state: BrainState):
"""Helper function to run a single test case against the brain."""
print("\n" + "="*70)
print(f"--- STARTING TEST: {test_name} ---")
print(f"--- User Prompt: {initial_state['messages'][0]['content']} ---")
run_id = f"brain-test-{uuid4()}"
config = {"configurable": {"thread_id": run_id}}
# Invoke the brain
final_state = brain_app.invoke(initial_state, config=config)
print("\n" + ("-"*25) + " BRAIN INVOCATION COMPLETE " + ("-"*25))
print("--- Final Task Result ---")
task_result_str = final_state.get('task_result', "No result found (task may have ended early).")
if task_result_str:
try:
output_json = json.loads(task_result_str)
print(json.dumps(output_json, indent=2))
except json.JSONDecodeError:
print(task_result_str)
else:
print("No final task result recorded.")
print("="*70)
def main():
"""
Main function modified to run a test suite for all 3 pipeline paths.
"""
# --- Test Data Setup (Simplified) ---
# For Test 1: Azure VLM Pipeline
cli_args_vlm = argparse.Namespace(
prompt="Generate a new HTML page from the wireframe at Images/2.png",
image="Images/2.png",
html=None
)
# For Test 1.5: Azure internal asset search
asset_test_image_path = "Images/asset_test.png"
cli_args_vlm_assets = argparse.Namespace(
prompt=f"Generate the page from {asset_test_image_path}, and find all image assets.",
image=asset_test_image_path,
html=None
)
# For Test 2: Code Editor Pipeline
test_edit_file = "Outputs/test_page_to_edit.html"
cli_args_edit = argparse.Namespace(
prompt="Change the title to 'Edited by Brain Agent' and make the h1 tag red.",
image=None,
html=test_edit_file
)
# For Test 3: Asset Search Pipeline
cli_args_asset = argparse.Namespace(
prompt="Find a high-quality photo of a 'modern office desk with a laptop'",
image=None,
html=None
)
# =================================================================
# --- TEST 1: AZURE VLM PIPELINE (Image-to-Code) ---
# =================================================================
initial_state_vlm = {
"messages": [{"role": "user", "content": cli_args_vlm.prompt}],
"cli_args": cli_args_vlm
}
if not pathlib.Path(cli_args_vlm.image).exists():
print(f"--- WARNING: Skipping Test 1 ---")
print(f"Test image not found at: {cli_args_vlm.image}")
else:
run_test("Test 1: Azure VLM Pipeline (Image-to-Code)", initial_state_vlm)
# =================================================================
# --- TEST 1.5: AZURE VLM PIPELINE (with Asset Search) ---
# =================================================================
initial_state_vlm_assets = {
"messages": [{"role": "user", "content": cli_args_vlm_assets.prompt}],
"cli_args": cli_args_vlm_assets
}
if not pathlib.Path(cli_args_vlm_assets.image).exists():
print(f"\n--- WARNING: Skipping Test 1.5 ---")
print(f"Asset test image not found at: {cli_args_vlm_assets.image}")
print("This is the specific test for the internal asset-search pipeline.")
else:
run_test("Test 1.5: Azure VLM Pipeline (with Asset Search)", initial_state_vlm_assets)
# =================================================================
# --- TEST 2: CODE EDITOR PIPELINE (Edit HTML) ---
# =================================================================
initial_state_edit = {
"messages": [{"role": "user", "content": cli_args_edit.prompt}],
"cli_args": cli_args_edit
}
run_test("Test 2: Code Editor Pipeline (qwen-local)", initial_state_edit)
# =================================================================
# --- TEST 3: ASSET SEARCH PIPELINE (Find Image) ---
# =================================================================
initial_state_asset = {
"messages": [{"role": "user", "content": cli_args_asset.prompt}],
"cli_args": cli_args_asset
}
run_test("Test 3: Asset Search Pipeline", initial_state_asset)
if __name__ == "__main__":
# Ensure output directories exist
pathlib.Path("Outputs/Assets").mkdir(parents=True, exist_ok=True)
pathlib.Path("Outputs/").mkdir(parents=True, exist_ok=True)
# --- Create a dummy HTML file for Test 2 ---
test_edit_file_path = "Outputs/test_page_to_edit.html"
dummy_html = """
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Original Test Title</title>
</head>
<body>
<h1>This is the original headline.</h1>
<p>This is a paragraph.</p>
</body>
</html>
"""
with open(test_edit_file_path, "w", encoding="utf-8") as f:
f.write(dummy_html)
print(f"Created dummy file for editing at: {test_edit_file_path}")
# Check for test image for Test 1
if not pathlib.Path("Images/2.png").exists():
print("\n--- WARNING ---")
print("Test 1 (Azure VLM) requires an image at 'Images/2.png'.")
print("Please add an image there or Test 1 will be skipped.")
print("---------------\n")
# Check for Test 1.5 image
if not pathlib.Path("Images/asset_test.png").exists():
print("\n--- WARNING ---")
print("Test 1.5 (Azure VLM + Asset Search) requires an image at 'Images/asset_test.png'.")
print("This test *specifically* verifies the internal asset search.")
print("Please add an image with clear image placeholders, or Test 1.5 will be skipped.")
print("---------------\n")
main() |