Spaces:
Running
Running
| """CloneForge — Gradio app. | |
| Tab 1 (Clone): image/webcam (+ optional extra views) -> agent swarm -> live | |
| transcript + multi-view mesh render + 3D preview + STL + quality vs ground truth. | |
| Tab 2 (Examples): curated reference objects with published ground truth; one click | |
| clones them and reports dimension/Chamfer accuracy (and the gap on complex parts). | |
| Tab 3 (Speed Race): same prompt on Cerebras vs OpenAI, live TTFT + tok/s. | |
| Run: python app.py | |
| """ | |
| from __future__ import annotations | |
| import os | |
| import sys | |
| import gradio as gr | |
| from PIL import Image, ImageDraw | |
| from cloneforge import examples, llm | |
| from cloneforge.orchestrator import clone_pipeline, refine_pipeline | |
| EXAMPLE_GOAL = "Clone this object as a 3D-printable model" | |
| llm.set_status_hook(lambda m: print(f"[status] {m}", file=sys.stderr)) | |
| MANIFEST = examples.load_manifest() | |
| def _uris(main_image, extra_files): | |
| uris = [] | |
| if main_image: | |
| uris.append(llm.encode_image(main_image)) | |
| for f in (extra_files or []): | |
| uris.append(llm.encode_image(f)) | |
| return uris[:5] # Gemma 4 limit | |
| def _draw_marker(path, xy): | |
| """Draw a red defect marker at pixel xy on a copy of the image. Returns (path, location-hint).""" | |
| im = Image.open(path).convert("RGB") | |
| w, h = im.size | |
| x, y = int(xy[0]), int(xy[1]) | |
| d = ImageDraw.Draw(im) | |
| r = max(10, int(min(w, h) * 0.05)) | |
| d.ellipse([x - r, y - r, x + r, y + r], outline=(255, 30, 30), width=max(3, r // 4)) | |
| d.line([x - r, y, x + r, y], fill=(255, 30, 30), width=2) | |
| d.line([x, y - r, x, y + r], fill=(255, 30, 30), width=2) | |
| out = path + ".marked.png" | |
| im.save(out) | |
| vert = "top" if y < h / 3 else "bottom" if y > 2 * h / 3 else "middle" | |
| horiz = "left" if x < w / 3 else "right" if x > 2 * w / 3 else "center" | |
| return out, f"{vert}-{horiz}" | |
| def _fmt_quality(q): | |
| if not q: | |
| return "" | |
| rows = [] | |
| if "dimension_score" in q: | |
| rows.append(f"| Dimension match | **{q['dimension_score']:.0%}** " | |
| f"(got {q['dims_got_mm']} vs {q['dims_target_mm']} mm) |") | |
| if "chamfer" in q: | |
| rows.append(f"| Chamfer (↓) | {q['chamfer']} |") | |
| rows.append(f"| Voxel IoU (↑) | {q.get('voxel_iou')} |") | |
| if "silhouette_iou" in q: | |
| vp = f" · best view {q['viewpoint']}" if q.get("viewpoint") else "" | |
| rows.append(f"| Silhouette match vs photo (↑) | **{q['silhouette_iou']:.0%}**{vp} |") | |
| if not rows: | |
| return "" | |
| return "### 📊 Accuracy vs ground truth\n| metric | value |\n|---|---|\n" + "\n".join(rows) | |
| def _outputs(st, n_uris=0, extra_summary=""): | |
| summary = f"**{st.total_calls} agent calls · {st.total_latency_s:.2f}s compute** {extra_summary}" | |
| return (st.transcript, st.render_png, st.glb_path, (st.code or ""), | |
| st.stl_path, _fmt_quality(st.quality), summary, st) | |
| async def run_clone(main_image, extra_files, goal, ex, best_of): | |
| uris = _uris(main_image, extra_files) | |
| if not uris: | |
| yield ([{"role": "assistant", "content": "Add a photo (upload/webcam) first."}], | |
| None, None, "", None, "", "", None) | |
| return | |
| goal = goal or EXAMPLE_GOAL | |
| target = ex.get("dims_mm") if ex else None | |
| ref = ex.get("reference_stl") if ex else None | |
| n = 4 if best_of else 1 | |
| async for st in clone_pipeline(uris, goal, target_dims_mm=target, reference_mesh=ref, n_candidates=n): | |
| yield _outputs(st, len(uris), f"({len(uris)} view{'s' if len(uris) > 1 else ''})") | |
| def on_mark(render_path, evt: gr.SelectData): | |
| """User clicked the MODEL RENDER → draw a marker there and remember its location.""" | |
| if not render_path: | |
| return None, None | |
| marked, loc = _draw_marker(render_path, evt.index) | |
| return marked, {"marked_path": marked, "loc": loc} | |
| async def run_correction(state, marker, text, ex): | |
| """One correction path: if the user marked a spot on the render, send the marked image + | |
| text as a localized fix; otherwise just apply the text request. Clears the marker after use. | |
| Output tuple has marker_state appended (last element).""" | |
| if state is None: | |
| yield (([{"role": "assistant", "content": "Clone something first, then ask for a correction."}], | |
| None, None, "", None, "", "", None) + (marker,)) | |
| return | |
| text = (text or "").strip() | |
| if marker: | |
| instr = (f"The user circled a problem area on the model render ({marker['loc']} region): " | |
| f"{text or 'fix this part of the model to better match the photo'}.") | |
| extra = [llm.encode_image(marker["marked_path"])] | |
| tag = f"(fixed {marker['loc']})" | |
| elif text: | |
| instr, extra, tag = text, None, "(corrected)" | |
| else: | |
| yield _outputs(state, extra_summary="(type a correction, or click the render to mark a spot)") + (marker,) | |
| return | |
| async for st in refine_pipeline(state, instr, extra_uris=extra, | |
| target_dims_mm=(ex or {}).get("dims_mm"), | |
| reference_mesh=(ex or {}).get("reference_stl")): | |
| yield _outputs(st, extra_summary=tag) + (None,) # clear marker once applied | |
| def _lane(provider: str): | |
| async def handler(prompt): | |
| prompt = prompt or "Explain how a 3D printer extrudes filament, in 5 sentences." | |
| async for acc, stats in llm.astream(provider, prompt): | |
| md = (f"**{stats['provider']}** · `{stats['model']}` \n" | |
| f"⏱ TTFT **{stats['ttft_ms']:.0f} ms** · " | |
| f"🚀 **{stats['tok_s']:.0f} tok/s** · {stats['elapsed_s']:.2f}s") | |
| yield [{"role": "assistant", "content": acc}], md | |
| return handler | |
| def build_ui(): | |
| with gr.Blocks(title="CloneForge") as demo: | |
| gr.Markdown("# ⚒️ CloneForge\n" | |
| "Real-time multimodal object-cloning agent swarm — **Gemma 4 31B on Cerebras**. " | |
| "Photo → vision → plan → generate → *visual* critique → printable STL.") | |
| ex_state = gr.State(None) | |
| clone_state = gr.State(None) | |
| marker_state = gr.State(None) | |
| with gr.Tab("Clone"): | |
| with gr.Row(): | |
| # --- input --- | |
| with gr.Column(scale=1): | |
| img = gr.Image(label="Object photo", sources=["upload", "webcam"], type="filepath", height=220) | |
| extra = gr.File(label="Extra views (optional: side/top)", | |
| file_count="multiple", file_types=["image"], type="filepath", height=90) | |
| goal = gr.Textbox(label="Goal", value=EXAMPLE_GOAL) | |
| best_of = gr.Checkbox(label="Best-of-4 (parallel candidates, higher quality)") | |
| run_btn = gr.Button("⚡ Clone it", variant="primary") | |
| summary = gr.Markdown() | |
| # --- agent swarm --- | |
| with gr.Column(scale=1): | |
| chat = gr.Chatbot(label="Agent swarm", height=560) | |
| # --- interactive workspace: render + 3D + refine/fix in one place --- | |
| with gr.Column(scale=1): | |
| render = gr.Image(label="Model render — click a spot to target a fix there", | |
| type="filepath", interactive=False, height=240) | |
| model3d = gr.Model3D(label="3D preview", display_mode="solid", height=220) | |
| correction_box = gr.Textbox( | |
| show_label=False, | |
| placeholder="Ask for a correction (e.g. make it 20% taller) — " | |
| "or click the render to target a spot, then describe the fix") | |
| correction_btn = gr.Button("🔁 Apply correction", variant="primary") | |
| stl = gr.File(label="Download STL", height=90) | |
| # --- compact bottom: validation + code --- | |
| with gr.Row(): | |
| quality = gr.Markdown() | |
| with gr.Accordion("Generated code", open=False): | |
| code = gr.Code(language="python") | |
| outs = [chat, render, model3d, code, stl, quality, summary, clone_state] | |
| run_btn.click(run_clone, [img, extra, goal, ex_state, best_of], outs) | |
| render.select(on_mark, [render], [render, marker_state]) | |
| correction_btn.click(run_correction, [clone_state, marker_state, correction_box, ex_state], | |
| outs + [marker_state]) | |
| with gr.Tab("Examples"): | |
| gr.Markdown("### Reference objects with ground truth\n" | |
| "**Standard parts** (washer/nut/die/LEGO) have published exact dimensions → " | |
| "numeric accuracy. **Real scans** (mug/teapot/panda, *Google Scanned Objects, " | |
| "CC-BY 4.0*) and the **gear** show the fidelity gap on complex geometry. " | |
| "Click a card to load it on the **Clone** tab, then press **⚡ Clone it**.") | |
| ex_note = gr.Markdown() | |
| gallery = gr.Gallery( | |
| value=[(it["preview"], f"{it['title']} · {it['category']}") for it in MANIFEST], | |
| columns=4, height=560, object_fit="contain", allow_preview=False, | |
| show_label=False) | |
| def pick(evt: gr.SelectData): | |
| it = MANIFEST[evt.index] | |
| tip = "Go to the **Clone** tab and press ⚡ Clone it." | |
| return (it["preview"], it["goal"], it, | |
| f"**Loaded: {it['title']}** — ground truth: {it['note']}. {tip}") | |
| gallery.select(pick, None, [img, goal, ex_state, ex_note]) | |
| with gr.Tab("Speed Race"): | |
| gr.Markdown("### Same prompt, two providers — watch the first token land.") | |
| prompt = gr.Textbox(label="Prompt", | |
| value="Explain how a 3D printer extrudes filament, in 5 sentences.") | |
| race_btn = gr.Button("🏁 Race", variant="primary") | |
| with gr.Row(): | |
| with gr.Column(): | |
| gr.Markdown("### ⚡ Cerebras · Gemma 4 31B") | |
| cb_stat = gr.Markdown() | |
| cb_chat = gr.Chatbot(height=320, show_label=False) | |
| with gr.Column(): | |
| gr.Markdown("### 🐢 OpenAI · gpt-5.4-mini") | |
| oa_stat = gr.Markdown() | |
| oa_chat = gr.Chatbot(height=320, show_label=False) | |
| race_btn.click(_lane("cerebras"), prompt, [cb_chat, cb_stat], concurrency_limit=None) | |
| race_btn.click(_lane("openai"), prompt, [oa_chat, oa_stat], concurrency_limit=None) | |
| demo.queue(default_concurrency_limit=None) | |
| return demo | |
| if __name__ == "__main__": | |
| build_ui().launch(theme=gr.themes.Soft()) | |