zhangyaowei
update viewer remove background
d6ed415
#!/usr/bin/env python3
from __future__ import annotations
import json
import os
from functools import lru_cache
from pathlib import Path
from typing import Any
import numpy as np
import trimesh
import uvicorn
from fastapi import FastAPI, HTTPException
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
DEFAULT_DATA_ROOT = Path(__file__).resolve().parent / "demo_data"
REQUIRED_FILES = ("mesh.ply", "instance_seg_mesh.ply", "metadata.json")
HIDDEN_CLASS_NAMES = {"wall", "floor", "ceiling"}
app = FastAPI(title="SceneVerse++ Demo")
def list_scene_ids(data_root: Path) -> list[str]:
if not data_root.is_dir():
raise FileNotFoundError(f"Data root does not exist: {data_root}")
scene_ids: list[str] = []
for scene_dir in sorted(path for path in data_root.iterdir() if path.is_dir()):
if all((scene_dir / name).is_file() for name in REQUIRED_FILES):
scene_ids.append(scene_dir.name)
return scene_ids
def load_mesh(path: Path) -> trimesh.Trimesh:
mesh = trimesh.load(str(path), process=False, force="mesh")
if isinstance(mesh, trimesh.Scene):
geometries = [geom for geom in mesh.geometry.values() if isinstance(geom, trimesh.Trimesh)]
if not geometries:
raise ValueError(f"No mesh geometry found in {path}")
mesh = trimesh.util.concatenate(geometries)
if not isinstance(mesh, trimesh.Trimesh):
raise TypeError(f"Unsupported mesh type for {path}: {type(mesh)!r}")
return mesh
def sanitize_ids(point_ids: list[int], vertex_count: int) -> np.ndarray:
array = np.asarray(point_ids, dtype=np.int64)
return array[(array >= 0) & (array < vertex_count)]
def compute_instances(metadata: dict[str, Any], vertices: np.ndarray) -> list[dict[str, Any]]:
instances: list[dict[str, Any]] = []
for instance_id, payload in metadata.items():
class_name = str(payload.get("pred_class_name", "unknown")).strip()
if class_name.lower() in HIDDEN_CLASS_NAMES:
continue
point_ids = sanitize_ids(payload.get("point_ids", []), len(vertices))
if point_ids.size == 0:
continue
points = vertices[point_ids]
mins = points.min(axis=0)
maxs = points.max(axis=0)
center = ((mins + maxs) / 2.0).tolist()
size = (maxs - mins).tolist()
instances.append(
{
"instance_id": str(instance_id),
"class_name": class_name or "unknown",
"describe": payload.get("pred_describe", ""),
"class_id": payload.get("pred_class_id"),
"bbox_min": mins.tolist(),
"bbox_max": maxs.tolist(),
"bbox_center": center,
"bbox_size": size,
}
)
instances.sort(key=lambda item: int(item["instance_id"]) if item["instance_id"].isdigit() else item["instance_id"])
return instances
@lru_cache(maxsize=32)
def load_scene_bundle(scene_id: str) -> dict[str, Any]:
scene_dir = DEFAULT_DATA_ROOT / scene_id
if not scene_dir.is_dir():
raise FileNotFoundError(f"Scene directory does not exist: {scene_dir}")
paths = {name: scene_dir / name for name in REQUIRED_FILES}
for name, path in paths.items():
if not path.is_file():
raise FileNotFoundError(f"Missing required file {name} for scene {scene_id}")
mesh = load_mesh(paths["mesh.ply"])
with open(paths["metadata.json"], "r", encoding="utf-8") as f:
metadata = json.load(f)
instances = compute_instances(metadata, np.asarray(mesh.vertices))
return {
"scene_id": scene_id,
"mesh_url": f"/api/scenes/{scene_id}/files/mesh.ply",
"seg_mesh_url": f"/api/scenes/{scene_id}/files/instance_seg_mesh.ply",
"metadata_url": f"/api/scenes/{scene_id}/files/metadata.json",
"instances": instances,
}
INDEX_HTML = """
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>SceneVerse++ Demo</title>
<script type="importmap">
{
"imports": {
"three": "https://unpkg.com/three@0.161.0/build/three.module.js",
"three/addons/": "https://unpkg.com/three@0.161.0/examples/jsm/"
}
}
</script>
<style>
:root {
--bg: #f4f6f8;
--panel: #ffffff;
--line: #d9dee5;
--ink: #16202a;
--muted: #637282;
--accent: #0ea5e9;
--accent-strong: #ef4444;
}
* { box-sizing: border-box; }
body {
margin: 0;
font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
background: linear-gradient(180deg, #edf4f7 0%, #f7f8fa 100%);
color: var(--ink);
}
.page {
max-width: 1600px;
margin: 0 auto;
padding: 24px;
display: grid;
gap: 18px;
}
.header, .toolbar, .panel {
background: var(--panel);
border: 1px solid var(--line);
border-radius: 16px;
box-shadow: 0 10px 30px rgba(15, 23, 42, 0.06);
}
.header {
padding: 24px;
}
.header h1 {
margin: 0 0 8px;
font-size: 28px;
}
.header p {
margin: 0;
color: var(--muted);
line-height: 1.5;
}
.toolbar {
padding: 16px;
display: grid;
grid-template-columns: 320px 1fr 220px;
gap: 16px;
align-items: end;
}
.field {
display: grid;
gap: 8px;
}
.field label {
font-weight: 600;
font-size: 14px;
}
.field input, .field select, .field textarea, button {
width: 100%;
border: 1px solid var(--line);
border-radius: 12px;
padding: 11px 14px;
font: inherit;
background: #fff;
color: var(--ink);
}
.field textarea {
min-height: 140px;
resize: vertical;
}
button {
cursor: pointer;
background: linear-gradient(135deg, #0ea5e9 0%, #0284c7 100%);
color: #fff;
font-weight: 700;
border: 0;
}
.status {
color: var(--muted);
font-size: 14px;
padding: 0 4px;
}
.grid {
display: grid;
grid-template-columns: minmax(0, 1fr) minmax(0, 1fr);
gap: 18px;
}
.panel {
overflow: hidden;
min-height: 660px;
display: grid;
grid-template-rows: auto 1fr;
}
.panel-header {
padding: 14px 16px;
border-bottom: 1px solid var(--line);
display: flex;
justify-content: space-between;
align-items: center;
gap: 16px;
}
.panel-title {
font-weight: 700;
font-size: 15px;
}
.panel-subtitle {
color: var(--muted);
font-size: 13px;
}
.viewer {
min-height: 600px;
position: relative;
background:
radial-gradient(circle at top left, rgba(14, 165, 233, 0.10), transparent 28%),
linear-gradient(180deg, #fbfcfe 0%, #eef2f6 100%);
}
.bottom {
display: grid;
grid-template-columns: 320px minmax(0, 1fr);
gap: 18px;
}
.hint {
color: var(--muted);
font-size: 13px;
line-height: 1.5;
}
.empty {
display: grid;
place-items: center;
height: 100%;
color: var(--muted);
font-size: 14px;
}
@media (max-width: 1100px) {
.toolbar, .grid, .bottom {
grid-template-columns: 1fr;
}
}
</style>
</head>
<body>
<div class="page">
<section class="header">
<h1>SceneVerse++ Demo</h1>
<p>Load SceneVerse++ scene, view <code>mesh.ply</code> and <code>instance_seg_mesh.ply</code> side by side, double-click a bbox wireframe (edges) to inspect its <code>pred_describe</code>, and keep both views synchronized while rotating.</p>
</section>
<section class="toolbar">
<div class="field">
<label for="scene-select">Scene</label>
<select id="scene-select"></select>
</div>
<div class="status" id="status">Loading scene list...</div>
<div class="field">
<button id="reload-btn" type="button">Reload Scenes</button>
</div>
</section>
<section class="grid">
<article class="panel">
<div class="panel-header">
<div class="panel-title">mesh.ply</div>
<div class="panel-subtitle">Original reconstructed scene mesh</div>
</div>
<div id="left-viewer" class="viewer"><div class="empty">Loading...</div></div>
</article>
<article class="panel">
<div class="panel-header">
<div class="panel-title">instance_seg_mesh.ply</div>
<div class="panel-subtitle">Instance-segmented scene mesh</div>
</div>
<div id="right-viewer" class="viewer"><div class="empty">Loading...</div></div>
</article>
</section>
<section class="bottom">
<article class="panel" style="min-height: 0;">
<div class="panel-header">
<div class="panel-title">Instance</div>
<div class="panel-subtitle">Select manually if needed</div>
</div>
<div style="padding: 16px;">
<div class="field">
<label for="instance-select">BBox / Class</label>
<select id="instance-select">
<option value="">Select an instance</option>
</select>
</div>
<p class="hint">Double-click any bbox wireframe (edges) in either viewer. The corresponding bbox will highlight in both viewers, and both cameras stay synchronized.</p>
</div>
</article>
<article class="panel" style="min-height: 0;">
<div class="panel-header">
<div class="panel-title">pred_describe</div>
<div class="panel-subtitle">Instance-level description</div>
</div>
<div style="padding: 16px;">
<div class="field">
<label for="description-box">Description</label>
<textarea id="description-box" readonly>Select an instance or click a bounding box.</textarea>
</div>
</div>
</article>
</section>
</div>
<script type="module">
import * as THREE from "three";
import { TrackballControls } from "three/addons/controls/TrackballControls.js";
import { PLYLoader } from "three/addons/loaders/PLYLoader.js";
const sceneSelect = document.getElementById("scene-select");
const instanceSelect = document.getElementById("instance-select");
const descriptionBox = document.getElementById("description-box");
const statusBox = document.getElementById("status");
const reloadBtn = document.getElementById("reload-btn");
const defaultDescription = "Select an instance or click a bounding box.";
let sceneList = [];
let currentScene = null;
let instances = [];
let currentSelection = null;
let syncingControls = false;
function createViewer(containerId, meshTint) {
const container = document.getElementById(containerId);
container.innerHTML = "";
const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true });
renderer.setPixelRatio(window.devicePixelRatio);
renderer.setSize(container.clientWidth, container.clientHeight);
renderer.outputColorSpace = THREE.SRGBColorSpace;
renderer.toneMapping = THREE.ACESFilmicToneMapping;
renderer.toneMappingExposure = 1.0;
container.appendChild(renderer.domElement);
const scene = new THREE.Scene();
scene.background = new THREE.Color(0xf5f5f7);
const camera = new THREE.PerspectiveCamera(55, container.clientWidth / container.clientHeight, 0.01, 2000);
camera.up.set(0, 0, 1);
camera.position.set(0, 1.5, 2.5);
const controls = new TrackballControls(camera, renderer.domElement);
controls.rotateSpeed = 4.0;
controls.zoomSpeed = 1.5;
controls.panSpeed = 1.0;
controls.dynamicDampingFactor = 0.12;
controls.noPan = false;
controls.staticMoving = false;
scene.add(new THREE.AmbientLight(0xffffff, 1.0));
const dir1 = new THREE.DirectionalLight(0xffffff, 1.8);
dir1.position.set(4, 6, 5);
scene.add(dir1);
const hemi = new THREE.HemisphereLight(0xffffff, 0x444444, 0.8);
hemi.position.set(0, 1, 0);
scene.add(hemi);
const dir2 = new THREE.DirectionalLight(0xffffff, 0.6);
dir2.position.set(-3, -2, 4);
scene.add(dir2);
const axes = new THREE.AxesHelper(0.8);
scene.add(axes);
const root = new THREE.Group();
const bboxGroup = new THREE.Group();
scene.add(root);
scene.add(bboxGroup);
const raycaster = new THREE.Raycaster();
raycaster.params.Line = { threshold: 0.08 };
const pointer = new THREE.Vector2();
const bboxObjects = [];
const resize = () => {
const width = container.clientWidth || 1;
const height = container.clientHeight || 1;
camera.aspect = width / height;
camera.updateProjectionMatrix();
renderer.setSize(width, height);
};
window.addEventListener("resize", resize);
renderer.domElement.addEventListener("dblclick", (event) => {
const rect = renderer.domElement.getBoundingClientRect();
pointer.x = ((event.clientX - rect.left) / rect.width) * 2 - 1;
pointer.y = -((event.clientY - rect.top) / rect.height) * 2 + 1;
raycaster.setFromCamera(pointer, camera);
const hits = raycaster.intersectObjects(bboxObjects, true);
const target = hits.find((hit) => hit.object.userData?.instanceId || hit.object.parent?.userData?.instanceId);
if (!target) return;
const instanceId = target.object.userData.instanceId || target.object.parent.userData.instanceId;
selectInstance(instanceId);
});
const animate = () => {
controls.update();
renderer.render(scene, camera);
requestAnimationFrame(animate);
};
animate();
return {
container,
renderer,
scene,
camera,
controls,
root,
bboxGroup,
bboxObjects,
meshTint,
fitToObject(mesh) {
const box = new THREE.Box3().setFromObject(mesh);
const size = box.getSize(new THREE.Vector3());
const center = box.getCenter(new THREE.Vector3());
controls.target.copy(center);
const radius = Math.max(size.x, size.y, size.z, 1e-3);
camera.position.copy(center.clone().add(new THREE.Vector3(0, radius * 1.6, radius * 2.2)));
camera.up.set(0, 0, 1);
camera.near = radius / 200;
camera.far = radius * 50;
camera.updateProjectionMatrix();
controls.update();
},
};
}
const leftViewer = createViewer("left-viewer", 0xc7c7c7);
const rightViewer = createViewer("right-viewer", 0x79aefc);
function syncViewerState(source, target) {
if (syncingControls) return;
syncingControls = true;
target.camera.position.copy(source.camera.position);
target.camera.quaternion.copy(source.camera.quaternion);
target.camera.up.copy(source.camera.up);
target.controls.target.copy(source.controls.target);
target.controls.update();
syncingControls = false;
}
leftViewer.controls.addEventListener("change", () => syncViewerState(leftViewer, rightViewer));
rightViewer.controls.addEventListener("change", () => syncViewerState(rightViewer, leftViewer));
function resetViewer(viewer) {
while (viewer.root.children.length) viewer.root.remove(viewer.root.children[0]);
while (viewer.bboxGroup.children.length) viewer.bboxGroup.remove(viewer.bboxGroup.children[0]);
viewer.bboxObjects.length = 0;
}
async function loadPlyMesh(url, tint) {
const loader = new PLYLoader();
const geometry = await loader.loadAsync(url);
geometry.computeVertexNormals();
let material;
if (geometry.hasAttribute("color")) {
material = new THREE.MeshStandardMaterial({
color: 0xffffff,
vertexColors: true,
metalness: 0.5,
roughness: 0.5,
side: THREE.FrontSide,
});
} else {
material = new THREE.MeshStandardMaterial({
color: tint,
metalness: 0.18,
roughness: 0.48,
side: THREE.FrontSide,
});
}
const mesh = new THREE.Mesh(geometry, material);
mesh.castShadow = false;
mesh.receiveShadow = false;
return mesh;
}
function makeBboxGroup(instance) {
const size = new THREE.Vector3(...instance.bbox_size);
const center = new THREE.Vector3(...instance.bbox_center);
const group = new THREE.Group();
group.userData.instanceId = instance.instance_id;
const solid = new THREE.Mesh(
new THREE.BoxGeometry(size.x, size.y, size.z),
new THREE.MeshBasicMaterial({ color: 0x00e5ff, transparent: true, opacity: 0.05, depthWrite: false })
);
solid.position.copy(center);
solid.userData.instanceId = instance.instance_id;
solid.raycast = () => null;
group.add(solid);
const edges = new THREE.LineSegments(
new THREE.EdgesGeometry(new THREE.BoxGeometry(size.x, size.y, size.z)),
new THREE.LineBasicMaterial({ color: 0x00e5ff, linewidth: 2 })
);
edges.position.copy(center);
edges.userData.instanceId = instance.instance_id;
group.add(edges);
group.userData.edgeMaterial = edges.material;
return group;
}
function applySelectionStyles(instanceId) {
[leftViewer, rightViewer].forEach((viewer) => {
viewer.bboxGroup.children.forEach((group) => {
const selected = group.userData.instanceId === instanceId;
const edgeMaterial = group.userData.edgeMaterial;
edgeMaterial.color.set(selected ? 0xef4444 : 0x00e5ff);
edgeMaterial.linewidth = selected ? 4 : 2;
const solid = group.children[0];
solid.material.color.set(selected ? 0xef4444 : 0x00e5ff);
solid.material.opacity = selected ? 0.12 : 0.05;
});
});
}
function updateInstanceSelect() {
instanceSelect.innerHTML = '<option value="">Select an instance</option>';
for (const instance of instances) {
const option = document.createElement("option");
option.value = instance.instance_id;
option.textContent = `${instance.instance_id} | ${instance.class_name}`;
instanceSelect.appendChild(option);
}
}
function selectInstance(instanceId) {
currentSelection = instanceId || null;
if (!instanceId) {
instanceSelect.value = "";
descriptionBox.value = defaultDescription;
applySelectionStyles(null);
return;
}
const instance = instances.find((item) => item.instance_id === instanceId);
if (!instance) return;
instanceSelect.value = instanceId;
descriptionBox.value = instance.describe || "(No pred_describe available for this instance.)";
applySelectionStyles(instanceId);
}
async function renderScene(sceneId) {
statusBox.textContent = `Loading ${sceneId} ...`;
resetViewer(leftViewer);
resetViewer(rightViewer);
descriptionBox.value = defaultDescription;
const response = await fetch(`/api/scenes/${encodeURIComponent(sceneId)}`);
if (!response.ok) {
throw new Error(await response.text());
}
currentScene = await response.json();
instances = currentScene.instances;
updateInstanceSelect();
const [leftMesh, rightMesh] = await Promise.all([
loadPlyMesh(currentScene.mesh_url, leftViewer.meshTint),
loadPlyMesh(currentScene.seg_mesh_url, rightViewer.meshTint),
]);
leftViewer.root.add(leftMesh);
rightViewer.root.add(rightMesh);
leftViewer.fitToObject(leftMesh);
syncViewerState(leftViewer, rightViewer);
for (const instance of instances) {
const leftBox = makeBboxGroup(instance);
const rightBox = makeBboxGroup(instance);
leftViewer.bboxGroup.add(leftBox);
rightViewer.bboxGroup.add(rightBox);
leftViewer.bboxObjects.push(leftBox.children[1]);
rightViewer.bboxObjects.push(rightBox.children[1]);
}
selectInstance(currentSelection && instances.some((item) => item.instance_id === currentSelection) ? currentSelection : null);
statusBox.textContent = `Loaded ${sceneId}. ${instances.length} instances.`;
}
async function loadSceneList() {
statusBox.textContent = "Loading scene list...";
const response = await fetch("/api/scenes");
if (!response.ok) {
statusBox.textContent = `Failed to load scene list: ${await response.text()}`;
return;
}
sceneList = await response.json();
sceneSelect.innerHTML = "";
for (const sceneId of sceneList) {
const option = document.createElement("option");
option.value = sceneId;
option.textContent = sceneId;
sceneSelect.appendChild(option);
}
if (!sceneList.length) {
statusBox.textContent = "No valid scenes found.";
return;
}
await renderScene(sceneList[0]);
}
sceneSelect.addEventListener("change", async () => {
currentSelection = null;
await renderScene(sceneSelect.value);
});
instanceSelect.addEventListener("change", () => {
selectInstance(instanceSelect.value || null);
});
reloadBtn.addEventListener("click", async () => {
currentSelection = null;
await loadSceneList();
});
loadSceneList().catch((error) => {
console.error(error);
statusBox.textContent = `Initialization failed: ${error.message}`;
});
</script>
</body>
</html>
"""
@app.get("/", response_class=HTMLResponse)
def index() -> str:
return INDEX_HTML
@app.get("/api/scenes", response_class=JSONResponse)
def api_scenes() -> list[str]:
try:
return list_scene_ids(DEFAULT_DATA_ROOT)
except FileNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.get("/api/scenes/{scene_id}", response_class=JSONResponse)
def api_scene(scene_id: str) -> dict[str, Any]:
try:
return load_scene_bundle(scene_id)
except FileNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except Exception as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
@app.get("/api/scenes/{scene_id}/files/{filename}")
def api_scene_file(scene_id: str, filename: str) -> FileResponse:
if filename not in REQUIRED_FILES:
raise HTTPException(status_code=404, detail=f"Unsupported file: {filename}")
file_path = DEFAULT_DATA_ROOT / scene_id / filename
if not file_path.is_file():
raise HTTPException(status_code=404, detail=f"File not found: {file_path}")
return FileResponse(file_path)
if __name__ == "__main__":
port = int(os.getenv("PORT", "7860"))
uvicorn.run("app:app", host="0.0.0.0", port=port, reload=False)