jasondo OpenAI Codex commited on
Commit
682623a
·
1 Parent(s): 628bf77

Expand renderer vocabulary and annotation fallback

Browse files

Co-authored-by: OpenAI Codex <codex@openai.com>

Files changed (6) hide show
  1. AGENTS.md +16 -0
  2. README.md +7 -0
  3. index.html +150 -26
  4. snap2sim/model_io.py +14 -7
  5. snap2sim/prompts.py +32 -18
  6. snap2sim/schema.py +98 -19
AGENTS.md CHANGED
@@ -284,6 +284,22 @@ technical cutaway animation.
284
  movement, and the server clamps/coerces the threshold before selecting
285
  `render_mode`. The browser now trusts the server's `render_mode` and only
286
  downgrades when geometry or annotation data is missing.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
287
 
288
  ## Next Work
289
 
 
284
  movement, and the server clamps/coerces the threshold before selecting
285
  `render_mode`. The browser now trusts the server's `render_mode` and only
286
  downgrades when geometry or annotation data is missing.
287
+ - Implemented the `FEATURE2.md` annotated-photo fallback pass: fixed portrait
288
+ and landscape image containment by giving the annotation frame a definite
289
+ viewport-bounded box, added numbered marker dots, leader lines, and optional
290
+ annotation bounding boxes, and kept all annotation text rendered through
291
+ `textContent`.
292
+ - Implemented the `FEATURE3.md` renderer/prompt vocabulary pass: added
293
+ deterministic `cone`, `capsule`, `torus`, and `spring` shapes; added `screw`,
294
+ `orbit`, and `pulse` motions; raised the usable part cap to 6 across prompt,
295
+ schema, coercion, and browser renderability; and updated the sample analysis
296
+ to exercise all four new primitives.
297
+ - Local verification after the `FEATURE2.md` / `FEATURE3.md` pass: schema and
298
+ coercion checks passed, FastAPI `TestClient` confirmed `/`, `/analyze_image`,
299
+ and `/generate_scene`, browser verification confirmed contained portrait and
300
+ landscape annotated-photo fallbacks with labels in bounds and no mobile
301
+ horizontal overflow, and the deterministic Three.js path reached
302
+ `CUTAWAY READY` with a nonblank six-part scene.
303
 
304
  ## Next Work
305
 
README.md CHANGED
@@ -144,6 +144,13 @@ play/pause control. The confidence slider defaults to 50%, matching the
144
  server's fallback threshold, and the generation route is authoritative for the
145
  visible render mode.
146
 
 
 
 
 
 
 
 
147
  The browser no longer injects model-authored HTML into the DOM. The model's
148
  job is limited to the structured analysis JSON contract in `snap2sim/schema.py`.
149
 
 
144
  server's fallback threshold, and the generation route is authoritative for the
145
  visible render mode.
146
 
147
+ The deterministic renderer supports `box`, `cylinder`, `sphere`, `gear`, `rod`,
148
+ `cone`, `capsule`, `torus`, and `spring` primitives, plus `static`, `rotate`,
149
+ `oscillate`, `translate`, `screw`, `orbit`, and `pulse` motions. The
150
+ annotated-photo fallback keeps the uploaded image contained in the viewport and
151
+ draws marker dots, leader lines, and optional normalized bounding boxes from
152
+ validated annotation data.
153
+
154
  The browser no longer injects model-authored HTML into the DOM. The model's
155
  job is limited to the structured analysis JSON contract in `snap2sim/schema.py`.
156
 
index.html CHANGED
@@ -488,30 +488,59 @@
488
  }
489
 
490
  .annotated-stage {
491
- display: grid;
492
- place-items: center;
493
- padding: 58px 18px 44px;
494
  background: rgba(15, 19, 24, 0.72);
495
  }
496
 
497
  .annotation-frame {
498
- position: relative;
499
- width: 100%;
500
- height: 100%;
501
- display: grid;
502
- place-items: center;
503
  }
504
 
505
  .annotation-image {
 
 
506
  width: 100%;
507
  height: 100%;
508
  object-fit: contain;
509
- border: 1px solid rgba(122, 84, 32, 0.72);
510
  background: var(--bg);
511
  }
512
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
513
  .photo-label {
514
  max-width: min(220px, 42vw);
 
515
  white-space: normal;
516
  text-transform: none;
517
  }
@@ -839,7 +868,7 @@
839
  return Boolean(analysis && Array.isArray(analysis.parts) && analysis.parts.some((part) => {
840
  const geometry = part && part.geometry;
841
  return geometry
842
- && ["box", "cylinder", "sphere", "gear", "rod"].includes(geometry.shape)
843
  && isNumberList(geometry.size, 3)
844
  && isNumberList(geometry.position, 3);
845
  }));
@@ -936,6 +965,10 @@
936
  mesh.userData.basePosition = mesh.position.clone();
937
  mesh.userData.baseRotation = mesh.rotation.clone();
938
  mesh.userData.part = part;
 
 
 
 
939
  mesh.scale.setScalar(0.001);
940
  scene.add(mesh);
941
  return mesh;
@@ -1032,19 +1065,48 @@
1032
  if (disposed) return;
1033
  const imageRect = containedImageRect(image);
1034
  const layerRect = labelLayer.getBoundingClientRect();
1035
- labelLayer.replaceChildren(...annotatedParts.map((part, index) => {
 
 
1036
  const annotation = part.annotation || {};
1037
  const point = annotation.point || [0.5, 0.5];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1038
  const label = document.createElement("div");
1039
  label.className = "scene-label photo-label";
1040
- label.style.left = Math.min(
1041
- layerRect.width - 72,
1042
- Math.max(72, imageRect.left - layerRect.left + clamp01(point[0]) * imageRect.width)
1043
- ) + "px";
1044
- label.style.top = Math.min(
1045
- layerRect.height - 36,
1046
- Math.max(56, imageRect.top - layerRect.top + clamp01(point[1]) * imageRect.height)
1047
- ) + "px";
1048
 
1049
  const title = document.createElement("span");
1050
  title.className = "label-title";
@@ -1057,8 +1119,19 @@
1057
  note.textContent = noteText;
1058
  label.append(note);
1059
  }
1060
- return label;
1061
- }));
 
 
 
 
 
 
 
 
 
 
 
1062
  }
1063
 
1064
  image.addEventListener("load", updatePhotoLabels, { once: true });
@@ -1141,18 +1214,34 @@
1141
  function buildPartMesh(part, index) {
1142
  const geometryData = part.geometry || {};
1143
  const size = Array.isArray(geometryData.size) ? geometryData.size : [1, 1, 1];
 
 
 
1144
  let geometry;
1145
  if (geometryData.shape === "cylinder") {
1146
- geometry = new THREE.CylinderGeometry(size[0] / 2, size[2] / 2, size[1], 48);
 
 
 
1147
  } else if (geometryData.shape === "sphere") {
1148
- geometry = new THREE.SphereGeometry(Math.max(size[0], size[1], size[2]) / 2, 36, 18);
 
 
 
1149
  } else if (geometryData.shape === "rod") {
1150
- geometry = new THREE.CylinderGeometry(Math.max(size[0], size[1]) / 2, Math.max(size[0], size[1]) / 2, size[2], 24);
1151
  geometry.rotateX(Math.PI / 2);
1152
  } else if (geometryData.shape === "gear") {
1153
- geometry = gearGeometry(Math.max(size[0], size[2]) / 2, size[1], geometryData.teeth || 18);
 
 
 
 
 
 
 
1154
  } else {
1155
- geometry = new THREE.BoxGeometry(size[0], size[1], size[2]);
1156
  }
1157
 
1158
  const color = colorFor(geometryData.color, index);
@@ -1186,6 +1275,25 @@
1186
  return geometry;
1187
  }
1188
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1189
  function revealMesh(mesh, elapsed, index) {
1190
  const local = Math.max(0, Math.min(1, (elapsed - index * 0.1) / 0.55));
1191
  const eased = 1 - Math.pow(1 - local, 3);
@@ -1207,6 +1315,22 @@
1207
  const range = motion.range || [-0.25, 0.25];
1208
  const offset = range[0] + (range[1] - range[0]) * ((Math.sin(elapsed * speed + phase) + 1) / 2);
1209
  mesh.position.add(axis.multiplyScalar(offset));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1210
  }
1211
  }
1212
 
 
488
  }
489
 
490
  .annotated-stage {
 
 
 
491
  background: rgba(15, 19, 24, 0.72);
492
  }
493
 
494
  .annotation-frame {
495
+ position: absolute;
496
+ inset: 58px 18px 44px;
 
 
 
497
  }
498
 
499
  .annotation-image {
500
+ position: absolute;
501
+ inset: 0;
502
  width: 100%;
503
  height: 100%;
504
  object-fit: contain;
505
+ outline: 1px solid rgba(122, 84, 32, 0.72);
506
  background: var(--bg);
507
  }
508
 
509
+ .photo-marker {
510
+ position: absolute;
511
+ width: 22px;
512
+ height: 22px;
513
+ transform: translate(-50%, -50%);
514
+ border: 1px solid var(--cyan);
515
+ border-radius: 50%;
516
+ color: var(--bg);
517
+ background: var(--cyan);
518
+ box-shadow: 0 0 16px rgba(95, 212, 208, 0.45);
519
+ display: grid;
520
+ place-items: center;
521
+ font: 0.68rem/1 "Fira Code", monospace;
522
+ pointer-events: none;
523
+ }
524
+
525
+ .photo-leader {
526
+ position: absolute;
527
+ height: 1px;
528
+ transform-origin: 0 50%;
529
+ background: rgba(95, 212, 208, 0.72);
530
+ pointer-events: none;
531
+ }
532
+
533
+ .photo-box {
534
+ position: absolute;
535
+ border: 1px solid rgba(95, 212, 208, 0.86);
536
+ background: rgba(95, 212, 208, 0.08);
537
+ box-shadow: inset 0 0 0 1px rgba(15, 19, 24, 0.45);
538
+ pointer-events: none;
539
+ }
540
+
541
  .photo-label {
542
  max-width: min(220px, 42vw);
543
+ transform: none;
544
  white-space: normal;
545
  text-transform: none;
546
  }
 
868
  return Boolean(analysis && Array.isArray(analysis.parts) && analysis.parts.some((part) => {
869
  const geometry = part && part.geometry;
870
  return geometry
871
+ && ["box", "cylinder", "sphere", "gear", "rod", "cone", "capsule", "torus", "spring"].includes(geometry.shape)
872
  && isNumberList(geometry.size, 3)
873
  && isNumberList(geometry.position, 3);
874
  }));
 
965
  mesh.userData.basePosition = mesh.position.clone();
966
  mesh.userData.baseRotation = mesh.rotation.clone();
967
  mesh.userData.part = part;
968
+ const pivot = part.motion && isNumberList(part.motion.pivot, 3)
969
+ ? part.motion.pivot
970
+ : [0, 0, 0];
971
+ mesh.userData.pivot = new THREE.Vector3(pivot[0], pivot[1], pivot[2]);
972
  mesh.scale.setScalar(0.001);
973
  scene.add(mesh);
974
  return mesh;
 
1065
  if (disposed) return;
1066
  const imageRect = containedImageRect(image);
1067
  const layerRect = labelLayer.getBoundingClientRect();
1068
+ if (imageRect.width < 1 || imageRect.height < 1 || layerRect.width < 1 || layerRect.height < 1) return;
1069
+ const nodes = [];
1070
+ for (const [index, part] of annotatedParts.entries()) {
1071
  const annotation = part.annotation || {};
1072
  const point = annotation.point || [0.5, 0.5];
1073
+ const pointX = imageRect.left - layerRect.left + clamp01(point[0]) * imageRect.width;
1074
+ const pointY = imageRect.top - layerRect.top + clamp01(point[1]) * imageRect.height;
1075
+ if (isNumberList(annotation.box, 4)) {
1076
+ const [boxX, boxY, boxW, boxH] = annotation.box.map(clamp01);
1077
+ const box = document.createElement("div");
1078
+ box.className = "photo-box";
1079
+ box.style.left = (imageRect.left - layerRect.left + boxX * imageRect.width) + "px";
1080
+ box.style.top = (imageRect.top - layerRect.top + boxY * imageRect.height) + "px";
1081
+ box.style.width = (Math.min(boxW, 1 - boxX) * imageRect.width) + "px";
1082
+ box.style.height = (Math.min(boxH, 1 - boxY) * imageRect.height) + "px";
1083
+ nodes.push(box);
1084
+ }
1085
+
1086
+ const marker = document.createElement("div");
1087
+ marker.className = "photo-marker";
1088
+ marker.style.left = pointX + "px";
1089
+ marker.style.top = pointY + "px";
1090
+ marker.textContent = String(index + 1);
1091
+ nodes.push(marker);
1092
+
1093
+ const labelWidth = Math.min(220, Math.max(120, layerRect.width * 0.42), Math.max(80, layerRect.width - 24));
1094
+ const preferLeft = pointX > layerRect.width * 0.62;
1095
+ const maxLabelX = Math.max(12, layerRect.width - labelWidth - 12);
1096
+ const maxLabelY = Math.max(12, layerRect.height - 82);
1097
+ const labelX = Math.min(
1098
+ maxLabelX,
1099
+ Math.max(12, pointX + (preferLeft ? -labelWidth - 28 : 28))
1100
+ );
1101
+ const labelY = Math.min(
1102
+ maxLabelY,
1103
+ Math.max(12, pointY - 42)
1104
+ );
1105
  const label = document.createElement("div");
1106
  label.className = "scene-label photo-label";
1107
+ label.style.left = labelX + "px";
1108
+ label.style.top = labelY + "px";
1109
+ label.style.width = labelWidth + "px";
 
 
 
 
 
1110
 
1111
  const title = document.createElement("span");
1112
  title.className = "label-title";
 
1119
  note.textContent = noteText;
1120
  label.append(note);
1121
  }
1122
+ const labelAnchorX = preferLeft ? labelX + labelWidth : labelX;
1123
+ const labelAnchorY = labelY + 24;
1124
+ const dx = labelAnchorX - pointX;
1125
+ const dy = labelAnchorY - pointY;
1126
+ const leader = document.createElement("div");
1127
+ leader.className = "photo-leader";
1128
+ leader.style.left = pointX + "px";
1129
+ leader.style.top = pointY + "px";
1130
+ leader.style.width = Math.max(0, Math.hypot(dx, dy) - 13) + "px";
1131
+ leader.style.transform = "rotate(" + Math.atan2(dy, dx) + "rad)";
1132
+ nodes.push(leader, label);
1133
+ }
1134
+ labelLayer.replaceChildren(...nodes);
1135
  }
1136
 
1137
  image.addEventListener("load", updatePhotoLabels, { once: true });
 
1214
  function buildPartMesh(part, index) {
1215
  const geometryData = part.geometry || {};
1216
  const size = Array.isArray(geometryData.size) ? geometryData.size : [1, 1, 1];
1217
+ const x = Math.max(0.05, Number(size[0]) || 1);
1218
+ const y = Math.max(0.05, Number(size[1]) || 1);
1219
+ const z = Math.max(0.05, Number(size[2]) || 1);
1220
  let geometry;
1221
  if (geometryData.shape === "cylinder") {
1222
+ const radius = Math.max(x, z) / 2;
1223
+ geometry = new THREE.CylinderGeometry(radius, radius, y, 48);
1224
+ } else if (geometryData.shape === "cone") {
1225
+ geometry = new THREE.ConeGeometry(Math.max(x, z) / 2, y, 48);
1226
  } else if (geometryData.shape === "sphere") {
1227
+ geometry = new THREE.SphereGeometry(Math.max(x, y, z) / 2, 36, 18);
1228
+ } else if (geometryData.shape === "capsule") {
1229
+ const radius = Math.min(Math.max(x, z) / 2, y / 2);
1230
+ geometry = new THREE.CapsuleGeometry(radius, Math.max(0.05, y - radius * 2), 8, 24);
1231
  } else if (geometryData.shape === "rod") {
1232
+ geometry = new THREE.CylinderGeometry(Math.max(x, y) / 2, Math.max(x, y) / 2, z, 24);
1233
  geometry.rotateX(Math.PI / 2);
1234
  } else if (geometryData.shape === "gear") {
1235
+ geometry = gearGeometry(Math.max(x, z) / 2, y, geometryData.teeth || 18);
1236
+ } else if (geometryData.shape === "torus") {
1237
+ const outerRadius = Math.max(x, z) / 2;
1238
+ const tube = Math.min(y / 2, outerRadius * 0.36);
1239
+ geometry = new THREE.TorusGeometry(Math.max(0.05, outerRadius - tube), Math.max(0.025, tube), 16, 64);
1240
+ geometry.rotateX(Math.PI / 2);
1241
+ } else if (geometryData.shape === "spring") {
1242
+ geometry = springGeometry(Math.max(x, z) / 2, y, geometryData.coils || 5, geometryData.wire);
1243
  } else {
1244
+ geometry = new THREE.BoxGeometry(x, y, z);
1245
  }
1246
 
1247
  const color = colorFor(geometryData.color, index);
 
1275
  return geometry;
1276
  }
1277
 
1278
+ function springGeometry(outerRadius, height, coils, wire) {
1279
+ const safeCoils = Math.max(2, Math.min(12, Number(coils) || 5));
1280
+ const radius = Math.max(0.08, outerRadius * 0.72);
1281
+ const wireRadius = Math.max(0.018, Math.min(Number(wire) || outerRadius * 0.08, outerRadius * 0.18));
1282
+ const points = [];
1283
+ const steps = Math.ceil(safeCoils * 28);
1284
+ for (let i = 0; i <= steps; i += 1) {
1285
+ const t = i / steps;
1286
+ const angle = t * safeCoils * Math.PI * 2;
1287
+ points.push(new THREE.Vector3(
1288
+ Math.cos(angle) * radius,
1289
+ (t - 0.5) * height,
1290
+ Math.sin(angle) * radius
1291
+ ));
1292
+ }
1293
+ const curve = new THREE.CatmullRomCurve3(points);
1294
+ return new THREE.TubeGeometry(curve, steps, wireRadius, 8, false);
1295
+ }
1296
+
1297
  function revealMesh(mesh, elapsed, index) {
1298
  const local = Math.max(0, Math.min(1, (elapsed - index * 0.1) / 0.55));
1299
  const eased = 1 - Math.pow(1 - local, 3);
 
1315
  const range = motion.range || [-0.25, 0.25];
1316
  const offset = range[0] + (range[1] - range[0]) * ((Math.sin(elapsed * speed + phase) + 1) / 2);
1317
  mesh.position.add(axis.multiplyScalar(offset));
1318
+ } else if (motion.type === "screw") {
1319
+ const range = motion.range || [-0.18, 0.18];
1320
+ const pitch = Number(motion.pitch || 0.18);
1321
+ const turn = elapsed * speed + phase;
1322
+ mesh.rotateOnAxis(axis, turn);
1323
+ const offset = range[0] + (range[1] - range[0]) * ((Math.sin(turn) + 1) / 2) + Math.sin(turn) * pitch;
1324
+ mesh.position.add(axis.multiplyScalar(offset));
1325
+ } else if (motion.type === "orbit") {
1326
+ const pivot = mesh.userData.pivot || new THREE.Vector3(0, 0, 0);
1327
+ const offset = mesh.userData.basePosition.clone().sub(pivot).applyAxisAngle(axis, elapsed * speed + phase);
1328
+ mesh.position.copy(pivot).add(offset);
1329
+ mesh.rotateOnAxis(axis, elapsed * speed + phase);
1330
+ } else if (motion.type === "pulse") {
1331
+ const amplitude = Number(motion.amplitude || 0.18);
1332
+ const scalar = Math.max(0.2, 1 + Math.sin(elapsed * speed + phase) * amplitude);
1333
+ mesh.scale.multiplyScalar(scalar);
1334
  }
1335
  }
1336
 
snap2sim/model_io.py CHANGED
@@ -114,7 +114,7 @@ def _coerce_analysis_payload(payload: dict[str, Any], fallback_component: str) -
114
  parts = payload.get("parts")
115
  if not isinstance(parts, list):
116
  parts = []
117
- coerced_parts = [_coerce_part(part, index) for index, part in enumerate(parts[:4]) if isinstance(part, dict)]
118
  coerced_parts = [part for part in coerced_parts if part is not None]
119
  if not coerced_parts:
120
  coerced_parts = _generic_analysis(fallback_component)["parts"]
@@ -163,22 +163,25 @@ def _coerce_part(part: dict[str, Any], index: int) -> dict[str, Any] | None:
163
  return base_part
164
 
165
  shape = geometry.get("shape")
166
- if shape not in {"box", "cylinder", "sphere", "gear", "rod"}:
167
  shape = "box"
168
  motion_type = motion.get("type")
169
- if motion_type not in {"rotate", "translate", "oscillate", "static"}:
170
  motion_type = "static"
171
 
172
  coerced_motion: dict[str, Any] = {"type": motion_type}
173
  axis = _axis_vector(motion.get("axis"))
174
  if axis:
175
  coerced_motion["axis"] = axis
176
- for key in ["speed", "amplitude", "phase"]:
177
  if isinstance(motion.get(key), (int, float)) and not isinstance(motion.get(key), bool):
178
  coerced_motion[key] = float(motion[key])
179
  values = _number_list(motion.get("range"), 2)
180
  if values:
181
  coerced_motion["range"] = values
 
 
 
182
 
183
  coerced_geometry: dict[str, Any] = {
184
  "shape": shape,
@@ -190,6 +193,10 @@ def _coerce_part(part: dict[str, Any], index: int) -> dict[str, Any] | None:
190
  coerced_geometry["rotation"] = values
191
  if isinstance(geometry.get("teeth"), int):
192
  coerced_geometry["teeth"] = geometry["teeth"]
 
 
 
 
193
  if isinstance(geometry.get("color"), str) and geometry["color"].strip():
194
  coerced_geometry["color"] = geometry["color"].strip()
195
 
@@ -276,7 +283,7 @@ def _geometry_size(geometry: dict[str, Any], shape: str) -> list[float]:
276
  length = _number_value(geometry.get("length"))
277
  width = _number_value(geometry.get("width"))
278
  depth = _number_value(geometry.get("depth"))
279
- if shape in {"cylinder", "gear"} and radius and height:
280
  diameter = radius * 2
281
  return [diameter, height, diameter]
282
  if shape == "rod" and (length or height) and radius:
@@ -310,9 +317,9 @@ def _number_value(value: Any) -> float | None:
310
 
311
  def _unit_number_list(value: Any, length: int) -> list[float] | None:
312
  values = _number_list(value, length)
313
- if not values or not all(0 <= item <= 1 for item in values):
314
  return None
315
- return values
316
 
317
 
318
  def _coerce_annotation(value: Any) -> dict[str, Any] | None:
 
114
  parts = payload.get("parts")
115
  if not isinstance(parts, list):
116
  parts = []
117
+ coerced_parts = [_coerce_part(part, index) for index, part in enumerate(parts[:6]) if isinstance(part, dict)]
118
  coerced_parts = [part for part in coerced_parts if part is not None]
119
  if not coerced_parts:
120
  coerced_parts = _generic_analysis(fallback_component)["parts"]
 
163
  return base_part
164
 
165
  shape = geometry.get("shape")
166
+ if shape not in {"box", "cylinder", "sphere", "gear", "rod", "cone", "capsule", "torus", "spring"}:
167
  shape = "box"
168
  motion_type = motion.get("type")
169
+ if motion_type not in {"rotate", "translate", "oscillate", "static", "screw", "orbit", "pulse"}:
170
  motion_type = "static"
171
 
172
  coerced_motion: dict[str, Any] = {"type": motion_type}
173
  axis = _axis_vector(motion.get("axis"))
174
  if axis:
175
  coerced_motion["axis"] = axis
176
+ for key in ["speed", "amplitude", "phase", "pitch"]:
177
  if isinstance(motion.get(key), (int, float)) and not isinstance(motion.get(key), bool):
178
  coerced_motion[key] = float(motion[key])
179
  values = _number_list(motion.get("range"), 2)
180
  if values:
181
  coerced_motion["range"] = values
182
+ values = _number_list(motion.get("pivot"), 3)
183
+ if values:
184
+ coerced_motion["pivot"] = values
185
 
186
  coerced_geometry: dict[str, Any] = {
187
  "shape": shape,
 
193
  coerced_geometry["rotation"] = values
194
  if isinstance(geometry.get("teeth"), int):
195
  coerced_geometry["teeth"] = geometry["teeth"]
196
+ if isinstance(geometry.get("coils"), int):
197
+ coerced_geometry["coils"] = geometry["coils"]
198
+ if isinstance(geometry.get("wire"), (int, float)) and not isinstance(geometry.get("wire"), bool):
199
+ coerced_geometry["wire"] = float(geometry["wire"])
200
  if isinstance(geometry.get("color"), str) and geometry["color"].strip():
201
  coerced_geometry["color"] = geometry["color"].strip()
202
 
 
283
  length = _number_value(geometry.get("length"))
284
  width = _number_value(geometry.get("width"))
285
  depth = _number_value(geometry.get("depth"))
286
+ if shape in {"cylinder", "gear", "cone", "capsule", "torus", "spring"} and radius and height:
287
  diameter = radius * 2
288
  return [diameter, height, diameter]
289
  if shape == "rod" and (length or height) and radius:
 
317
 
318
  def _unit_number_list(value: Any, length: int) -> list[float] | None:
319
  values = _number_list(value, length)
320
+ if not values:
321
  return None
322
+ return [max(0.0, min(1.0, item)) for item in values]
323
 
324
 
325
  def _coerce_annotation(value: Any) -> dict[str, Any] | None:
snap2sim/prompts.py CHANGED
@@ -2,37 +2,51 @@
2
 
3
  from __future__ import annotations
4
 
5
- VISION_SYSTEM_PROMPT = """You are a mechanical teardown analyst.
6
- Given an image of a hardware component, infer the most likely internal
7
- mechanism and return only JSON matching the provided schema. Prefer clear,
8
- physically plausible primitive geometry over speculative detail. If the photo is
9
- ambiguous, state lower confidence and identify visible components with photo
10
- annotations instead of forcing a speculative 3D mechanism."""
11
 
12
 
13
  def build_vision_prompt() -> str:
14
  return (
15
  "Analyze the uploaded hardware component as a cutaway mechanism. "
16
- "You may reason first if needed, but the final answer must be one JSON "
17
- "object. Do not include markdown. Keep the final payload compact and "
18
- "physically plausible for primitive Three.js rendering.\n\n"
19
  "Required top-level keys: component, confidence, summary, trigger, "
20
  "motion_sequence, parts. Optional top-level render_mode is three, "
21
  "annotate, or unavailable.\n"
22
  "Each part requires: id, name, role, and either geometry plus motion, "
23
  "or annotation when the visible component can be located but 3D "
24
  "geometry is uncertain.\n"
25
- "Allowed geometry.shape values: box, cylinder, sphere, gear, rod.\n"
26
- "Allowed motion.type values: rotate, translate, oscillate, static.\n"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  "Every geometry must use size: [x, y, z] and position: [x, y, z]. "
28
  "Do not use radius, height, length, width, or depth fields. Every "
29
  "motion axis must be a numeric vector such as [0, 1, 0], never a string "
30
  "like x, y, or z.\n\n"
31
- "Use 2 to 4 parts. Keep names and descriptions short. When possible, "
 
32
  "include annotation.point in normalized image coordinates [x, y] with "
33
  "origin at top-left, plus a short annotation.note. Optional "
34
  "annotation.box is [x, y, width, height], also normalized from 0 to 1.\n\n"
35
- "Use this shape:\n"
36
  "{\n"
37
  ' "component": "short component name",\n'
38
  ' "confidence": 0.7,\n'
@@ -44,13 +58,13 @@ def build_vision_prompt() -> str:
44
  ' "id": "part_id",\n'
45
  ' "name": "part name",\n'
46
  ' "role": "mechanical role",\n'
47
- ' "geometry": {"shape": "box", "size": [1, 1, 1], "position": [0, 0, 0]},\n'
48
- ' "motion": {"type": "static"},\n'
49
  ' "annotation": {"point": [0.5, 0.5], "label": "visible label", "note": "short visible clue"}\n'
50
  " }\n"
51
  " ]\n"
52
  "}\n\n"
53
- "Optional geometry fields: rotation, teeth, color. Optional motion "
54
- "fields: axis, speed, amplitude, phase, range. Include optional fields "
55
- "only when useful."
56
  )
 
2
 
3
  from __future__ import annotations
4
 
5
+ VISION_SYSTEM_PROMPT = """You are a mechanical teardown analyst for an annotated
6
+ technical cutaway demo. Reason briefly if useful, then emit exactly one JSON
7
+ object matching the schema. Prefer the simplest truthful primitive mechanism.
8
+ If the photo is ambiguous, lower confidence and use visible photo annotations
9
+ instead of forcing speculative 3D geometry."""
 
10
 
11
 
12
  def build_vision_prompt() -> str:
13
  return (
14
  "Analyze the uploaded hardware component as a cutaway mechanism. "
15
+ "Final answer must be one JSON object with no markdown.\n\n"
 
 
16
  "Required top-level keys: component, confidence, summary, trigger, "
17
  "motion_sequence, parts. Optional top-level render_mode is three, "
18
  "annotate, or unavailable.\n"
19
  "Each part requires: id, name, role, and either geometry plus motion, "
20
  "or annotation when the visible component can be located but 3D "
21
  "geometry is uncertain.\n"
22
+ "Shapes, with size always [x, y, z] extents:\n"
23
+ "- box: plates, housings, blocks, levers, selectors\n"
24
+ "- cylinder: shafts, sleeves, bushings, drums, pins\n"
25
+ "- cone: valve cones, tips, nozzles, tapers\n"
26
+ "- capsule: pistons, rollers, dowel pins, plungers, bearings\n"
27
+ "- sphere: balls, detents, ball bearings, nodes\n"
28
+ "- rod: links, tie rods, thin axles, connecting rods\n"
29
+ "- gear: toothed wheels, ratchets, cogs; set teeth when useful\n"
30
+ "- torus: o-rings, snap rings, seals, washers, single coils\n"
31
+ "- spring: helical springs and coils; set coils when useful\n\n"
32
+ "Motions, with axis as a numeric vector like [0, 1, 0]:\n"
33
+ "- static: fixed structure or housing\n"
34
+ "- rotate: continuous spin; use speed\n"
35
+ "- oscillate: sinusoidal twist; use amplitude and speed\n"
36
+ "- translate: slide along axis; use range [min, max]\n"
37
+ "- screw: spin plus advance along axis; use pitch for helical action\n"
38
+ "- orbit: revolve around pivot [x, y, z]\n"
39
+ "- pulse: scale breathing for diaphragms, springs, valves, pumps\n\n"
40
  "Every geometry must use size: [x, y, z] and position: [x, y, z]. "
41
  "Do not use radius, height, length, width, or depth fields. Every "
42
  "motion axis must be a numeric vector such as [0, 1, 0], never a string "
43
  "like x, y, or z.\n\n"
44
+ "Use 2 to 6 parts; prefer the fewest that explain the mechanism. "
45
+ "Keep names and descriptions short. When possible, "
46
  "include annotation.point in normalized image coordinates [x, y] with "
47
  "origin at top-left, plus a short annotation.note. Optional "
48
  "annotation.box is [x, y, width, height], also normalized from 0 to 1.\n\n"
49
+ "Use this compact shape:\n"
50
  "{\n"
51
  ' "component": "short component name",\n'
52
  ' "confidence": 0.7,\n'
 
58
  ' "id": "part_id",\n'
59
  ' "name": "part name",\n'
60
  ' "role": "mechanical role",\n'
61
+ ' "geometry": {"shape": "spring", "size": [0.4, 1.2, 0.4], "position": [0, 0, 0], "coils": 6},\n'
62
+ ' "motion": {"type": "pulse", "speed": 2, "amplitude": 0.08},\n'
63
  ' "annotation": {"point": [0.5, 0.5], "label": "visible label", "note": "short visible clue"}\n'
64
  " }\n"
65
  " ]\n"
66
  "}\n\n"
67
+ "Optional geometry fields: rotation, teeth, coils, wire, color. Optional "
68
+ "motion fields: axis, speed, amplitude, phase, range, pitch, pivot. "
69
+ "Include optional fields only when useful."
70
  )
snap2sim/schema.py CHANGED
@@ -5,7 +5,7 @@ from __future__ import annotations
5
  import math
6
  from typing import Any, Literal
7
 
8
- MotionType = Literal["rotate", "translate", "oscillate", "static"]
9
 
10
 
11
  EXAMPLE_ANALYSIS: dict[str, Any] = {
@@ -19,7 +19,7 @@ EXAMPLE_ANALYSIS: dict[str, Any] = {
19
  "trigger": "handle swings clockwise and counterclockwise",
20
  "motion_sequence": [
21
  "handle applies torque to the outer head",
22
- "pawl tooth locks into the ratchet gear",
23
  "gear and socket rotate on the drive stroke",
24
  "pawl rides over gear teeth on the return stroke",
25
  ],
@@ -70,10 +70,10 @@ EXAMPLE_ANALYSIS: dict[str, Any] = {
70
  "name": "spring pawl",
71
  "role": "locks and releases against gear teeth",
72
  "geometry": {
73
- "shape": "box",
74
- "size": [0.32, 0.35, 1.1],
75
  "position": [1.25, 0.3, 0.18],
76
- "rotation": [0, 0.35, 0],
77
  "color": "amber",
78
  },
79
  "motion": {
@@ -90,20 +90,63 @@ EXAMPLE_ANALYSIS: dict[str, Any] = {
90
  },
91
  },
92
  {
93
- "id": "selector",
94
- "name": "direction selector",
95
- "role": "flips the pawl angle for reverse drive",
96
  "geometry": {
97
- "shape": "box",
98
- "size": [0.9, 0.2, 0.32],
99
- "position": [0, 0.68, -1.05],
 
 
 
100
  "color": "orange",
101
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
102
  "motion": {"type": "static"},
103
  "annotation": {
104
- "point": [0.5, 0.28],
105
- "label": "selector",
106
- "note": "external control that reverses drive direction",
107
  },
108
  },
109
  ],
@@ -122,6 +165,7 @@ ANALYSIS_SCHEMA: dict[str, Any] = {
122
  "motion_sequence": {"type": "array", "items": {"type": "string"}},
123
  "parts": {
124
  "type": "array",
 
125
  "items": {
126
  "type": "object",
127
  "required": ["id", "name", "role"],
@@ -135,7 +179,17 @@ ANALYSIS_SCHEMA: dict[str, Any] = {
135
  "properties": {
136
  "shape": {
137
  "type": "string",
138
- "enum": ["box", "cylinder", "sphere", "gear", "rod"],
 
 
 
 
 
 
 
 
 
 
139
  },
140
  "size": {
141
  "type": "array",
@@ -156,6 +210,8 @@ ANALYSIS_SCHEMA: dict[str, Any] = {
156
  "maxItems": 3,
157
  },
158
  "teeth": {"type": "integer", "minimum": 6, "maximum": 80},
 
 
159
  "color": {"type": "string"},
160
  },
161
  },
@@ -165,7 +221,15 @@ ANALYSIS_SCHEMA: dict[str, Any] = {
165
  "properties": {
166
  "type": {
167
  "type": "string",
168
- "enum": ["rotate", "translate", "oscillate", "static"],
 
 
 
 
 
 
 
 
169
  },
170
  "axis": {
171
  "type": "array",
@@ -182,6 +246,13 @@ ANALYSIS_SCHEMA: dict[str, Any] = {
182
  "minItems": 2,
183
  "maxItems": 2,
184
  },
 
 
 
 
 
 
 
185
  },
186
  },
187
  "annotation": {
@@ -211,8 +282,8 @@ ANALYSIS_SCHEMA: dict[str, Any] = {
211
  }
212
 
213
 
214
- _SHAPES = {"box", "cylinder", "sphere", "gear", "rod"}
215
- _MOTIONS = {"rotate", "translate", "oscillate", "static"}
216
  _RENDER_MODES = {"three", "annotate", "unavailable"}
217
  DEFAULT_CONFIDENCE_THRESHOLD = 0.5
218
 
@@ -274,6 +345,8 @@ def validate_analysis(payload: dict[str, Any]) -> dict[str, Any]:
274
  parts = payload.get("parts")
275
  if not isinstance(parts, list) or not parts:
276
  raise ValueError("Invalid analysis payload at parts: expected a non-empty list")
 
 
277
 
278
  for index, part in enumerate(parts):
279
  path = f"parts.{index}"
@@ -296,6 +369,10 @@ def validate_analysis(payload: dict[str, Any]) -> dict[str, Any]:
296
  _require_number_list(geometry, "rotation", f"{path}.geometry.rotation", 3)
297
  if "teeth" in geometry and not isinstance(geometry["teeth"], int):
298
  raise ValueError(f"Invalid analysis payload at {path}.geometry.teeth: expected an integer")
 
 
 
 
299
  has_geometry = True
300
 
301
  has_annotation = False
@@ -327,7 +404,9 @@ def validate_analysis(payload: dict[str, Any]) -> dict[str, Any]:
327
  _require_number_list(motion, "axis", f"{path}.motion.axis", 3)
328
  if "range" in motion:
329
  _require_number_list(motion, "range", f"{path}.motion.range", 2)
330
- for key in ["speed", "amplitude", "phase"]:
 
 
331
  if key in motion and not _is_number(motion[key]):
332
  raise ValueError(f"Invalid analysis payload at {path}.motion.{key}: expected a number")
333
  elif has_geometry:
 
5
  import math
6
  from typing import Any, Literal
7
 
8
+ MotionType = Literal["rotate", "translate", "oscillate", "static", "screw", "orbit", "pulse"]
9
 
10
 
11
  EXAMPLE_ANALYSIS: dict[str, Any] = {
 
19
  "trigger": "handle swings clockwise and counterclockwise",
20
  "motion_sequence": [
21
  "handle applies torque to the outer head",
22
+ "spring-loaded pawl locks into the ratchet gear",
23
  "gear and socket rotate on the drive stroke",
24
  "pawl rides over gear teeth on the return stroke",
25
  ],
 
70
  "name": "spring pawl",
71
  "role": "locks and releases against gear teeth",
72
  "geometry": {
73
+ "shape": "cone",
74
+ "size": [0.34, 0.62, 0.34],
75
  "position": [1.25, 0.3, 0.18],
76
+ "rotation": [0.55, 0, -0.75],
77
  "color": "amber",
78
  },
79
  "motion": {
 
90
  },
91
  },
92
  {
93
+ "id": "pawl_spring",
94
+ "name": "pawl return spring",
95
+ "role": "pushes the pawl back into the gear teeth",
96
  "geometry": {
97
+ "shape": "spring",
98
+ "coils": 6,
99
+ "wire": 0.035,
100
+ "size": [0.34, 0.9, 0.34],
101
+ "position": [1.55, 0.32, -0.2],
102
+ "rotation": [0.2, 0, 0.45],
103
  "color": "orange",
104
  },
105
+ "motion": {"type": "pulse", "speed": 2.4, "amplitude": 0.08},
106
+ "annotation": {
107
+ "point": [0.72, 0.5],
108
+ "label": "return spring",
109
+ "note": "small spring biasing the locking pawl",
110
+ },
111
+ },
112
+ {
113
+ "id": "selector_pin",
114
+ "name": "selector pin",
115
+ "role": "slides the selector and changes pawl bias direction",
116
+ "geometry": {
117
+ "shape": "capsule",
118
+ "size": [0.22, 0.85, 0.22],
119
+ "position": [-0.95, 0.56, -0.95],
120
+ "rotation": [1.5708, 0, 0.25],
121
+ "color": "steel",
122
+ },
123
+ "motion": {
124
+ "type": "translate",
125
+ "axis": [1, 0, 0],
126
+ "range": [-0.08, 0.08],
127
+ "speed": 1.3,
128
+ },
129
+ "annotation": {
130
+ "point": [0.38, 0.3],
131
+ "label": "selector pin",
132
+ "note": "rounded pin under the direction selector",
133
+ },
134
+ },
135
+ {
136
+ "id": "retaining_ring",
137
+ "name": "retaining ring",
138
+ "role": "keeps the socket gear captured in the head",
139
+ "geometry": {
140
+ "shape": "torus",
141
+ "size": [1.9, 0.18, 1.9],
142
+ "position": [0, 0.43, 0],
143
+ "color": "cyan",
144
+ },
145
  "motion": {"type": "static"},
146
  "annotation": {
147
+ "point": [0.5, 0.36],
148
+ "label": "retaining ring",
149
+ "note": "circular clip around the socket drive",
150
  },
151
  },
152
  ],
 
165
  "motion_sequence": {"type": "array", "items": {"type": "string"}},
166
  "parts": {
167
  "type": "array",
168
+ "maxItems": 6,
169
  "items": {
170
  "type": "object",
171
  "required": ["id", "name", "role"],
 
179
  "properties": {
180
  "shape": {
181
  "type": "string",
182
+ "enum": [
183
+ "box",
184
+ "cylinder",
185
+ "sphere",
186
+ "gear",
187
+ "rod",
188
+ "cone",
189
+ "capsule",
190
+ "torus",
191
+ "spring",
192
+ ],
193
  },
194
  "size": {
195
  "type": "array",
 
210
  "maxItems": 3,
211
  },
212
  "teeth": {"type": "integer", "minimum": 6, "maximum": 80},
213
+ "coils": {"type": "integer", "minimum": 2, "maximum": 12},
214
+ "wire": {"type": "number"},
215
  "color": {"type": "string"},
216
  },
217
  },
 
221
  "properties": {
222
  "type": {
223
  "type": "string",
224
+ "enum": [
225
+ "rotate",
226
+ "translate",
227
+ "oscillate",
228
+ "static",
229
+ "screw",
230
+ "orbit",
231
+ "pulse",
232
+ ],
233
  },
234
  "axis": {
235
  "type": "array",
 
246
  "minItems": 2,
247
  "maxItems": 2,
248
  },
249
+ "pitch": {"type": "number"},
250
+ "pivot": {
251
+ "type": "array",
252
+ "items": {"type": "number"},
253
+ "minItems": 3,
254
+ "maxItems": 3,
255
+ },
256
  },
257
  },
258
  "annotation": {
 
282
  }
283
 
284
 
285
+ _SHAPES = {"box", "cylinder", "sphere", "gear", "rod", "cone", "capsule", "torus", "spring"}
286
+ _MOTIONS = {"rotate", "translate", "oscillate", "static", "screw", "orbit", "pulse"}
287
  _RENDER_MODES = {"three", "annotate", "unavailable"}
288
  DEFAULT_CONFIDENCE_THRESHOLD = 0.5
289
 
 
345
  parts = payload.get("parts")
346
  if not isinstance(parts, list) or not parts:
347
  raise ValueError("Invalid analysis payload at parts: expected a non-empty list")
348
+ if len(parts) > 6:
349
+ raise ValueError("Invalid analysis payload at parts: expected no more than 6 items")
350
 
351
  for index, part in enumerate(parts):
352
  path = f"parts.{index}"
 
369
  _require_number_list(geometry, "rotation", f"{path}.geometry.rotation", 3)
370
  if "teeth" in geometry and not isinstance(geometry["teeth"], int):
371
  raise ValueError(f"Invalid analysis payload at {path}.geometry.teeth: expected an integer")
372
+ if "coils" in geometry and not isinstance(geometry["coils"], int):
373
+ raise ValueError(f"Invalid analysis payload at {path}.geometry.coils: expected an integer")
374
+ if "wire" in geometry and not _is_number(geometry["wire"]):
375
+ raise ValueError(f"Invalid analysis payload at {path}.geometry.wire: expected a number")
376
  has_geometry = True
377
 
378
  has_annotation = False
 
404
  _require_number_list(motion, "axis", f"{path}.motion.axis", 3)
405
  if "range" in motion:
406
  _require_number_list(motion, "range", f"{path}.motion.range", 2)
407
+ if "pivot" in motion:
408
+ _require_number_list(motion, "pivot", f"{path}.motion.pivot", 3)
409
+ for key in ["speed", "amplitude", "phase", "pitch"]:
410
  if key in motion and not _is_number(motion[key]):
411
  raise ValueError(f"Invalid analysis payload at {path}.motion.{key}: expected a number")
412
  elif has_geometry: