Split vision prompt roles

Co-authored-by: OpenAI Codex <codex@openai.com>

AGENTS.md

@@ -311,6 +311,24 @@ technical cutaway animation.
   real Modal analysis for `piston mechanism` at `0.85` confidence with 3 parts,
   `/generate_scene` returned `renderer: three`, `render_mode: three`, and no
   HTML field, and a high confidence threshold returned `render_mode: annotate`.
+- Implemented the `FEATURE4.md` prompt-role split on June 15, 2026: moved the
+  invariant analysis contract into `VISION_SYSTEM_PROMPT`, reduced
+  `build_vision_prompt()` to a backward-compatible per-image user-prompt alias,
+  added `build_vision_messages()`, and wired Modal llama.cpp analysis calls to
+  pass `-sys` with a fallback that prepends the system text only if the deployed
+  binary rejects the system-prompt flag.
+- Verification after the `FEATURE4.md` pass: prompt-helper checks,
+  schema/parser/coercion checks, FastAPI `TestClient` checks for `/`,
+  `/analyze_image`, and `/generate_scene`, and a `modal_app.py` import/helper
+  check all passed locally. Modal dev raw analysis parsed strict JSON in
+  `28.59s` with `coerced_render_mode: three`, and the endpoint-style Modal
+  diagnostic returned a validated two-part mechanism payload. Stable Modal
+  deploy completed on June 15, 2026, and the stable unauthenticated
+  `runtime_probe` request returned `401 Unauthorized`. Authenticated stable
+  Modal and private-Space verification could not be run from the local shell
+  because `SNAP2SIM_API_TOKEN` / Hugging Face auth tokens were not set locally.
+  GitHub push/sync and authenticated private-Space verification still need to
+  run before public demo claims rely on the prompt-role split.
 
 ## Next Work
 

modal_app.py

@@ -17,7 +17,7 @@ import modal
 from fastapi import Header, HTTPException
 
 from snap2sim.model_io import coerce_analysis_response, parse_analysis_response
-from snap2sim.prompts import build_vision_prompt
+from snap2sim.prompts import build_vision_messages
 from snap2sim.schema import EXAMPLE_ANALYSIS, select_render_mode, validate_analysis
 
 
@@ -291,6 +291,7 @@ def smoke_test_llamacpp_image() -> dict[str, Any]:
 def run_llamacpp_prompt(
     prompt: str,
     image_path: Path | None = None,
+    system_prompt: str | None = None,
     max_tokens: int = 3072,
     timeout_seconds: int = 300,
     ctx_size: int = 8192,
@@ -299,41 +300,72 @@ def run_llamacpp_prompt(
     import subprocess
 
     model_path, mmproj_path = ensure_runtime_assets()
-    cmd = [
-        "/opt/llama.cpp/build/bin/llama-mtmd-cli",
-        "-m",
-        str(model_path),
-        "--mmproj",
-        str(mmproj_path),
-        "-p",
-        prompt,
-        "-n",
-        str(max_tokens),
-        "-c",
-        str(ctx_size),
-        "--temp",
-        "0.2",
-    ]
-    if image_path is not None:
-        cmd.extend(["--image", str(image_path)])
 
-    try:
-        proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout_seconds)
-    except subprocess.TimeoutExpired as exc:
-        partial_output = "\n".join(
-            part.decode("utf-8", errors="replace") if isinstance(part, bytes) else part
-            for part in [exc.stdout, exc.stderr]
-            if part
-        ).strip()
-        raise TimeoutError(
-            f"llama.cpp timed out after {timeout_seconds}s: {partial_output[-2000:]}"
-        ) from exc
+    def build_cmd(user_prompt: str, system: str | None) -> list[str]:
+        cmd = [
+            "/opt/llama.cpp/build/bin/llama-mtmd-cli",
+            "-m",
+            str(model_path),
+            "--mmproj",
+            str(mmproj_path),
+        ]
+        if system:
+            cmd.extend(["-sys", system])
+        if image_path is not None:
+            cmd.extend(["--image", str(image_path)])
+        cmd.extend(
+            [
+                "-p",
+                user_prompt,
+                "-n",
+                str(max_tokens),
+                "-c",
+                str(ctx_size),
+                "--temp",
+                "0.2",
+            ]
+        )
+        return cmd
+
+    def run_cmd(cmd: list[str]) -> subprocess.CompletedProcess[str]:
+        try:
+            return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout_seconds)
+        except subprocess.TimeoutExpired as exc:
+            partial_output = "\n".join(
+                part.decode("utf-8", errors="replace") if isinstance(part, bytes) else part
+                for part in [exc.stdout, exc.stderr]
+                if part
+            ).strip()
+            raise TimeoutError(
+                f"llama.cpp timed out after {timeout_seconds}s: {partial_output[-2000:]}"
+            ) from exc
+
+    proc = run_cmd(build_cmd(prompt, system_prompt))
     output = "\n".join(part for part in [proc.stdout, proc.stderr] if part).strip()
+    if proc.returncode != 0 and system_prompt and _system_prompt_flag_unsupported(output):
+        combined_prompt = f"{system_prompt.strip()}\n\nUser request:\n{prompt}"
+        proc = run_cmd(build_cmd(combined_prompt, None))
+        output = "\n".join(part for part in [proc.stdout, proc.stderr] if part).strip()
     if proc.returncode != 0:
         raise RuntimeError(f"llama.cpp exited with {proc.returncode}: {output[-2000:]}")
     return output
 
 
+def _system_prompt_flag_unsupported(output: str) -> bool:
+    lowered = output.lower()
+    return any(
+        marker in lowered
+        for marker in [
+            "unknown argument: -sys",
+            "unknown argument '-sys'",
+            "unknown option: -sys",
+            "unrecognized option '-sys'",
+            "invalid option -- 'sys'",
+            "error: unknown argument",
+        ]
+    )
+
+
 def write_payload_image(payload: dict[str, Any]) -> Path:
     """Decode an image_base64 payload into a temporary RGB JPEG."""
     import base64
@@ -385,9 +417,11 @@ def write_payload_image(payload: dict[str, Any]) -> Path:
 def analyze_image_llamacpp_payload(payload: dict[str, Any]) -> dict[str, Any]:
     image_path = write_payload_image(payload)
     try:
+        system_prompt, user_prompt = build_vision_messages()
         response = run_llamacpp_prompt(
-            build_vision_prompt(),
+            user_prompt,
             image_path=image_path,
+            system_prompt=system_prompt,
             max_tokens=4096,
             timeout_seconds=300,
             ctx_size=8192,
@@ -485,8 +519,6 @@ def runtime_probe(authorization: str = Header(default="")) -> dict[str, Any]:
 @modal.fastapi_endpoint(method="POST")
 def analyze_image(payload: dict[str, Any], authorization: str = Header(default="")) -> dict[str, Any]:
     require_authorization(authorization)
-    _ = payload.get("image_base64", "")
-    _prompt = build_vision_prompt()
     if runtime_config()["runtime_mode"] != "placeholder":
         raise NotImplementedError(
             "Use the analyze_image_llamacpp endpoint for the verified Nemotron "
@@ -509,9 +541,11 @@ def analyze_image_llamacpp_raw_task(payload: dict[str, Any]) -> dict[str, Any]:
     image_path = write_payload_image(payload)
     start = time.monotonic()
     try:
+        system_prompt, user_prompt = build_vision_messages()
         response = run_llamacpp_prompt(
-            build_vision_prompt(),
+            user_prompt,
             image_path=image_path,
+            system_prompt=system_prompt,
             max_tokens=4096,
             timeout_seconds=300,
             ctx_size=8192,

snap2sim/prompts.py

@@ -4,67 +4,81 @@ from __future__ import annotations
 
 VISION_SYSTEM_PROMPT = """You are a mechanical teardown analyst for an annotated
 technical cutaway demo. Reason briefly if useful, then emit exactly one JSON
-object matching the schema. Prefer the simplest truthful primitive mechanism.
+object with no markdown. Prefer the simplest truthful primitive mechanism.
 If the photo is ambiguous, lower confidence and use visible photo annotations
-instead of forcing speculative 3D geometry."""
+instead of forcing speculative 3D geometry.
 
+Required top-level keys: component, confidence, summary, trigger,
+motion_sequence, parts. Optional top-level render_mode is three, annotate, or
+unavailable.
+Each part requires: id, name, role, and either geometry plus motion, or
+annotation when the visible component can be located but 3D geometry is
+uncertain.
 
-def build_vision_prompt() -> str:
+Shapes, with size always [x, y, z] extents:
+- box: plates, housings, blocks, levers, selectors
+- cylinder: shafts, sleeves, bushings, drums, pins
+- cone: valve cones, tips, nozzles, tapers
+- capsule: pistons, rollers, dowel pins, plungers, bearings
+- sphere: balls, detents, ball bearings, nodes
+- rod: links, tie rods, thin axles, connecting rods
+- gear: toothed wheels, ratchets, cogs; set teeth when useful
+- torus: o-rings, snap rings, seals, washers, single coils
+- spring: helical springs and coils; set coils when useful
+
+Motions, with axis as a numeric vector like [0, 1, 0]:
+- static: fixed structure or housing
+- rotate: continuous spin; use speed
+- oscillate: sinusoidal twist; use amplitude and speed
+- translate: slide along axis; use range [min, max]
+- screw: spin plus advance along axis; use pitch for helical action
+- orbit: revolve around pivot [x, y, z]
+- pulse: scale breathing for diaphragms, springs, valves, pumps
+
+Every geometry must use size: [x, y, z] and position: [x, y, z]. Do not use
+radius, height, length, width, or depth fields. Every motion axis must be a
+numeric vector such as [0, 1, 0], never a string like x, y, or z.
+
+Use 2 to 6 parts; prefer the fewest that explain the mechanism. Keep names and
+descriptions short. When possible, include annotation.point in normalized image
+coordinates [x, y] with origin at top-left, plus a short annotation.note.
+Optional annotation.box is [x, y, width, height], also normalized from 0 to 1.
+
+Use this compact shape:
+{
+  "component": "short component name",
+  "confidence": 0.7,
+  "summary": "one or two sentences",
+  "trigger": "what starts the mechanism",
+  "motion_sequence": ["step one", "step two"],
+  "parts": [
+    {
+      "id": "part_id",
+      "name": "part name",
+      "role": "mechanical role",
+      "geometry": {"shape": "spring", "size": [0.4, 1.2, 0.4], "position": [0, 0, 0], "coils": 6},
+      "motion": {"type": "pulse", "speed": 2, "amplitude": 0.08},
+      "annotation": {"point": [0.5, 0.5], "label": "visible label", "note": "short visible clue"}
+    }
+  ]
+}
+
+Optional geometry fields: rotation, teeth, coils, wire, color. Optional motion
+fields: axis, speed, amplitude, phase, range, pitch, pivot. Include optional
+fields only when useful."""
+
+
+def build_vision_user_prompt() -> str:
     return (
-        "Analyze the uploaded hardware component as a cutaway mechanism. "
-        "Final answer must be one JSON object with no markdown.\n\n"
-        "Required top-level keys: component, confidence, summary, trigger, "
-        "motion_sequence, parts. Optional top-level render_mode is three, "
-        "annotate, or unavailable.\n"
-        "Each part requires: id, name, role, and either geometry plus motion, "
-        "or annotation when the visible component can be located but 3D "
-        "geometry is uncertain.\n"
-        "Shapes, with size always [x, y, z] extents:\n"
-        "- box: plates, housings, blocks, levers, selectors\n"
-        "- cylinder: shafts, sleeves, bushings, drums, pins\n"
-        "- cone: valve cones, tips, nozzles, tapers\n"
-        "- capsule: pistons, rollers, dowel pins, plungers, bearings\n"
-        "- sphere: balls, detents, ball bearings, nodes\n"
-        "- rod: links, tie rods, thin axles, connecting rods\n"
-        "- gear: toothed wheels, ratchets, cogs; set teeth when useful\n"
-        "- torus: o-rings, snap rings, seals, washers, single coils\n"
-        "- spring: helical springs and coils; set coils when useful\n\n"
-        "Motions, with axis as a numeric vector like [0, 1, 0]:\n"
-        "- static: fixed structure or housing\n"
-        "- rotate: continuous spin; use speed\n"
-        "- oscillate: sinusoidal twist; use amplitude and speed\n"
-        "- translate: slide along axis; use range [min, max]\n"
-        "- screw: spin plus advance along axis; use pitch for helical action\n"
-        "- orbit: revolve around pivot [x, y, z]\n"
-        "- pulse: scale breathing for diaphragms, springs, valves, pumps\n\n"
-        "Every geometry must use size: [x, y, z] and position: [x, y, z]. "
-        "Do not use radius, height, length, width, or depth fields. Every "
-        "motion axis must be a numeric vector such as [0, 1, 0], never a string "
-        "like x, y, or z.\n\n"
-        "Use 2 to 6 parts; prefer the fewest that explain the mechanism. "
-        "Keep names and descriptions short. When possible, "
-        "include annotation.point in normalized image coordinates [x, y] with "
-        "origin at top-left, plus a short annotation.note. Optional "
-        "annotation.box is [x, y, width, height], also normalized from 0 to 1.\n\n"
-        "Use this compact shape:\n"
-        "{\n"
-        '  "component": "short component name",\n'
-        '  "confidence": 0.7,\n'
-        '  "summary": "one or two sentences",\n'
-        '  "trigger": "what starts the mechanism",\n'
-        '  "motion_sequence": ["step one", "step two"],\n'
-        '  "parts": [\n'
-        "    {\n"
-        '      "id": "part_id",\n'
-        '      "name": "part name",\n'
-        '      "role": "mechanical role",\n'
-        '      "geometry": {"shape": "spring", "size": [0.4, 1.2, 0.4], "position": [0, 0, 0], "coils": 6},\n'
-        '      "motion": {"type": "pulse", "speed": 2, "amplitude": 0.08},\n'
-        '      "annotation": {"point": [0.5, 0.5], "label": "visible label", "note": "short visible clue"}\n'
-        "    }\n"
-        "  ]\n"
-        "}\n\n"
-        "Optional geometry fields: rotation, teeth, coils, wire, color. Optional "
-        "motion fields: axis, speed, amplitude, phase, range, pitch, pivot. "
-        "Include optional fields only when useful."
+        "Analyze the hardware component in this photo as a cutaway mechanism. "
+        "Return the analysis JSON for the visible component."
     )
+
+
+def build_vision_prompt() -> str:
+    """Backward-compatible alias for the per-image user prompt."""
+    return build_vision_user_prompt()
+
+
+def build_vision_messages() -> tuple[str, str]:
+    return VISION_SYSTEM_PROMPT, build_vision_user_prompt()