Added option to create a cube at position the user is looking at

app.py

@@ -151,7 +151,7 @@ def get_gpt_client():
     return gpt_client
 
 
-def chat_response(message, history):
+def chat_response(message, history, crosshair_position=None):
     """Handle chat messages using GPT with tool calling"""
     global current_scene_id
 
@@ -184,7 +184,8 @@ I'm an AI assistant that can help you build 3D scenes using natural language.
 
     try:
         client = get_gpt_client()
-        response, action_data = client.chat(message)
+        # Pass crosshair position to chat client for context-aware object placement
+        response, action_data = client.chat(message, crosshair_position=crosshair_position)
         return response, action_data
     except Exception as e:
         import traceback
@@ -334,6 +335,16 @@ with gr.Blocks(title="GCP - Game Context Protocol") as demo:
                 if (event.data && event.data.action === 'objectDeselected') {
                     window.selectedObjectId = null;
                 }
+
+                if (event.data && event.data.action === 'crosshairPosition') {
+                    window.crosshairPosition = event.data.data;
+                    // Update hidden Gradio textbox so Python can read it
+                    const crosshairInput = document.querySelector('#crosshair-pos textarea');
+                    if (crosshairInput) {
+                        crosshairInput.value = event.data.data ? JSON.stringify(event.data.data) : '';
+                        crosshairInput.dispatchEvent(new Event('input', { bubbles: true }));
+                    }
+                }
             });
 
             // Initialize toast container on load
@@ -363,6 +374,10 @@ with gr.Blocks(title="GCP - Game Context Protocol") as demo:
     # The flow: bot() returns JSON → action_data updates → .change() fires → JS sends postMessage to iframe
     action_data = gr.Textbox(value="", elem_id="action-data", visible=True, elem_classes=["hidden-action"])
 
+    # Hidden textbox to receive crosshair position from JavaScript
+    # JS updates this via: document.querySelector('#crosshair-pos textarea').value = JSON.stringify(pos)
+    crosshair_pos = gr.Textbox(value="", elem_id="crosshair-pos", visible=True, elem_classes=["hidden-action"])
+
     # Main container - side by side layout: Chat (left) | Viewer (right)
     with gr.Row(elem_id="main-container", equal_height=True):
         # Left column: Chat interface (scale=1 = ~25% width)
@@ -401,7 +416,7 @@ with gr.Blocks(title="GCP - Game Context Protocol") as demo:
         history.append({"role": "user", "content": user_message})
         return "", history
 
-    def bot(history):
+    def bot(history, crosshair_position):
         """Generate bot response"""
         # Gradio 6: content can be a string or list of content blocks
         content = history[-1]["content"]
@@ -414,8 +429,16 @@ with gr.Blocks(title="GCP - Game Context Protocol") as demo:
         else:
             user_message = content
 
-        # Process command (history not used by chat_response)
-        bot_message, action_result = chat_response(user_message, [])
+        # Parse crosshair position from JSON string
+        crosshair_pos_dict = None
+        if crosshair_position:
+            try:
+                crosshair_pos_dict = json.loads(crosshair_position)
+            except:
+                pass
+
+        # Process command with crosshair context
+        bot_message, action_result = chat_response(user_message, [], crosshair_pos_dict)
         history.append({"role": "assistant", "content": bot_message})
 
         # Handle action_result
@@ -507,7 +530,7 @@ with gr.Blocks(title="GCP - Game Context Protocol") as demo:
         queue=False
     ).then(
         bot,
-        [chatbot],
+        [chatbot, crosshair_pos],
         [chatbot, viewer, action_data]
     )
 

chat_client.py

@@ -767,10 +767,15 @@ Be concise but helpful. After making changes, briefly confirm what was done."""
         else:
             return {"error": f"Unknown tool: {name}"}
 
-    def chat(self, user_message: str) -> tuple[str, Optional[Dict[str, Any]]]:
+    def chat(self, user_message: str, crosshair_position: Optional[Dict[str, float]] = None) -> tuple[str, Optional[Dict[str, Any]]]:
         """
         Process a user message and return the response.
 
+        Args:
+            user_message: The user's message
+            crosshair_position: Optional dict with x, y, z coordinates where the crosshair
+                               is pointing on the floor. Used for context-aware object placement.
+
         Returns:
             tuple: (response_text, action_data)
             - response_text: The assistant's response
@@ -782,8 +787,21 @@ Be concise but helpful. After making changes, briefly confirm what was done."""
             "content": user_message
         })
 
+        # Build system prompt with crosshair context
+        system_prompt = self.system_prompt
+        if crosshair_position:
+            system_prompt += f"""
+
+IMPORTANT - Current crosshair position:
+The player is looking at position ({crosshair_position['x']}, {crosshair_position['y']}, {crosshair_position['z']}) on the floor.
+When the user asks to create/add an object WITHOUT specifying a position (e.g., "add a cube", "create a sphere"),
+place it at the crosshair position: x={crosshair_position['x']}, y={crosshair_position['y']}, z={crosshair_position['z']}.
+The y coordinate should be adjusted based on object size (e.g., y=0.5 for a cube of scale 1 so it sits on the floor).
+
+If the user DOES specify a position (e.g., "add a cube at 0, 0, 0"), use their specified position instead."""
+
         # Build messages with system prompt
-        messages = [{"role": "system", "content": self.system_prompt}] + self.conversation_history
+        messages = [{"role": "system", "content": system_prompt}] + self.conversation_history
 
         # Track actions for frontend
         actions = []

frontend/game_viewer.html

@@ -993,6 +993,9 @@
             // Update looked-at object (FPS mode only)
             updateLookedAtObject();
 
+            // Update crosshair floor intersection and send to parent
+            updateCrosshairPosition();
+
             // Render using composer (for outlines) instead of direct renderer
             if (composer) {
                 composer.render();
@@ -1004,6 +1007,72 @@
             if (stats) stats.end();
         }
 
+        // ==================== Crosshair Position Tracking ====================
+
+        let lastCrosshairPosition = null;
+        let crosshairUpdateThrottle = 0;
+
+        /**
+         * Update crosshair floor intersection and send to parent window
+         * This allows the chat to know where to place objects when user doesn't specify position
+         */
+        function updateCrosshairPosition() {
+            // Throttle updates to every 10 frames (~6 times per second at 60fps)
+            crosshairUpdateThrottle++;
+            if (crosshairUpdateThrottle < 10) return;
+            crosshairUpdateThrottle = 0;
+
+            // Only track in FPS mode
+            if (controlMode !== 'fps') {
+                if (lastCrosshairPosition !== null) {
+                    lastCrosshairPosition = null;
+                    window.parent.postMessage({
+                        action: 'crosshairPosition',
+                        data: null
+                    }, '*');
+                }
+                return;
+            }
+
+            // Raycast from camera center (crosshair)
+            const crosshairRaycaster = new THREE.Raycaster();
+            crosshairRaycaster.setFromCamera(new THREE.Vector2(0, 0), camera);
+
+            // Find intersection with ground
+            const groundObjects = scene.children.filter(obj => obj.userData.isGround);
+            const intersects = crosshairRaycaster.intersectObjects(groundObjects);
+
+            if (intersects.length > 0) {
+                const point = intersects[0].point;
+                const newPosition = {
+                    x: Math.round(point.x * 100) / 100,
+                    y: Math.round(point.y * 100) / 100,
+                    z: Math.round(point.z * 100) / 100
+                };
+
+                // Only send update if position changed significantly
+                if (!lastCrosshairPosition ||
+                    Math.abs(newPosition.x - lastCrosshairPosition.x) > 0.1 ||
+                    Math.abs(newPosition.z - lastCrosshairPosition.z) > 0.1) {
+
+                    lastCrosshairPosition = newPosition;
+                    window.parent.postMessage({
+                        action: 'crosshairPosition',
+                        data: newPosition
+                    }, '*');
+                }
+            } else {
+                // No intersection - looking at sky or beyond floor
+                if (lastCrosshairPosition !== null) {
+                    lastCrosshairPosition = null;
+                    window.parent.postMessage({
+                        action: 'crosshairPosition',
+                        data: null
+                    }, '*');
+                }
+            }
+        }
+
         // ==================== Scene Control Functions ====================
 
         /**