Spaces:
Sleeping
Sleeping
| """ | |
| Multi-LLM Chat Client for GCP (Game Context Protocol) | |
| This module provides an intelligent chat interface that uses either OpenAI GPT | |
| or Google Gemini with function calling to interact with the GCP tools. | |
| Supports: | |
| - OpenAI GPT-4o-mini (default) | |
| - Google Gemini 2.0 Flash | |
| """ | |
| import os | |
| import json | |
| from typing import Optional, Dict, Any, List, Literal | |
| # Load .env file if present | |
| from dotenv import load_dotenv | |
| load_dotenv() | |
| from openai import OpenAI | |
| # Gemini import (optional - may not be installed) | |
| try: | |
| import google.generativeai as genai | |
| GEMINI_AVAILABLE = True | |
| except ImportError: | |
| GEMINI_AVAILABLE = False | |
| genai = None | |
| LLMProvider = Literal["openai", "gemini"] | |
| # Import GCP tools | |
| from backend.tools.scene_tools import ( | |
| create_game_scene, | |
| add_game_object, | |
| remove_game_object, | |
| set_scene_lighting, | |
| get_scene_info, | |
| add_brick, | |
| ) | |
| from backend.tools.player_tools import ( | |
| set_player_speed, | |
| set_jump_force, | |
| set_mouse_sensitivity, | |
| set_gravity, | |
| set_player_dimensions, | |
| set_movement_acceleration, | |
| set_air_control, | |
| set_camera_fov, | |
| set_vertical_look_limits, | |
| get_player_config, | |
| ) | |
| from backend.tools.rendering_tools import ( | |
| add_light, | |
| remove_light, | |
| update_light, | |
| get_lights, | |
| update_object_material, | |
| set_background_color, | |
| set_fog, | |
| update_material_to_toon, | |
| ) | |
| from backend.tools.environment_tools import ( | |
| add_skybox, | |
| remove_skybox, | |
| add_particles, | |
| remove_particles, | |
| ) | |
| from backend.tools.ui_tools import ( | |
| render_text_on_screen, | |
| render_bar_on_screen, | |
| remove_ui_element, | |
| ) | |
| from backend.game_models import create_vector3, create_material | |
| # Tool definitions for OpenAI function calling | |
| TOOLS = [ | |
| # Scene Tools | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "create_scene", | |
| "description": "Create a new 3D scene/level", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "name": {"type": "string", "description": "Name of the scene"}, | |
| "description": {"type": "string", "description": "Description of the scene"}, | |
| "world_width": {"type": "number", "description": "Width of the world in units (default: 100)"}, | |
| "world_height": {"type": "number", "description": "Height of the world in units (default: 100)"}, | |
| "world_depth": {"type": "number", "description": "Depth of the world in units (default: 100)"}, | |
| "lighting_preset": {"type": "string", "enum": ["day", "night", "sunset", "studio"], "description": "Lighting preset"}, | |
| }, | |
| "required": [] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "add_object", | |
| "description": "Add a 3D object to the scene. If no position (x, y, z) is specified, the object will spawn in front of the player's camera (Minecraft-style placement).", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "object_type": {"type": "string", "enum": ["cube", "sphere", "cylinder", "plane", "cone", "torus"], "description": "Type of object"}, | |
| "name": {"type": "string", "description": "Name for the object"}, | |
| "x": {"type": "number", "description": "X position (optional - omit to place in front of player)"}, | |
| "y": {"type": "number", "description": "Y position (optional - auto-calculated to sit on ground)"}, | |
| "z": {"type": "number", "description": "Z position (optional - omit to place in front of player)"}, | |
| "scale_x": {"type": "number", "description": "X scale (default: 1)"}, | |
| "scale_y": {"type": "number", "description": "Y scale (default: 1)"}, | |
| "scale_z": {"type": "number", "description": "Z scale (default: 1)"}, | |
| "color": {"type": "string", "description": "Hex color code (e.g., #ff0000 for red)"}, | |
| }, | |
| "required": ["scene_id"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "remove_object", | |
| "description": "Remove an object from the scene", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "object_id": {"type": "string", "description": "ID of the object to remove"}, | |
| }, | |
| "required": ["scene_id", "object_id"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "set_lighting", | |
| "description": "Set the lighting preset for the scene", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "preset": {"type": "string", "enum": ["day", "night", "sunset", "studio"], "description": "Lighting preset"}, | |
| }, | |
| "required": ["scene_id"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "get_scene_info", | |
| "description": "Get detailed information about a scene including all objects and settings", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| }, | |
| "required": ["scene_id"] | |
| } | |
| } | |
| }, | |
| # Player Tools | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "set_player_speed", | |
| "description": "Set the player's movement speed in units per second", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "walk_speed": {"type": "number", "description": "Movement speed in units/second"}, | |
| }, | |
| "required": ["scene_id", "walk_speed"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "set_jump_force", | |
| "description": "Set the player's jump force (initial upward velocity)", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "jump_force": {"type": "number", "description": "Jump force in m/s"}, | |
| }, | |
| "required": ["scene_id", "jump_force"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "set_mouse_sensitivity", | |
| "description": "Set mouse look sensitivity and Y-axis inversion", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "sensitivity": {"type": "number", "description": "Mouse sensitivity multiplier"}, | |
| "invert_y": {"type": "boolean", "description": "Invert Y-axis"}, | |
| }, | |
| "required": ["scene_id"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "set_gravity", | |
| "description": "Set the world's gravity strength", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "gravity": {"type": "number", "description": "Gravity in m/s² (negative = downward, e.g., -9.82 for Earth)"}, | |
| }, | |
| "required": ["scene_id", "gravity"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "set_player_dimensions", | |
| "description": "Set player collision capsule dimensions", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "height": {"type": "number", "description": "Player height in meters"}, | |
| "radius": {"type": "number", "description": "Player radius in meters"}, | |
| "eye_height": {"type": "number", "description": "Camera height from feet"}, | |
| }, | |
| "required": ["scene_id"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "set_camera_fov", | |
| "description": "Set camera field of view", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "fov": {"type": "number", "description": "Field of view in degrees (60-120)"}, | |
| }, | |
| "required": ["scene_id", "fov"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "get_player_config", | |
| "description": "Get current player configuration including speed, jump force, gravity, dimensions, etc.", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| }, | |
| "required": ["scene_id"] | |
| } | |
| } | |
| }, | |
| # Rendering Tools | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "add_light", | |
| "description": "Add a light source to the scene", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "light_type": {"type": "string", "enum": ["ambient", "directional", "point", "spot"], "description": "Type of light"}, | |
| "name": {"type": "string", "description": "Unique name for the light"}, | |
| "color": {"type": "string", "description": "Hex color code"}, | |
| "intensity": {"type": "number", "description": "Light intensity (0-2)"}, | |
| "x": {"type": "number", "description": "X position"}, | |
| "y": {"type": "number", "description": "Y position"}, | |
| "z": {"type": "number", "description": "Z position"}, | |
| "cast_shadow": {"type": "boolean", "description": "Enable shadows"}, | |
| }, | |
| "required": ["scene_id", "light_type", "name"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "remove_light", | |
| "description": "Remove a light from the scene", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "light_name": {"type": "string", "description": "Name of the light to remove"}, | |
| }, | |
| "required": ["scene_id", "light_name"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "get_lights", | |
| "description": "Get all lights in the scene", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| }, | |
| "required": ["scene_id"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "update_object_material", | |
| "description": "Update an object's material properties (color, metalness, roughness, opacity, emissive glow)", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "object_id": {"type": "string", "description": "ID of the object"}, | |
| "color": {"type": "string", "description": "Hex color code"}, | |
| "metalness": {"type": "number", "description": "Metalness (0=matte, 1=metal)"}, | |
| "roughness": {"type": "number", "description": "Roughness (0=shiny, 1=rough)"}, | |
| "opacity": {"type": "number", "description": "Opacity (0=invisible, 1=solid)"}, | |
| "emissive": {"type": "string", "description": "Emissive color for glow"}, | |
| "emissive_intensity": {"type": "number", "description": "Emissive intensity (0-1)"}, | |
| }, | |
| "required": ["scene_id", "object_id"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "set_background_color", | |
| "description": "Set scene background color or gradient", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "color": {"type": "string", "description": "Hex color for solid background"}, | |
| "bg_type": {"type": "string", "enum": ["solid", "gradient"], "description": "Background type"}, | |
| "gradient_top": {"type": "string", "description": "Top color for gradient"}, | |
| "gradient_bottom": {"type": "string", "description": "Bottom color for gradient"}, | |
| }, | |
| "required": ["scene_id"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "set_fog", | |
| "description": "Add or remove atmospheric fog", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "enabled": {"type": "boolean", "description": "Enable or disable fog"}, | |
| "color": {"type": "string", "description": "Fog color"}, | |
| "near": {"type": "number", "description": "Fog start distance"}, | |
| "far": {"type": "number", "description": "Fog end distance"}, | |
| "density": {"type": "number", "description": "Fog density (for exponential fog)"}, | |
| }, | |
| "required": ["scene_id", "enabled"] | |
| } | |
| } | |
| }, | |
| # Environment Tools | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "add_skybox", | |
| "description": "Add a procedural sky to the scene. Presets: day, sunset, noon, dawn, night", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "preset": {"type": "string", "enum": ["day", "sunset", "noon", "dawn", "night"], "description": "Sky preset"}, | |
| "turbidity": {"type": "number", "description": "Haziness 2-20 (default: 10)"}, | |
| "rayleigh": {"type": "number", "description": "Blue sky intensity 0-4 (default: 2)"}, | |
| "sun_elevation": {"type": "number", "description": "Sun angle from horizon -90 to 90 (default: 45)"}, | |
| "sun_azimuth": {"type": "number", "description": "Sun compass direction 0-360 (default: 180)"}, | |
| }, | |
| "required": ["scene_id"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "remove_skybox", | |
| "description": "Remove the skybox from the scene", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| }, | |
| "required": ["scene_id"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "add_particles", | |
| "description": "Add particle effects. Presets: fire, smoke, sparkle, rain, snow", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "preset": {"type": "string", "enum": ["fire", "smoke", "sparkle", "rain", "snow"], "description": "Particle preset"}, | |
| "x": {"type": "number", "description": "X position for localized effects"}, | |
| "y": {"type": "number", "description": "Y position for localized effects"}, | |
| "z": {"type": "number", "description": "Z position for localized effects"}, | |
| "particle_id": {"type": "string", "description": "Optional unique ID"}, | |
| }, | |
| "required": ["scene_id", "preset"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "remove_particles", | |
| "description": "Remove a particle system from the scene", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "particle_id": {"type": "string", "description": "ID of the particle system to remove"}, | |
| }, | |
| "required": ["scene_id", "particle_id"] | |
| } | |
| } | |
| }, | |
| # UI Tools | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "render_text_on_screen", | |
| "description": "Render text on the screen as a 2D overlay", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "text": {"type": "string", "description": "Text to display"}, | |
| "x": {"type": "number", "description": "Horizontal position 0-100% (default: 50)"}, | |
| "y": {"type": "number", "description": "Vertical position 0-100% (default: 10)"}, | |
| "font_size": {"type": "integer", "description": "Font size in pixels (default: 24)"}, | |
| "color": {"type": "string", "description": "Text color hex (default: #ffffff)"}, | |
| "text_id": {"type": "string", "description": "Optional unique ID for updates"}, | |
| "background_color": {"type": "string", "description": "Optional background color"}, | |
| }, | |
| "required": ["scene_id", "text"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "render_bar_on_screen", | |
| "description": "Render a progress/health bar on screen", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "x": {"type": "number", "description": "Horizontal position 0-100% (default: 10)"}, | |
| "y": {"type": "number", "description": "Vertical position 0-100% (default: 10)"}, | |
| "width": {"type": "number", "description": "Bar width in pixels (default: 200)"}, | |
| "height": {"type": "number", "description": "Bar height in pixels (default: 20)"}, | |
| "value": {"type": "number", "description": "Current value (default: 100)"}, | |
| "max_value": {"type": "number", "description": "Max value (default: 100)"}, | |
| "bar_color": {"type": "string", "description": "Fill color (default: #00ff00)"}, | |
| "bar_id": {"type": "string", "description": "Optional unique ID for updates"}, | |
| "label": {"type": "string", "description": "Optional label above bar"}, | |
| "show_value": {"type": "boolean", "description": "Show numeric value"}, | |
| }, | |
| "required": ["scene_id"] | |
| } | |
| } | |
| }, | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "remove_ui_element", | |
| "description": "Remove a UI element from the screen", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "element_id": {"type": "string", "description": "ID of the text or bar to remove"}, | |
| }, | |
| "required": ["scene_id", "element_id"] | |
| } | |
| } | |
| }, | |
| # Toon Material | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "update_material_to_toon", | |
| "description": "Apply toon/cel-shading to an object for cartoon-like appearance", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "object_id": {"type": "string", "description": "ID of the object"}, | |
| "enabled": {"type": "boolean", "description": "Enable/disable toon shading (default: true)"}, | |
| "color": {"type": "string", "description": "Base color (optional)"}, | |
| "gradient_steps": {"type": "integer", "description": "Shading steps 2-10 (default: 3)"}, | |
| "outline": {"type": "boolean", "description": "Add outline effect (default: true)"}, | |
| "outline_color": {"type": "string", "description": "Outline color (default: #000000)"}, | |
| "outline_thickness": {"type": "number", "description": "Outline thickness 0.01-0.1 (default: 0.03)"}, | |
| }, | |
| "required": ["scene_id", "object_id"] | |
| } | |
| } | |
| }, | |
| # Brick Blocks | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "add_brick", | |
| "description": "Add a LEGO-style brick from the Kenney brick kit. Types: brick_1x1, brick_1x2, brick_1x4, brick_2x2, brick_2x4, plate_1x2, plate_2x2, plate_2x4, plate_4x4, slope_2x2", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "scene_id": {"type": "string", "description": "ID of the scene"}, | |
| "brick_type": {"type": "string", "enum": ["brick_1x1", "brick_1x2", "brick_1x4", "brick_2x2", "brick_2x4", "plate_1x2", "plate_2x2", "plate_2x4", "plate_4x4", "slope_2x2"], "description": "Type of brick"}, | |
| "x": {"type": "number", "description": "X position"}, | |
| "y": {"type": "number", "description": "Y position"}, | |
| "z": {"type": "number", "description": "Z position"}, | |
| "rotation_y": {"type": "number", "description": "Y rotation in degrees"}, | |
| "color": {"type": "string", "description": "Hex color (default: #ff0000)"}, | |
| "name": {"type": "string", "description": "Optional name"}, | |
| }, | |
| "required": ["scene_id", "brick_type"] | |
| } | |
| } | |
| }, | |
| ] | |
| def _convert_schema_for_gemini(schema: Dict) -> Dict: | |
| """Convert OpenAI JSON schema to Gemini format.""" | |
| if not schema: | |
| return {} | |
| result = {} | |
| # Convert type | |
| if "type" in schema: | |
| type_map = { | |
| "object": "OBJECT", | |
| "string": "STRING", | |
| "number": "NUMBER", | |
| "integer": "INTEGER", | |
| "boolean": "BOOLEAN", | |
| "array": "ARRAY" | |
| } | |
| result["type"] = type_map.get(schema["type"], "STRING") | |
| # Convert properties | |
| if "properties" in schema: | |
| result["properties"] = {} | |
| for key, val in schema["properties"].items(): | |
| result["properties"][key] = _convert_schema_for_gemini(val) | |
| # Copy other fields | |
| if "description" in schema: | |
| result["description"] = schema["description"] | |
| if "enum" in schema: | |
| result["enum"] = schema["enum"] | |
| if "required" in schema: | |
| result["required"] = schema["required"] | |
| if "items" in schema: | |
| result["items"] = _convert_schema_for_gemini(schema["items"]) | |
| return result | |
| def _convert_tools_to_gemini() -> List[Dict]: | |
| """Convert OpenAI tool format to Gemini function declarations.""" | |
| gemini_tools = [] | |
| for tool in TOOLS: | |
| func = tool["function"] | |
| gemini_tools.append({ | |
| "name": func["name"], | |
| "description": func["description"], | |
| "parameters": _convert_schema_for_gemini(func["parameters"]) | |
| }) | |
| return gemini_tools | |
| class GCPChatClient: | |
| """Multi-LLM chat client for GCP - supports OpenAI and Gemini""" | |
| def __init__( | |
| self, | |
| scene_id: str, | |
| base_url: str = "http://localhost:8000", | |
| provider: LLMProvider = "openai" | |
| ): | |
| self.scene_id = scene_id | |
| self.base_url = base_url | |
| self.provider = provider | |
| self.conversation_history: List[Dict[str, Any]] = [] | |
| # Initialize the appropriate client | |
| if provider == "gemini": | |
| if not GEMINI_AVAILABLE: | |
| raise ImportError("google-generativeai not installed. Run: pip install google-generativeai") | |
| genai.configure(api_key=os.getenv("GOOGLE_API_KEY") or os.getenv("GEMINI_API_KEY")) | |
| self.gemini_model = genai.GenerativeModel( | |
| model_name="gemini-2.0-flash", | |
| tools=_convert_tools_to_gemini() | |
| ) | |
| self.client = None | |
| else: | |
| self.client = OpenAI() # Uses OPENAI_API_KEY env var | |
| self.gemini_model = None | |
| # System prompt | |
| self.system_prompt = f"""You are a helpful assistant for GCP (Game Context Protocol), a 3D scene building system. | |
| You help users create and modify 3D scenes using natural language. You have access to tools for: | |
| - Creating scenes and adding/removing objects (cubes, spheres, cylinders, etc.) | |
| - Configuring player movement (speed, jump, gravity, dimensions) | |
| - Managing lights (ambient, directional, point, spot) | |
| - Updating materials (color, metalness, roughness, opacity, glow) | |
| - Setting backgrounds and fog effects | |
| The current scene ID is: {scene_id} | |
| IMPORTANT - Object Placement: | |
| When users ask to create/add an object WITHOUT specifying a position, DO NOT provide x, y, z coordinates. | |
| Simply omit the position parameters entirely and the object will automatically spawn in front of the player | |
| (Minecraft-style placement based on camera direction). | |
| Only provide x, y, z when the user explicitly specifies a location (e.g., "add a cube at 5, 0, 3"). | |
| When users ask to modify something relatively (like "half the speed" or "make it twice as big"), | |
| ALWAYS first get the current state using the appropriate get_* function, then calculate the new value, | |
| then apply it. | |
| Be concise but helpful. After making changes, briefly confirm what was done.""" | |
| def execute_tool(self, name: str, args: Dict[str, Any]) -> Any: | |
| """Execute a GCP tool and return the result""" | |
| # Inject scene_id if not provided | |
| if "scene_id" not in args: | |
| args["scene_id"] = self.scene_id | |
| # Scene tools | |
| if name == "create_scene": | |
| return create_game_scene( | |
| name=args.get("name", "New Scene"), | |
| description=args.get("description"), | |
| world_width=args.get("world_width", 100.0), | |
| world_height=args.get("world_height", 100.0), | |
| world_depth=args.get("world_depth", 100.0), | |
| lighting_preset=args.get("lighting_preset", "day"), | |
| base_url=self.base_url, | |
| ) | |
| elif name == "add_object": | |
| position = create_vector3( | |
| args.get("x", 0), | |
| args.get("y", 0), | |
| args.get("z", 0) | |
| ) | |
| scale = create_vector3( | |
| args.get("scale_x", 1), | |
| args.get("scale_y", 1), | |
| args.get("scale_z", 1) | |
| ) | |
| material = create_material(color=args.get("color", "#ffffff")) | |
| return add_game_object( | |
| scene_id=args["scene_id"], | |
| object_type=args.get("object_type", "cube"), | |
| name=args.get("name"), | |
| position=position, | |
| scale=scale, | |
| material=material, | |
| base_url=self.base_url, | |
| ) | |
| elif name == "remove_object": | |
| return remove_game_object( | |
| scene_id=args["scene_id"], | |
| object_id=args["object_id"], | |
| base_url=self.base_url, | |
| ) | |
| elif name == "set_lighting": | |
| return set_scene_lighting( | |
| scene_id=args["scene_id"], | |
| preset=args.get("preset", "day"), | |
| base_url=self.base_url, | |
| ) | |
| elif name == "get_scene_info": | |
| return get_scene_info(args["scene_id"], self.base_url) | |
| # Player tools | |
| elif name == "set_player_speed": | |
| return set_player_speed(args["scene_id"], args["walk_speed"]) | |
| elif name == "set_jump_force": | |
| return set_jump_force(args["scene_id"], args["jump_force"]) | |
| elif name == "set_mouse_sensitivity": | |
| return set_mouse_sensitivity( | |
| args["scene_id"], | |
| args.get("sensitivity", 0.002), | |
| args.get("invert_y", False) | |
| ) | |
| elif name == "set_gravity": | |
| return set_gravity(args["scene_id"], args["gravity"]) | |
| elif name == "set_player_dimensions": | |
| return set_player_dimensions( | |
| args["scene_id"], | |
| args.get("height", 1.7), | |
| args.get("radius", 0.3), | |
| args.get("eye_height") | |
| ) | |
| elif name == "set_camera_fov": | |
| return set_camera_fov(args["scene_id"], args["fov"]) | |
| elif name == "get_player_config": | |
| return get_player_config(args["scene_id"]) | |
| # Rendering tools | |
| elif name == "add_light": | |
| position = None | |
| if "x" in args or "y" in args or "z" in args: | |
| position = {"x": args.get("x", 0), "y": args.get("y", 5), "z": args.get("z", 0)} | |
| return add_light( | |
| args["scene_id"], | |
| args["light_type"], | |
| args["name"], | |
| args.get("color", "#ffffff"), | |
| args.get("intensity", 1.0), | |
| position, | |
| None, # target | |
| args.get("cast_shadow", False), | |
| None # spot_angle | |
| ) | |
| elif name == "remove_light": | |
| return remove_light(args["scene_id"], args["light_name"]) | |
| elif name == "get_lights": | |
| return get_lights(args["scene_id"]) | |
| elif name == "update_object_material": | |
| return update_object_material( | |
| args["scene_id"], | |
| args["object_id"], | |
| args.get("color"), | |
| args.get("metalness"), | |
| args.get("roughness"), | |
| args.get("opacity"), | |
| args.get("emissive"), | |
| args.get("emissive_intensity") | |
| ) | |
| elif name == "set_background_color": | |
| return set_background_color( | |
| args["scene_id"], | |
| args.get("color"), | |
| args.get("bg_type", "solid"), | |
| args.get("gradient_top"), | |
| args.get("gradient_bottom") | |
| ) | |
| elif name == "set_fog": | |
| return set_fog( | |
| args["scene_id"], | |
| args["enabled"], | |
| args.get("color"), | |
| args.get("near"), | |
| args.get("far"), | |
| args.get("density") | |
| ) | |
| # Environment tools | |
| elif name == "add_skybox": | |
| return add_skybox( | |
| args["scene_id"], | |
| args.get("preset", "day"), | |
| args.get("turbidity", 10.0), | |
| args.get("rayleigh", 2.0), | |
| args.get("sun_elevation", 45.0), | |
| args.get("sun_azimuth", 180.0) | |
| ) | |
| elif name == "remove_skybox": | |
| return remove_skybox(args["scene_id"]) | |
| elif name == "add_particles": | |
| position = None | |
| if "x" in args or "y" in args or "z" in args: | |
| position = {"x": args.get("x", 0), "y": args.get("y", 1), "z": args.get("z", 0)} | |
| return add_particles( | |
| args["scene_id"], | |
| args["preset"], | |
| position, | |
| args.get("particle_id") | |
| ) | |
| elif name == "remove_particles": | |
| return remove_particles(args["scene_id"], args["particle_id"]) | |
| # UI tools | |
| elif name == "render_text_on_screen": | |
| return render_text_on_screen( | |
| args["scene_id"], | |
| args["text"], | |
| args.get("x", 50.0), | |
| args.get("y", 10.0), | |
| args.get("font_size", 24), | |
| args.get("color", "#ffffff"), | |
| args.get("text_id"), | |
| args.get("font_family", "Arial"), | |
| args.get("text_align", "center"), | |
| args.get("background_color"), | |
| args.get("padding", 8) | |
| ) | |
| elif name == "render_bar_on_screen": | |
| return render_bar_on_screen( | |
| args["scene_id"], | |
| args.get("x", 10.0), | |
| args.get("y", 10.0), | |
| args.get("width", 200.0), | |
| args.get("height", 20.0), | |
| args.get("value", 100.0), | |
| args.get("max_value", 100.0), | |
| args.get("bar_color", "#00ff00"), | |
| args.get("background_color", "#333333"), | |
| args.get("border_color", "#ffffff"), | |
| args.get("bar_id"), | |
| args.get("label"), | |
| args.get("show_value", False) | |
| ) | |
| elif name == "remove_ui_element": | |
| return remove_ui_element(args["scene_id"], args["element_id"]) | |
| # Toon material | |
| elif name == "update_material_to_toon": | |
| return update_material_to_toon( | |
| args["scene_id"], | |
| args["object_id"], | |
| args.get("enabled", True), | |
| args.get("color"), | |
| args.get("gradient_steps", 3), | |
| args.get("outline", True), | |
| args.get("outline_color", "#000000"), | |
| args.get("outline_thickness", 0.03) | |
| ) | |
| # Brick blocks | |
| elif name == "add_brick": | |
| position = {"x": args.get("x", 0), "y": args.get("y", 0), "z": args.get("z", 0)} | |
| rotation = {"x": 0, "y": args.get("rotation_y", 0), "z": 0} | |
| return add_brick( | |
| args["scene_id"], | |
| args["brick_type"], | |
| position, | |
| rotation, | |
| args.get("color", "#ff0000"), | |
| args.get("name"), | |
| self.base_url | |
| ) | |
| else: | |
| return {"error": f"Unknown tool: {name}"} | |
| def chat(self, user_message: str) -> tuple[str, Optional[Dict[str, Any]]]: | |
| """ | |
| Process a user message and return the response. | |
| Args: | |
| user_message: The user's message | |
| Returns: | |
| tuple: (response_text, action_data) | |
| - response_text: The assistant's response | |
| - action_data: Optional dict with action info for the frontend | |
| """ | |
| # Add user message to history | |
| self.conversation_history.append({ | |
| "role": "user", | |
| "content": user_message | |
| }) | |
| # Track actions for frontend | |
| actions = [] | |
| # Route to appropriate provider | |
| if self.provider == "gemini": | |
| return self._chat_gemini(user_message, self.system_prompt, actions) | |
| else: | |
| return self._chat_openai(user_message, self.system_prompt, actions) | |
| def _chat_openai(self, user_message: str, system_prompt: str, actions: List) -> tuple[str, Optional[Dict[str, Any]]]: | |
| """Handle chat with OpenAI GPT""" | |
| # Build messages with system prompt | |
| messages = [{"role": "system", "content": system_prompt}] + self.conversation_history | |
| # Call GPT with tools | |
| while True: | |
| response = self.client.chat.completions.create( | |
| model="gpt-4o-mini", # or "gpt-4o" for better reasoning | |
| messages=messages, | |
| tools=TOOLS, | |
| tool_choice="auto" | |
| ) | |
| assistant_message = response.choices[0].message | |
| # Check if there are tool calls | |
| if assistant_message.tool_calls: | |
| # Add assistant message with tool calls to history | |
| messages.append({ | |
| "role": "assistant", | |
| "content": assistant_message.content, | |
| "tool_calls": [ | |
| { | |
| "id": tc.id, | |
| "type": "function", | |
| "function": { | |
| "name": tc.function.name, | |
| "arguments": tc.function.arguments | |
| } | |
| } | |
| for tc in assistant_message.tool_calls | |
| ] | |
| }) | |
| # Execute each tool call | |
| for tool_call in assistant_message.tool_calls: | |
| function_name = tool_call.function.name | |
| function_args = json.loads(tool_call.function.arguments) | |
| # Execute the tool | |
| try: | |
| result = self.execute_tool(function_name, function_args) | |
| actions.append({ | |
| "tool": function_name, | |
| "args": function_args, | |
| "result": result | |
| }) | |
| result_str = json.dumps(result) | |
| except Exception as e: | |
| result_str = json.dumps({"error": str(e)}) | |
| # Add tool result to messages | |
| messages.append({ | |
| "role": "tool", | |
| "tool_call_id": tool_call.id, | |
| "content": result_str | |
| }) | |
| else: | |
| # No more tool calls, we have the final response | |
| final_response = assistant_message.content or "Done!" | |
| # Add to conversation history | |
| self.conversation_history.append({ | |
| "role": "assistant", | |
| "content": final_response | |
| }) | |
| # Build action data for frontend | |
| action_data = None | |
| if actions: | |
| # Return the last significant action for the frontend | |
| last_action = actions[-1] | |
| action_data = self._build_frontend_action(last_action) | |
| return final_response, action_data | |
| def _chat_gemini(self, user_message: str, system_prompt: str, actions: List) -> tuple[str, Optional[Dict[str, Any]]]: | |
| """Handle chat with Google Gemini""" | |
| # Start a chat session with Gemini | |
| chat = self.gemini_model.start_chat(history=[]) | |
| # Combine system prompt with user message for first turn | |
| full_prompt = f"{system_prompt}\n\nUser: {user_message}" | |
| while True: | |
| response = chat.send_message(full_prompt) | |
| # Check for function calls | |
| function_calls = [] | |
| for part in response.parts: | |
| if hasattr(part, 'function_call') and part.function_call: | |
| function_calls.append(part.function_call) | |
| if function_calls: | |
| # Execute each function call | |
| function_responses = [] | |
| for fc in function_calls: | |
| function_name = fc.name | |
| function_args = dict(fc.args) | |
| # Execute the tool | |
| try: | |
| result = self.execute_tool(function_name, function_args) | |
| actions.append({ | |
| "tool": function_name, | |
| "args": function_args, | |
| "result": result | |
| }) | |
| function_responses.append(genai.protos.Part( | |
| function_response=genai.protos.FunctionResponse( | |
| name=function_name, | |
| response={"result": result} | |
| ) | |
| )) | |
| except Exception as e: | |
| function_responses.append(genai.protos.Part( | |
| function_response=genai.protos.FunctionResponse( | |
| name=function_name, | |
| response={"error": str(e)} | |
| ) | |
| )) | |
| # Send function results back to Gemini | |
| full_prompt = function_responses | |
| else: | |
| # No function calls, extract text response | |
| final_response = response.text or "Done!" | |
| # Add to conversation history | |
| self.conversation_history.append({ | |
| "role": "assistant", | |
| "content": final_response | |
| }) | |
| # Build action data for frontend | |
| action_data = None | |
| if actions: | |
| last_action = actions[-1] | |
| action_data = self._build_frontend_action(last_action) | |
| return final_response, action_data | |
| def _build_frontend_action(self, action: Dict[str, Any]) -> Optional[Dict[str, Any]]: | |
| """Convert tool result to frontend action""" | |
| tool = action["tool"] | |
| result = action["result"] | |
| # Map tool names to frontend actions | |
| if tool == "add_object": | |
| from backend.storage import storage | |
| scene = storage.get(self.scene_id) | |
| if scene and scene.get("objects"): | |
| return {"action": "addObject", "data": scene["objects"][-1]} | |
| elif tool == "remove_object": | |
| # Send removeObject action with the object_id that was removed | |
| return {"action": "removeObject", "data": {"object_id": action["args"].get("object_id")}} | |
| elif tool == "set_lighting": | |
| from backend.storage import storage | |
| scene = storage.get(self.scene_id) | |
| if scene and scene.get("lights"): | |
| return {"action": "setLighting", "data": {"lights": scene["lights"]}} | |
| elif tool == "set_player_speed": | |
| return {"action": "setPlayerSpeed", "data": {"walk_speed": result.get("move_speed")}} | |
| elif tool == "set_jump_force": | |
| return {"action": "setJumpForce", "data": {"jump_force": result.get("jump_force")}} | |
| elif tool == "set_gravity": | |
| return {"action": "setGravity", "data": {"gravity": result.get("gravity")}} | |
| elif tool == "set_camera_fov": | |
| return {"action": "setCameraFov", "data": {"fov": result.get("camera_fov")}} | |
| elif tool == "set_mouse_sensitivity": | |
| return {"action": "setMouseSensitivity", "data": { | |
| "sensitivity": result.get("mouse_sensitivity"), | |
| "invert_y": result.get("invert_y") | |
| }} | |
| elif tool == "set_player_dimensions": | |
| return {"action": "setPlayerDimensions", "data": { | |
| "height": result.get("player_height"), | |
| "radius": result.get("player_radius"), | |
| "eye_height": result.get("eye_height") | |
| }} | |
| elif tool == "add_light": | |
| return {"action": "addLight", "data": result.get("light")} | |
| elif tool == "remove_light": | |
| return {"action": "removeLight", "data": {"light_name": action["args"].get("light_name")}} | |
| elif tool == "update_light": | |
| return {"action": "updateLight", "data": { | |
| "light_name": action["args"].get("light_name"), | |
| **result.get("light", {}) | |
| }} | |
| elif tool == "update_object_material": | |
| return {"action": "updateMaterial", "data": result} | |
| elif tool == "set_background_color": | |
| return {"action": "setBackground", "data": result.get("background")} | |
| elif tool == "set_fog": | |
| return {"action": "setFog", "data": result.get("fog")} | |
| # Environment tools | |
| elif tool == "add_skybox": | |
| return {"action": "addSkybox", "data": result.get("skybox")} | |
| elif tool == "remove_skybox": | |
| return {"action": "removeSkybox", "data": {}} | |
| elif tool == "add_particles": | |
| return {"action": "addParticles", "data": result.get("particle_system")} | |
| elif tool == "remove_particles": | |
| return {"action": "removeParticles", "data": {"particle_id": action["args"].get("particle_id")}} | |
| # UI tools | |
| elif tool == "render_text_on_screen": | |
| return {"action": "renderText", "data": result.get("text_element")} | |
| elif tool == "render_bar_on_screen": | |
| return {"action": "renderBar", "data": result.get("bar_element")} | |
| elif tool == "remove_ui_element": | |
| return {"action": "removeUIElement", "data": {"element_id": action["args"].get("element_id")}} | |
| # Toon shading | |
| elif tool == "update_material_to_toon": | |
| return {"action": "updateToonMaterial", "data": { | |
| "object_id": action["args"].get("object_id"), | |
| "enabled": action["args"].get("enabled", True), | |
| "color": action["args"].get("color"), | |
| "gradient_steps": action["args"].get("gradient_steps", 3), | |
| "outline": action["args"].get("outline", True), | |
| "outline_color": action["args"].get("outline_color", "#000000"), | |
| "outline_thickness": action["args"].get("outline_thickness", 0.03) | |
| }} | |
| # Brick blocks | |
| elif tool == "add_brick": | |
| return {"action": "addBrick", "data": result.get("brick")} | |
| return None | |
| def clear_history(self): | |
| """Clear conversation history""" | |
| self.conversation_history = [] | |
| # Convenience function for simple usage | |
| def create_chat_client( | |
| scene_id: str = "welcome", | |
| base_url: str = "http://localhost:8000", | |
| provider: LLMProvider = "openai" | |
| ) -> GCPChatClient: | |
| """Create a new GCP chat client with specified LLM provider""" | |
| return GCPChatClient(scene_id, base_url, provider) | |