""" GCP - Game Context Protocol Build 3D game scenes with natural language using AI. Architecture: - Gradio: Chat interface and static file serving - FastAPI: Scene API and MCP tools (local dev only) - Three.js: 3D rendering in embedded iframe - OpenAI: Natural language processing with function calling """ import os import json import time import gradio as gr # Static file serving for 3D models (GLB files) # Accessible at /gradio_api/file=models/ gr.set_static_paths(paths=["models"]) # ============================================================================= # Environment Configuration # ============================================================================= IS_HF_SPACES = bool(os.getenv("SPACE_ID")) FASTAPI_INTERNAL = "http://localhost:8000" FASTAPI_URL = "" # Relative URLs for HF Spaces if IS_HF_SPACES: print(f"๐ŸŒ HF Spaces: {os.getenv('SPACE_ID')}") else: # Local dev: FastAPI on 8000, Gradio on 7860 FASTAPI_URL = os.getenv("FASTAPI_URL", "http://localhost:8000") FASTAPI_INTERNAL = FASTAPI_URL print(f"๐Ÿ’ป Local dev: {FASTAPI_URL}") # Global state current_scene_id = None current_provider = "openai" # "openai" or "gemini" def _get_toast_message(action_type: str, data: dict) -> str: """Generate user-friendly toast message for an action.""" # Simple static messages STATIC_MESSAGES = { "removeObject": "Object removed", "setLighting": "Lighting updated", "updateMaterial": "Material updated", "removeLight": "Light removed", "updateLight": "Light updated", "setBackground": "Background updated", "setFog": "Fog updated", "removeSkybox": "Skybox removed", "removeParticles": "Particles removed", "removeUIElement": "UI element removed", } if action_type in STATIC_MESSAGES: return STATIC_MESSAGES[action_type] # Dynamic messages with data if action_type == "addObject": return f"Added {data.get('type', 'object')}" if action_type == "setControlMode": return f"Switched to {data.get('mode', '').upper()} mode" if action_type == "addLight": return f"Added light: {data.get('name', 'Light')}" if action_type == "setPlayerSpeed": return f"Speed: {data.get('walk_speed', 5)} m/s" if action_type == "setJumpForce": return f"Jump: {data.get('jump_force', 5)} m/s" if action_type == "setGravity": return f"Gravity: {data.get('gravity', -9.82)} m/sยฒ" if action_type == "setCameraFov": return f"FOV: {data.get('fov', 75)}ยฐ" if action_type == "setMouseSensitivity": return f"Sensitivity: {data.get('sensitivity', 0.002)}" if action_type == "setPlayerDimensions": return f"Height: {data.get('height', 1.7)}m" if action_type == "addSkybox": return f"Skybox: {data.get('preset', 'custom')}" if action_type == "addParticles": return f"Particles: {data.get('preset', 'effect')}" if action_type == "renderText": return f"Text: {data.get('text', '')[:15]}..." if action_type == "renderBar": return f"Bar: {data.get('label', 'Bar')}" if action_type == "updateToonMaterial": return "Toon " + ("on" if data.get("enabled", True) else "off") if action_type == "addBrick": return f"Added {data.get('brick_type', 'brick').replace('_', ' ')}" return action_type # Fallback to action name # ============================================================================= # FastAPI Server (Local Dev Only) # ============================================================================= if not IS_HF_SPACES: import threading import uvicorn import requests from backend.main import app as fastapi_app def _wait_for_fastapi(): """Wait for FastAPI health check.""" print("โณ Waiting for FastAPI...") for _ in range(30): try: if requests.get(f"{FASTAPI_INTERNAL}/health", timeout=2).ok: print("โœ… FastAPI ready") return except: time.sleep(1) print("โš ๏ธ FastAPI timeout") threading.Thread( target=lambda: uvicorn.run(fastapi_app, host="0.0.0.0", port=8000, log_level="warning"), daemon=True ).start() _wait_for_fastapi() def get_viewer_html(scene_id="welcome"): """Generate viewer HTML with embedded scene data.""" from backend.storage import storage scene_data = storage.get(scene_id) if not scene_data: return '
Scene not found
' # Add the static assets base URL for model loading # On local dev: FastAPI serves at /static/models/ # On HF Spaces: Gradio serves via set_static_paths at /gradio_api/file=models/ if IS_HF_SPACES: scene_data["static_base_url"] = "/gradio_api/file=models/" else: scene_data["static_base_url"] = f"{FASTAPI_INTERNAL}/static/models/" # Read the viewer HTML template viewer_path = os.path.join(os.path.dirname(__file__), "frontend", "game_viewer.html") with open(viewer_path, 'r') as f: html = f.read() # Inject scene data before the closing tag scene_json = json.dumps(scene_data) inject_script = f'' html = html.replace('', f'{inject_script}') return html def create_default_scene(): """Use the clean default Welcome Scene created on server startup""" global current_scene_id try: current_scene_id = "welcome" print(f"โœ… Using default Welcome Scene (ID: {current_scene_id})") return get_viewer_html(current_scene_id) except Exception as e: import traceback print(f"โŒ Error loading default scene: {e}") traceback.print_exc() return None # Initialize the chat client chat_client = None def get_chat_client(provider: str = "openai"): """Get or create the chat client with specified provider""" global chat_client, current_scene_id, current_provider # Recreate client if scene or provider changed if chat_client is None or chat_client.scene_id != current_scene_id or chat_client.provider != provider: from chat_client import GCPChatClient chat_client = GCPChatClient(scene_id=current_scene_id, base_url=FASTAPI_URL, provider=provider) current_provider = provider return chat_client def chat_response(message, history, provider="openai"): """Handle chat messages using LLM with tool calling""" global current_scene_id # Handle help command locally (no need for LLM) if message.lower().strip() == "help": return """**GCP - Game Context Protocol** I'm an AI assistant that can help you build 3D scenes using natural language. **What I can do:** - Add objects: "add a red cube" (spawns in front of you, Minecraft-style!) - Add at position: "add a blue sphere at 2, 1, 0" - Change lighting: "set lighting to night" - Configure player: "set speed to 10" or "make the player move half as fast" - Add lights: "add a point light above the cube" - Update materials: "make it shiny and metallic" - Set backgrounds: "gradient background from blue to orange" - Add fog: "add some fog to the scene" - Query state: "what's the current player speed?" or "show me the scene info" **I understand context**, so you can say things like: - "double the jump force" - "make it twice as bright" - "reduce gravity by half" **Tips:** - Press C in viewer to toggle FPS/Orbit camera - WASD to move, Space to jump in FPS mode - Click in viewer to enable mouse-look - Objects spawn in front of your camera when no position is specified! **LLM Provider:** Currently using {provider.upper()} """.format(provider=provider), None try: client = get_chat_client(provider) response, action_data = client.chat(message) return response, action_data except Exception as e: import traceback traceback.print_exc() return f"Error: {str(e)}", None # Create default scene on startup print("Creating default scene...") default_viewer_html = create_default_scene() print(f"Default viewer HTML loaded: {len(default_viewer_html) if default_viewer_html else 0} bytes") if not default_viewer_html: print("โš ๏ธ WARNING: Default scene creation failed! No viewer URL generated.") # Minimal CSS - only essential styling, let Gradio handle layout APP_CSS = """ /* Viewer iframe needs explicit sizing */ #viewer-container { width: 100%; height: 600px; } #viewer-container iframe { width: 100%; height: 100%; border: none; } /* Toast Notifications */ #toast-container { position: fixed; top: 20px; right: 20px; z-index: 200; } .toast { background: rgba(0, 0, 0, 0.9); color: white; padding: 15px 20px; border-radius: 8px; margin-bottom: 10px; border-left: 4px solid #2196f3; } .toast.success { border-left-color: #4caf50; } .toast.error { border-left-color: #f44336; } /* LLM Toggle Buttons */ #llm-toggle-row { margin-top: 8px; gap: 0 !important; } .llm-btn { border-radius: 0 !important; border: 1px solid #444 !important; background: #2a2a2a !important; color: #888 !important; font-size: 13px !important; padding: 8px 16px !important; min-height: 36px !important; transition: all 0.2s ease !important; } .llm-btn:first-child { border-radius: 6px 0 0 6px !important; border-right: none !important; } .llm-btn:last-child { border-radius: 0 6px 6px 0 !important; } .llm-btn:hover { background: #3a3a3a !important; color: #aaa !important; } .llm-btn-active { background: #4a6fa5 !important; color: #fff !important; border-color: #5a7fb5 !important; } .llm-btn-active:hover { background: #5a7fb5 !important; color: #fff !important; } /* Hidden action textbox - must be in DOM but invisible for .change() to fire */ .hidden-action { position: absolute !important; width: 1px !important; height: 1px !important; padding: 0 !important; margin: -1px !important; overflow: hidden !important; clip: rect(0, 0, 0, 0) !important; white-space: nowrap !important; border: 0 !important; opacity: 0 !important; pointer-events: none !important; } """ # Build immersive chat interface with overlay with gr.Blocks(title="GCP - Game Context Protocol") as demo: # Inject custom CSS via HTML style tag (Gradio 6 doesn't support css= parameter) gr.HTML(f"") # Initialize JavaScript functionality (minimal essentials only) gr.HTML(""" """) # State for selected LLM provider provider_state = gr.State("openai") # Main container - side by side layout: Chat (left) | Viewer (right) with gr.Row(elem_id="main-container", equal_height=True): # Left column: Chat interface (scale=1 = ~25% width) with gr.Column(elem_id="chat-column", scale=1, min_width=350): gr.Markdown("### ๐ŸŽฎ GCP - Game Context Protocol") # Initial welcome message initial_message = [ { "role": "assistant", "content": "Describe what you want to build. For example: \"create a blue cube\", \"create fire\", \"show a text that says: MAIN WORLD\"." } ] chatbot = gr.Chatbot( value=initial_message, height=480, show_label=False, elem_id="chatbot", ) msg = gr.Textbox( placeholder="Add your instructions here", show_label=False, container=False, elem_id="chat-input", ) # LLM Provider toggle buttons with gr.Row(elem_id="llm-toggle-row"): btn_openai = gr.Button("OpenAI", elem_id="btn-openai", elem_classes=["llm-btn", "llm-btn-active"], scale=1) btn_gemini = gr.Button("Gemini", elem_id="btn-gemini", elem_classes=["llm-btn"], scale=1) # Right column: 3D Viewer (scale=3 = ~75% width) with gr.Column(elem_id="viewer-column", scale=3): if default_viewer_html: initial_viewer_html = f'
' print(f"๐Ÿ“Š Setting up viewer with embedded scene data") else: initial_viewer_html = '

โš ๏ธ Scene failed to load. Check console logs.

' print("โš ๏ธ No viewer HTML available - showing error message") viewer = gr.HTML( value=initial_viewer_html, elem_id="viewer-fullscreen" ) # About section - project description and credits with gr.Row(elem_id="about-panel"): gr.Markdown(""" ### About This Project This is my proposal for creating the **GCP โ€“ Game Context Protocol**. An MCP server that allows LLMs to interact in the game creation process, communicating between tools, engines, developers, artists, and everyone involved in making games. A big challenge in game development, as in other creative professions, is the "blank page problem." This is when we start from scratch, have lots of ideas, but don't know where to begin. Should I learn to code? Should I learn to make art? Which engine should I use? At a larger scale, game development requires thousands of hours of human labor, much of which can be tedious. In those cases, it would be valuable to have a toolโ€”in this case, an MCP toolโ€”that allows humans and LLMs to express their goals and guide the creative process, rather than having to do everything by hand. """) gr.HTML(""" """) gr.Markdown(""" ### X post: [https://x.com/arturonereu/status/1994998558806687810](https://x.com/arturonereu/status/1994998558806687810) **Built by:** [Arturo Nereu](https://arturonereu.com/) and [Claude Code](https://claude.ai/code) **Technologies:** [Three.js](https://threejs.org/), [Gradio](https://gradio.app/), [Hugging Face](https://huggingface.co/), [Python](https://www.python.org/), [FastAPI](https://fastapi.tiangolo.com/), [Google Gemini](https://deepmind.google/technologies/gemini/), and [OpenAI GPT](https://openai.com/) **Assets by:** [Kenney](https://kenney.nl/) and [Norod78](https://sketchfab.com/Norod) """) # Tools panel - full width below chat and viewer with gr.Row(elem_id="tools-panel"): gr.Markdown(""" ### GCP Available Tools (40+)
**๐ŸŽฌ Scene Tools** - `add_object` - Add primitives (cube, sphere, cylinder, cone, torus, plane) - `add_brick` - Add LEGO bricks (1x1, 2x4, slopes, plates) - `remove_object` - Remove an object - `set_lighting` - Presets: day, night, sunset, studio - `get_scene_info` - Get scene details
**๐ŸŒ… Environment** - `add_skybox` - Add sky (day, sunset, night, dawn) - `remove_skybox` - Remove skybox - `add_particles` - Effects: fire, smoke, rain, snow, sparkle - `remove_particles` - Remove particles
**๐ŸŽฎ Player Tools** - `set_player_speed` - Movement speed - `set_jump_force` - Jump strength - `set_mouse_sensitivity` - Look sensitivity - `set_gravity` - World gravity - `set_camera_fov` - Field of view - `get_player_config` - Get settings
**๐Ÿ’ก Lighting** - `add_light` - Add light (ambient, directional, point, spot) - `remove_light` - Remove light - `update_light` - Modify light properties - `get_lights` - List all lights
**๐ŸŽจ Materials** - `update_object_material` - Color, metalness, roughness, opacity - `update_material_to_toon` - Apply cel-shading - `set_background_color` - Solid or gradient - `set_fog` - Atmospheric fog
**๐Ÿ“Š UI Overlay** - `render_text_on_screen` - 2D text at any position - `render_bar_on_screen` - Health/progress bars - `remove_ui_element` - Remove UI element - `get_ui_elements` - List UI elements
""") def user(user_message, history): """Handle user input""" history = history or [] history.append({"role": "user", "content": user_message}) return "", history def bot(history, provider): """Generate bot response using selected LLM provider""" # Gradio 6: content can be a string or list of content blocks content = history[-1]["content"] if isinstance(content, list): # Extract text from content blocks user_message = " ".join( block.get("text", "") if isinstance(block, dict) else str(block) for block in content ) else: user_message = content # Process command with selected provider # Objects will spawn in front of camera automatically (Minecraft-style) bot_message, action_result = chat_response(user_message, [], provider) # Prefix response with provider name prefix = "GPT>" if provider == "openai" else "Gemini>" formatted_message = f"**{prefix}** {bot_message}" history.append({"role": "assistant", "content": formatted_message}) # Handle action_result action_json = "" # Default: no action (empty string) if action_result: action_type = action_result.get("action") if action_type != "reload": # Generate toast message for the action toast_message = _get_toast_message(action_type, action_result.get("data", {})) # Send action to viewer via postMessage (handled by JS in .then()) action_json = json.dumps({ "action": action_result["action"], "data": action_result["data"], "toast": toast_message, "toastType": "success", "timestamp": time.time() }) return history, action_json # Toggle button click handlers def select_openai(current_provider): return "openai" def select_gemini(current_provider): return "gemini" btn_openai.click( fn=select_openai, inputs=[provider_state], outputs=[provider_state], js=""" (provider) => { document.getElementById('btn-openai').classList.add('llm-btn-active'); document.getElementById('btn-gemini').classList.remove('llm-btn-active'); return provider; } """ ) btn_gemini.click( fn=select_gemini, inputs=[provider_state], outputs=[provider_state], js=""" (provider) => { document.getElementById('btn-gemini').classList.add('llm-btn-active'); document.getElementById('btn-openai').classList.remove('llm-btn-active'); return provider; } """ ) # Hidden textbox to trigger JS via .change() event # NOTE: Must be visible=True with CSS hiding, as visible=False prevents .change() from firing action_data = gr.Textbox(value="", elem_id="action-data", visible=True, elem_classes=["hidden-action"]) msg.submit( user, [msg, chatbot], [msg, chatbot], queue=False ).then( bot, [chatbot, provider_state], [chatbot, action_data], ).then( fn=lambda x: x, inputs=[action_data], outputs=[action_data], js=""" (actionJson) => { // Send postMessage to iframe if there's action data if (actionJson && actionJson.length > 2) { try { const actionData = JSON.parse(actionJson); const iframe = document.querySelector('#viewer-container iframe'); if (iframe && iframe.contentWindow) { iframe.contentWindow.postMessage({ action: actionData.action, data: actionData.data }, '*'); if (actionData.toast && window.showToast) { window.showToast(actionData.toast, actionData.toastType || 'success'); } } else { console.error('โŒ iframe not found'); } } catch (e) { console.error('Failed to parse action:', e); } } return actionJson; } """ ) if __name__ == "__main__": demo.queue() demo.launch(server_name="0.0.0.0", server_port=7860, ssr_mode=False)