initial commit less goo

app.py

@@ -0,0 +1,135 @@
+import os
+import cv2
+import numpy as np
+import gradio as gr
+import modal
+from fastrtc import WebRTC, get_cloudflare_turn_credentials
+
+# Environment control flag to swap pipelines from the HF Space configuration dashboard
+USE_GPU_INFERENCE = os.getenv("USE_GPU_INFERENCE", "false").lower() == "true"
+
+# Connect to your deployed serverless backend running on Modal
+try:
+    if USE_GPU_INFERENCE:
+        print("🚀 Mode: Full GPU FLUX.2 Klein Inference")
+        voxel_backend = modal.Function.lookup("flux-klein-voxel-backend", "VoxelModel.process_frame")
+    else:
+        print("🏎️ Mode: Zero-latency WebRTC Passthrough Demo")
+        voxel_backend = modal.Function.lookup("flux-klein-voxel-backend", "demo_stream_frame")
+except Exception as e:
+    print(f"⚠️ Could not bind Modal backend function layout: {e}")
+    voxel_backend = None
+
+def process_video_stream(frame: np.ndarray, prompt: str, strength: float) -> np.ndarray:
+    """
+    Receives real-time video frames from the browser via WebRTC, compresses them, 
+    ships them to the Modal GPU cluster, and returns the voxelized matrix.
+    """
+    if frame is None:
+        return None
+
+    # Fallback state if the backend app isn't active or authenticated yet
+    if voxel_backend is None:
+        output_frame = frame.copy()
+        cv2.putText(output_frame, "ERROR: Backend App Offline", (20, 40), 
+                    cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 255), 2)
+        cv2.putText(output_frame, "Check HF Space Secrets for MODAL keys.", (20, 70), 
+                    cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1)
+        return output_frame
+
+    # Step 1: Compress high-res frames to a lean JPEG byte stream to prevent browser pipe congestion
+    success, encoded_image = cv2.imencode(".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), 85])
+    if not success:
+        return frame
+    
+    frame_bytes = encoded_image.tobytes()
+
+    # Step 2: Route request to serverless GPU infrastructure
+    try:
+        # Dynamically matches positional signature parameters to prevent signature TypeErrors
+        if USE_GPU_INFERENCE:
+            processed_bytes = voxel_backend.remote(frame_bytes, prompt, strength)
+        else:
+            processed_bytes = voxel_backend.remote(frame_bytes)
+        
+        # Step 3: Reconstruction of the returned processed image array
+        numpy_buffer = np.frombuffer(processed_bytes, dtype=np.uint8)
+        voxel_frame = cv2.imdecode(numpy_buffer, cv2.IMREAD_COLOR)
+        return voxel_frame
+
+    except Exception as err:
+        # Handle serverless cold starts visually instead of freezing or crashing the stream
+        fallback_frame = frame.copy()
+        cv2.putText(fallback_frame, "⚡ Modal Serverless Warm-up (15-30s)...", (20, 40), 
+                    cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 255), 2)
+        cv2.putText(fallback_frame, "Loading FLUX Klein weights into cloud VRAM", (20, 70), 
+                    cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1)
+        return fallback_frame
+
+# --- CUSTOM CSS FOR HIGH-TECH SPATIAL AESTHETICS ---
+custom_css = """
+#container { max-width: 1100px; margin: 0 auto; padding-top: 20px; }
+.header-text { text-align: center; margin-bottom: 25px; }
+.header-text h1 { color: #5c8e32; font-family: 'Courier New', Courier, monospace; font-weight: bold; margin-bottom: 5px; }
+.header-text p { color: #666; font-size: 1.1em; }
+"""
+
+# --- GRADIO INTERFACE ARCHITECTURE ---
+with gr.Blocks(css=custom_css, title="Minecraft Spatial Voxel Filter") as demo:
+    
+    with gr.Div(elem_id="container"):
+        with gr.Div(elem_classes="header-text"):
+            gr.Markdown("# ⛏️ MINECRAFT SPATIAL VOXEL FILTER ⛏️")
+            gr.Markdown("Transform your physical environment into an interactive, real-time 3D blocky landscape running on FLUX.2 Klein.")
+        
+        gr.HTML("<hr style='border: 1px solid #ddd; margin-bottom: 25px;'>")
+        
+        with gr.Row():
+            # Left Hand Side: Dynamic Parameters & Controls
+            with gr.Column(scale=1):
+                gr.Markdown("### 🎛️ Environmental Filters")
+                
+                prompt_input = gr.Textbox(
+                    value="vanilla minecraft voxel landscape, 3d blocky style, retro game cube aesthetic, highly detailed texture pack",
+                    label="Biome Environment Blueprint (Prompt)",
+                    lines=3,
+                    placeholder="Describe your voxel theme..."
+                )
+                
+                with gr.Accordion("Advanced Tuning", open=True):
+                    denoise_strength = gr.Slider(
+                        minimum=0.1,
+                        maximum=1.0,
+                        step=0.05,
+                        value=0.55,
+                        label="Voxelization Denoising Strength"
+                    )
+                
+                gr.Markdown(
+                    f"""
+                    > **💡 Pipeline Status:** Deployed via FastRTC. Current Target Mode: `{"GPU Inference (FLUX)" if USE_GPU_INFERENCE else "CPU Passthrough Demo"}`. Change this via the `USE_GPU_INFERENCE` Space Secret.
+                    """
+                )
+            
+            # Right Hand Side: High-Speed WebRTC Viewport
+            with gr.Column(scale=2):
+                gr.Markdown("### 📺 Spatial Render Pipeline")
+                
+                # FastRTC custom WebRTC component with auto-configured cloudflare turn discovery
+                webrtc_stream = WebRTC(
+                    label="Live Voxel Viewport",
+                    modality="video",
+                    mode="send-receive",
+                    rtc_configuration=get_cloudflare_turn_credentials
+                )
+
+    # Establish the bidirectional stream wire link
+    webrtc_stream.stream(
+        fn=process_video_stream,
+        inputs=[webrtc_stream, prompt_input, denoise_strength],
+        outputs=[webrtc_stream],
+        time_limit=150 # Automatically closes connection after inactivity to prevent runaway token spend
+    )
+
+if __name__ == "__main__":
+    demo.launch()

backend/backend.py

@@ -0,0 +1,99 @@
+import io
+import os
+import modal
+
+# Define container environment optimized for lightning-fast image-to-image processing
+image = modal.Image.debian_slim(python_version="3.12").pip_install(
+    "diffusers",
+    "transformers",
+    "accelerate",
+    "pillow",
+    "torch"
+)
+
+app = modal.App("flux-klein-voxel-backend", image=image)
+
+# ==============================================================================
+# 🏎️ 1. THE DEMO PIPELINE (Zero Cold-Start / Instant WebRTC Echo Test)
+# ==============================================================================
+@app.function()
+def demo_stream_frame(img_bytes: bytes) -> bytes:
+    """
+    Decodes the incoming WebRTC frame and returns it instantly.
+    Does not spin up a GPU or load a model. Use this to verify that the 
+    frontend WebRTC connection is 100% functional.
+    """
+    from PIL import Image, ImageDraw
+    
+    # Unpack the binary stream sent by FastRTC
+    input_image = Image.open(io.BytesIO(img_bytes)).convert("RGB")
+    
+    # Optional visual overlay so you know the demo bypass is active
+    draw = ImageDraw.Draw(input_image)
+    draw.text((20, 20), "🛠️ WEBRTC PASSTHROUGH DEMO ACTIVE", fill=(0, 255, 0))
+    draw.text((20, 40), "Inference model bypassed.", fill=(255, 255, 255))
+    
+    # Pack back into high-speed compressed JPEG format
+    output_buffer = io.BytesIO()
+    input_image.save(output_buffer, format="JPEG", quality=85)
+    return output_buffer.getvalue()
+
+
+# ==============================================================================
+# 🚀 2. THE REAL-TIME VOXEL ENGINE (GPU-Accelerated Inference)
+# ==============================================================================
+@app.cls(
+    gpu="A10G",
+    secrets=[modal.Secret.from_name("huggingface")],
+    concurrency_limit=10  # Scales automatically up to 10 parallel video streams
+)
+class VoxelModel:
+    
+    @modal.enter()
+    def load_pipeline(self):
+        """Pre-loads model checkpoints into serverless VRAM exactly once upon container initialization"""
+        import torch
+        from diffusers import DiffusionPipeline
+        
+        # Target your specific fine-tuned space or the base black-forest-labs/FLUX.2-klein-4B
+        model_id = "AnimeOverlord/flux2-klein-4b-mc"
+        print(f"📦 Spin up sequence initiated. Pulling weights for {model_id}...")
+        
+        # DiffusionPipeline dynamically handles custom fine-tune repo definitions via model_index.json
+        self.pipe = DiffusionPipeline.from_pretrained(
+            model_id,
+            torch_dtype=torch.bfloat16,
+            token=os.environ["HF_TOKEN"]
+        )
+        self.pipe.to("cuda")
+        
+        # Performance Tweaks for low-latency video loops
+        self.pipe.enable_attention_slicing()
+        print("⚡ Core weights successfully loaded into cloud VRAM.")
+
+    @modal.function()
+    def process_frame(self, img_bytes: bytes, prompt: str, strength: float) -> bytes:
+        """Executes targeted frame transformations without saving overhead data to memory"""
+        from PIL import Image
+        import torch
+        
+        # 1. Unpack compressed binary frame directly from network interface
+        input_image = Image.open(io.BytesIO(img_bytes)).convert("RGB")
+        
+        # Hard clamping constraint resolution guarantees predictable frame-rates
+        input_image = input_image.resize((512, 512)) 
+        
+        # 2. Process frame via low-step inference execution 
+        with torch.inference_mode():
+            output_image = self.pipe(
+                prompt=prompt,
+                image=input_image,
+                strength=strength,
+                num_inference_steps=4, # Hard locked to match FLUX.2 Klein's step-distilled architecture
+                guidance_scale=3.5,
+            ).images[0]
+            
+        # 3. Re-pack processing output back to JPEG format for transit
+        output_buffer = io.BytesIO()
+        output_image.save(output_buffer, format="JPEG", quality=85)
+        return output_buffer.getvalue()

requirements.txt

@@ -0,0 +1,6 @@
+gradio>=5.7.1
+fastrtc>=0.0.34
+opencv-python-headless
+modal
+pillow
+numpy