still initial commit

app.py

@@ -5,192 +5,198 @@ import gradio as gr
 import modal
 from fastrtc import WebRTC, get_cloudflare_turn_credentials
 
-# Initialize Modal connection securely on startup with clear logging
+# ── Modal Backend Connection ────────────────────────────────────────────────
 try:
-    print("🚀 Attempting to connect to Modal VoxelModel class...")
+    print("🚀 Connecting to Modal VoxelModel...")
     VoxelModelCls = modal.Cls.from_name("flux-klein-voxel-backend", "VoxelModel")
     voxel_backend = VoxelModelCls().process_frame
+    print("✅ Modal Cls connected.")
 except Exception as e:
-    print(f"⚠️ Modal Cls lookup failed: {e}. Falling back to Function lookup.")
+    print(f"⚠️ Modal Cls failed: {e}. Trying Function fallback...")
     try:
         voxel_backend = modal.Function.from_name("flux-klein-voxel-backend", "demo_stream_frame")
+        print("✅ Modal Function connected.")
     except Exception as ex:
-        print(f"⚠️ Modal Function lookup also failed: {ex}. Backend will be offline.")
+        print(f"❌ Modal backend offline: {ex}")
         voxel_backend = None
 
 
-# --- STREAM HANDLING FUNCTIONS ---
-
-def process_video_stream(frame: np.ndarray, prompt: str, strength: float) -> np.ndarray:
+# ── Shared Core Processing ──────────────────────────────────────────────────
+def _run_voxel_backend(frame: np.ndarray, prompt: str, strength: float) -> np.ndarray:
     """
-    Handles single viewport mode (Minecraft Filter).
-    Receives frames via WebRTC, processes them, and returns them to the same viewport.
+    Encodes frame → sends to Modal → decodes result.
+    Returns processed frame, or annotated original on failure.
     """
-    if frame is None:
-        return None
-
-    if voxel_backend is None:
-        output_frame = frame.copy()
-        cv2.putText(output_frame, "ERROR: Modal Backend Offline", (20, 40), 
-                    cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 255), 2)
-        return output_frame
-
-    success, encoded_image = cv2.imencode(".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), 85])
+    success, encoded = cv2.imencode(".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), 85])
     if not success:
         return frame
-    
-    frame_bytes = encoded_image.tobytes()
 
     try:
         try:
-            processed_bytes = voxel_backend.remote(frame_bytes, prompt, strength)
+            processed_bytes = voxel_backend.remote(encoded.tobytes(), prompt, strength)
         except TypeError:
-            processed_bytes = voxel_backend.remote(frame_bytes)
-        
-        numpy_buffer = np.frombuffer(processed_bytes, dtype=np.uint8)
-        return cv2.imdecode(numpy_buffer, cv2.IMREAD_COLOR)
+            # Backend may not accept prompt/strength yet
+            processed_bytes = voxel_backend.remote(encoded.tobytes())
+
+        result = cv2.imdecode(np.frombuffer(processed_bytes, dtype=np.uint8), cv2.IMREAD_COLOR)
+        # Guard against decode failure
+        return result if result is not None else frame
+
     except Exception as err:
-        fallback_frame = frame.copy()
-        cv2.putText(fallback_frame, "Serverless Processing...", (20, 40), 
-                    cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 255), 2)
-        return fallback_frame
+        out = frame.copy()
+        cv2.putText(out, f"Modal error: {str(err)[:45]}", (10, 30),
+                    cv2.FONT_HERSHEY_SIMPLEX, 0.55, (0, 0, 255), 1)
+        return out
 
 
-def process_demo_stream(frame: np.ndarray) -> np.ndarray:
+def _offline_frame(frame: np.ndarray, message: str) -> np.ndarray:
+    """Returns an error-annotated frame matching the input dimensions."""
+    # FIX BUG 3: match input frame shape instead of hardcoded (480, 640)
+    h, w = frame.shape[:2] if frame is not None else (480, 640)
+    out = np.zeros((h, w, 3), dtype=np.uint8)
+    cv2.putText(out, message, (max(10, w // 8), h // 2),
+                cv2.FONT_HERSHEY_DUPLEX, 0.8, (0, 0, 220), 2)
+    return out
+
+
+# ── Stream Handlers ─────────────────────────────────────────────────────────
+def process_video_stream(frame: np.ndarray, prompt: str, strength: float) -> np.ndarray:
     """
-    Handles separate multi-viewport tracking (Streaming Demo).
-    Ingests raw hardware frames and renders processed outputs to a separate container.
+    Minecraft Filter mode: transforms frame with user-controlled prompt + strength.
+    Returns single processed frame → goes back to the same send-receive WebRTC.
     """
     if frame is None:
         return None
-
     if voxel_backend is None:
-        error_frame = np.zeros((480, 640, 3), dtype=np.uint8)
-        error_frame[:] = 30
-        cv2.putText(error_frame, "BACKEND APP OFFLINE", (40, 200), 
-                    cv2.FONT_HERSHEY_DUPLEX, 0.9, (0, 0, 255), 2)
-        return error_frame
+        return _offline_frame(frame, "Modal Backend Offline")
 
-    success, encoded_image = cv2.imencode(".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), 85])
-    if not success:
-        return frame
-    
-    frame_bytes = encoded_image.tobytes()
+    return _run_voxel_backend(frame, prompt, strength)
 
-    try:
-        try:
-            processed_bytes = voxel_backend.remote(frame_bytes, "vanilla minecraft voxel landscape", 0.55)
-        except TypeError:
-            processed_bytes = voxel_backend.remote(frame_bytes)
-        
-        numpy_buffer = np.frombuffer(processed_bytes, dtype=np.uint8)
-        return cv2.imdecode(numpy_buffer, cv2.IMREAD_COLOR)
 
-    except Exception as err:
-        error_frame = np.zeros((480, 640, 3), dtype=np.uint8)
-        error_frame[:] = 20
-        cv2.putText(error_frame, "MODAL RUNTIME EXCEPTION", (40, 180), 
-                    cv2.FONT_HERSHEY_DUPLEX, 0.8, (0, 140, 255), 2)
-        cv2.putText(error_frame, f"Error: {str(err)[:40]}...", (40, 240), 
-                    cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1)
-        return error_frame
+def process_demo_stream(frame: np.ndarray) -> np.ndarray:
+    """
+    Streaming Demo mode: fixed prompt, side-by-side raw + processed.
 
+    FIX BUG 1+2: Instead of routing to a separate mode='receive' WebRTC
+    (which FastRTC doesn't support via stream()), we combine both feeds
+    into one frame and return it through the same send-receive component.
+    """
+    if frame is None:
+        return None
+    if voxel_backend is None:
+        err = _offline_frame(frame, "Backend Offline")
+        # Still return side-by-side so layout stays consistent
+        return np.hstack([frame, err])
+
+    processed = _run_voxel_backend(
+        frame,
+        prompt="vanilla minecraft voxel landscape, 3d blocky style, cube aesthetic",
+        strength=0.55,
+    )
+
+    # Resize processed to match raw height if they differ (rare but safe)
+    if processed.shape[0] != frame.shape[0]:
+        processed = cv2.resize(processed, (frame.shape[1], frame.shape[0]))
 
-# --- PURE NATIVE GRADIO UI LAYOUT ---
+    # Label both panels
+    raw_labeled = frame.copy()
+    cv2.putText(raw_labeled, "RAW", (10, 28),
+                cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 255, 255), 2)
+    cv2.putText(processed, "MINECRAFT", (10, 28),
+                cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 255, 255), 2)
 
+    # Side-by-side in a single frame returned to one WebRTC component
+    return np.hstack([raw_labeled, processed])
+
+
+# ── UI ──────────────────────────────────────────────────────────────────────
 with gr.Blocks(title="Minecraft Spatial Voxel Filter") as demo:
-    
-    # Header Section
+
     gr.Markdown("<h1 style='text-align: center;'>⛏️ MINECRAFT SPATIAL VOXEL FILTER ⛏️</h1>")
-    gr.Markdown("<p style='text-align: center;'>Transform your physical environment into an interactive, real-time 3D blocky landscape.</p>")
-    
-    # Global Controls
+    gr.Markdown("<p style='text-align: center;'>Transform your environment into a real-time blocky landscape.</p>")
+
     mode_dropdown = gr.Dropdown(
         choices=["Minecraft Filter", "Streaming Demo"],
         value="Minecraft Filter",
         label="🎯 Pipeline View Configuration",
-        interactive=True
+        interactive=True,
     )
-    
-    # Main Application Area (Row keeps things side-by-side cleanly)
+
     with gr.Row():
-        
-        # Left Column: Settings
+
+        # ── Left: Controls ─────────────────────────────────────────────────
         with gr.Column(scale=1):
             gr.Markdown("### 🎛️ Environmental Filters")
-            
+
             prompt_input = gr.Textbox(
                 value="vanilla minecraft voxel landscape, 3d blocky style, retro game cube aesthetic",
-                label="Biome Environment Blueprint (Prompt)",
-                lines=3
+                label="Biome Environment Blueprint",
+                lines=3,
             )
-            
             denoise_strength = gr.Slider(
                 minimum=0.1, maximum=1.0, step=0.05, value=0.55,
-                label="Voxelization Denoising Strength"
+                label="Voxelization Denoising Strength",
             )
-            
-            status_msg = "Connected" if voxel_backend is not None else "Offline / Error"
-            gr.Markdown(f"**Pipeline Status:** Deployed via FastRTC. Modal Backend: `{status_msg}`.")
-        
-        # Right Column: Video Viewports
+
+            status_color = "🟢" if voxel_backend is not None else "🔴"
+            status_text  = "Connected" if voxel_backend is not None else "Offline"
+            gr.Markdown(f"**Modal Backend:** {status_color} `{status_text}`")
+            gr.Markdown("_Controls only apply in Minecraft Filter mode._")
+
+        # ── Right: Viewports ───────────────────────────────────────────────
         with gr.Column(scale=2):
-            
-            # Variant 1: Minecraft Filter
+
+            # Mode 1: Minecraft Filter — single send-receive, processed output
             with gr.Group(visible=True) as minecraft_layout:
                 gr.Markdown("### 📺 Spatial Render Pipeline")
-                # The WebRTC component natively provides Start/Stop buttons in its UI
                 webrtc_single = WebRTC(
                     label="Live Voxel Viewport",
                     modality="video",
                     mode="send-receive",
-                    rtc_configuration=get_cloudflare_turn_credentials
+                    rtc_configuration=get_cloudflare_turn_credentials,
                 )
-            
-            # Variant 2: Streaming Demo
+
+            # Mode 2: Streaming Demo
+            # FIX BUG 1+2: single send-receive WebRTC, side-by-side frame
+            # returned internally — no separate mode="receive" component needed
             with gr.Group(visible=False) as demo_layout:
                 gr.Markdown("### 📺 Dual-Feed Stream Monitor")
-                with gr.Row():
-                    webrtc_raw = WebRTC(
-                        label="Raw Local Camera",
-                        modality="video",
-                        mode="send",
-                        rtc_configuration=get_cloudflare_turn_credentials
-                    )
-                    webrtc_processed = WebRTC(
-                        label="Processed Backend Output",
-                        modality="video",
-                        mode="receive",
-                        rtc_configuration=get_cloudflare_turn_credentials
-                    )
-
-    # --- ROUTING LOGIC & EVENT WIRES ---
-    
+                gr.Markdown("_Left: Raw camera · Right: Minecraft output_")
+                webrtc_demo = WebRTC(
+                    label="Live Side-by-Side Feed",
+                    modality="video",
+                    mode="send-receive",
+                    rtc_configuration=get_cloudflare_turn_credentials,
+                )
+
+    # ── Mode Switch ─────────────────────────────────────────────────────────
     def switch_layout(selected_mode):
-        if selected_mode == "Minecraft Filter":
-            return gr.update(visible=True), gr.update(visible=False)
-        else:
-            return gr.update(visible=False), gr.update(visible=True)
+        return (
+            gr.update(visible=selected_mode == "Minecraft Filter"),
+            gr.update(visible=selected_mode == "Streaming Demo"),
+        )
 
     mode_dropdown.change(
         fn=switch_layout,
         inputs=[mode_dropdown],
-        outputs=[minecraft_layout, demo_layout]
+        outputs=[minecraft_layout, demo_layout],
     )
 
-    # Stream bindings
+    # ── Stream Bindings ─────────────────────────────────────────────────────
     webrtc_single.stream(
         fn=process_video_stream,
         inputs=[webrtc_single, prompt_input, denoise_strength],
         outputs=[webrtc_single],
-        time_limit=150
+        time_limit=150,
+        concurrency_limit=4,
     )
 
-    webrtc_raw.stream(
+    webrtc_demo.stream(
         fn=process_demo_stream,
-        inputs=[webrtc_raw],
-        outputs=[webrtc_processed],
-        time_limit=150
+        inputs=[webrtc_demo],
+        outputs=[webrtc_demo],          # ← same component, not a separate one
+        time_limit=150,
+        concurrency_limit=4,
     )
 
 if __name__ == "__main__":

backend/backend.py

@@ -2,7 +2,7 @@ import io
 import os
 import modal
 
-# Define container environment optimized for lightning-fast image-to-image processing
+# Define container environment
 image = modal.Image.debian_slim(python_version="3.12").pip_install(
     "diffusers",
     "transformers",
@@ -14,86 +14,65 @@ image = modal.Image.debian_slim(python_version="3.12").pip_install(
 app = modal.App("flux-klein-voxel-backend", image=image)
 
 # ==============================================================================
-# 🏎️ 1. THE DEMO PIPELINE (Zero Cold-Start / Instant WebRTC Echo Test)
+# 🏎️ 1. THE DEMO PIPELINE
 # ==============================================================================
 @app.function()
 def demo_stream_frame(img_bytes: bytes) -> bytes:
-    """
-    Decodes the incoming WebRTC frame and returns it instantly.
-    Does not spin up a GPU or load a model. Use this to verify that the 
-    frontend WebRTC connection is 100% functional.
-    """
     from PIL import Image, ImageDraw
     
-    # Unpack the binary stream sent by FastRTC
     input_image = Image.open(io.BytesIO(img_bytes)).convert("RGB")
-    
-    # Optional visual overlay so you know the demo bypass is active
     draw = ImageDraw.Draw(input_image)
     draw.text((20, 20), "🛠️ WEBRTC PASSTHROUGH DEMO ACTIVE", fill=(0, 255, 0))
-    draw.text((20, 40), "Inference model bypassed.", fill=(255, 255, 255))
     
-    # Pack back into high-speed compressed JPEG format
     output_buffer = io.BytesIO()
     input_image.save(output_buffer, format="JPEG", quality=85)
     return output_buffer.getvalue()
 
 
 # ==============================================================================
-# 🚀 2. THE REAL-TIME VOXEL ENGINE (GPU-Accelerated Inference)
+# 🚀 2. THE REAL-TIME VOXEL ENGINE
 # ==============================================================================
 @app.cls(
     gpu="A10G",
-    secrets=[modal.Secret.from_name("huggingface")],
-    concurrency_limit=10  # Scales automatically up to 10 parallel video streams
+    # Using the secret name from your example: "huggingface-secret"
+    secrets=[modal.Secret.from_name("huggingface-secret")],
+    max_containers=5
 )
 class VoxelModel:
     
     @modal.enter()
     def load_pipeline(self):
-        """Pre-loads model checkpoints into serverless VRAM exactly once upon container initialization"""
         import torch
-        from diffusers import DiffusionPipeline
+        from diffusers import AutoPipelineForImage2Image
         
-        # Target your specific fine-tuned space or the base black-forest-labs/FLUX.2-klein-4B
         model_id = "AnimeOverlord/flux2-klein-4b-mc"
-        print(f"📦 Spin up sequence initiated. Pulling weights for {model_id}...")
         
-        # DiffusionPipeline dynamically handles custom fine-tune repo definitions via model_index.json
-        self.pipe = DiffusionPipeline.from_pretrained(
+        # Following your provided pattern for auth
+        self.pipe = AutoPipelineForImage2Image.from_pretrained(
             model_id,
             torch_dtype=torch.bfloat16,
-            token=os.environ["HF_TOKEN"]
+            use_auth_token=os.environ["HF_TOKEN"] 
         )
         self.pipe.to("cuda")
-        
-        # Performance Tweaks for low-latency video loops
         self.pipe.enable_attention_slicing()
-        print("⚡ Core weights successfully loaded into cloud VRAM.")
 
-    @modal.function()
+    @modal.method()
     def process_frame(self, img_bytes: bytes, prompt: str, strength: float) -> bytes:
-        """Executes targeted frame transformations without saving overhead data to memory"""
         from PIL import Image
         import torch
         
-        # 1. Unpack compressed binary frame directly from network interface
         input_image = Image.open(io.BytesIO(img_bytes)).convert("RGB")
-        
-        # Hard clamping constraint resolution guarantees predictable frame-rates
         input_image = input_image.resize((512, 512)) 
         
-        # 2. Process frame via low-step inference execution 
         with torch.inference_mode():
             output_image = self.pipe(
                 prompt=prompt,
                 image=input_image,
                 strength=strength,
-                num_inference_steps=4, # Hard locked to match FLUX.2 Klein's step-distilled architecture
+                num_inference_steps=4,
                 guidance_scale=3.5,
             ).images[0]
             
-        # 3. Re-pack processing output back to JPEG format for transit
         output_buffer = io.BytesIO()
         output_image.save(output_buffer, format="JPEG", quality=85)
         return output_buffer.getvalue()