Commit ·
b97cf3f
1
Parent(s): 3a269bf
still initial commit
Browse files- app.py +10 -27
- backend/backend.py +6 -2
app.py
CHANGED
|
@@ -61,14 +61,14 @@ def _offline_frame(frame: np.ndarray, message: str) -> np.ndarray:
|
|
| 61 |
return out
|
| 62 |
|
| 63 |
|
| 64 |
-
def _run_voxel_backend(frame: np.ndarray
|
| 65 |
-
"""Encodes and ships raw image bytes
|
| 66 |
success, encoded = cv2.imencode(".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), 85])
|
| 67 |
if not success:
|
| 68 |
return frame
|
| 69 |
try:
|
| 70 |
-
#
|
| 71 |
-
processed_bytes = voxel_backend.remote(encoded.tobytes()
|
| 72 |
result = cv2.imdecode(np.frombuffer(processed_bytes, dtype=np.uint8), cv2.IMREAD_COLOR)
|
| 73 |
return result if result is not None else frame
|
| 74 |
except Exception as err:
|
|
@@ -81,9 +81,9 @@ def _run_voxel_backend(frame: np.ndarray, prompt: str, strength: float) -> np.nd
|
|
| 81 |
# ── Core Stream Handler ─────────────────────────────────────────────────────
|
| 82 |
frame_counter = 0
|
| 83 |
|
| 84 |
-
def process_video_stream(frame: np.ndarray, mode: str, is_running: bool
|
| 85 |
"""
|
| 86 |
-
Accepts incoming frame from the webcam, pipeline settings,
|
| 87 |
"""
|
| 88 |
global frame_counter
|
| 89 |
if frame is None:
|
|
@@ -104,8 +104,8 @@ def process_video_stream(frame: np.ndarray, mode: str, is_running: bool, prompt:
|
|
| 104 |
if frame_counter % 15 == 0:
|
| 105 |
print(f"🚀 [LIVE PIPELINE] Transmitting frames. Dispatched {frame_counter} payloads to Modal.")
|
| 106 |
|
| 107 |
-
# Process via the single-image pipeline
|
| 108 |
-
processed = _run_voxel_backend(frame
|
| 109 |
|
| 110 |
# Mode A: Full view rendering
|
| 111 |
if mode == "Minecraft Filter":
|
|
@@ -154,23 +154,6 @@ with gr.Blocks(title="⛏️ Minecraft Spatial Voxel Filter") as demo:
|
|
| 154 |
label="🎯 Pipeline Mode",
|
| 155 |
interactive=True,
|
| 156 |
)
|
| 157 |
-
|
| 158 |
-
# Added controls to dynamically configure the FLUX diffusion backend
|
| 159 |
-
prompt_input = gr.Textbox(
|
| 160 |
-
value="isometric 3d minecraft block voxel style, high resolution, volumetric lighting",
|
| 161 |
-
label="✨ Generation Prompt",
|
| 162 |
-
lines=2,
|
| 163 |
-
interactive=True
|
| 164 |
-
)
|
| 165 |
-
|
| 166 |
-
strength_slider = gr.Slider(
|
| 167 |
-
minimum=0.1,
|
| 168 |
-
maximum=1.0,
|
| 169 |
-
value=0.45,
|
| 170 |
-
step=0.05,
|
| 171 |
-
label="🎛️ Image Transformation Strength",
|
| 172 |
-
interactive=True
|
| 173 |
-
)
|
| 174 |
|
| 175 |
with gr.Row():
|
| 176 |
start_btn = gr.Button("🚀 Start Processing", variant="primary")
|
|
@@ -185,10 +168,10 @@ with gr.Blocks(title="⛏️ Minecraft Spatial Voxel Filter") as demo:
|
|
| 185 |
start_btn.click(fn=lambda: True, inputs=None, outputs=is_running)
|
| 186 |
stop_btn.click(fn=lambda: False, inputs=None, outputs=is_running)
|
| 187 |
|
| 188 |
-
# Core engine transmission loop linked up with the
|
| 189 |
input_stream.stream(
|
| 190 |
fn=process_video_stream,
|
| 191 |
-
inputs=[input_stream, mode_dropdown, is_running
|
| 192 |
outputs=[output_stream],
|
| 193 |
trigger_mode="always_last", # Drops intermediate backlog frames when backend is busy
|
| 194 |
concurrency_limit=1 # Ensures only one frame flies over the network at a time
|
|
|
|
| 61 |
return out
|
| 62 |
|
| 63 |
|
| 64 |
+
def _run_voxel_backend(frame: np.ndarray) -> np.ndarray:
|
| 65 |
+
"""Encodes and ships ONLY raw image bytes to the Modal worker."""
|
| 66 |
success, encoded = cv2.imencode(".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), 85])
|
| 67 |
if not success:
|
| 68 |
return frame
|
| 69 |
try:
|
| 70 |
+
# Pushing ONLY the camera feed byte data across the network
|
| 71 |
+
processed_bytes = voxel_backend.remote(encoded.tobytes())
|
| 72 |
result = cv2.imdecode(np.frombuffer(processed_bytes, dtype=np.uint8), cv2.IMREAD_COLOR)
|
| 73 |
return result if result is not None else frame
|
| 74 |
except Exception as err:
|
|
|
|
| 81 |
# ── Core Stream Handler ─────────────────────────────────────────────────────
|
| 82 |
frame_counter = 0
|
| 83 |
|
| 84 |
+
def process_video_stream(frame: np.ndarray, mode: str, is_running: bool) -> np.ndarray:
|
| 85 |
"""
|
| 86 |
+
Accepts incoming frame from the webcam, pipeline settings, and execution state.
|
| 87 |
"""
|
| 88 |
global frame_counter
|
| 89 |
if frame is None:
|
|
|
|
| 104 |
if frame_counter % 15 == 0:
|
| 105 |
print(f"🚀 [LIVE PIPELINE] Transmitting frames. Dispatched {frame_counter} payloads to Modal.")
|
| 106 |
|
| 107 |
+
# Process via the single-image pipeline
|
| 108 |
+
processed = _run_voxel_backend(frame)
|
| 109 |
|
| 110 |
# Mode A: Full view rendering
|
| 111 |
if mode == "Minecraft Filter":
|
|
|
|
| 154 |
label="🎯 Pipeline Mode",
|
| 155 |
interactive=True,
|
| 156 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 157 |
|
| 158 |
with gr.Row():
|
| 159 |
start_btn = gr.Button("🚀 Start Processing", variant="primary")
|
|
|
|
| 168 |
start_btn.click(fn=lambda: True, inputs=None, outputs=is_running)
|
| 169 |
stop_btn.click(fn=lambda: False, inputs=None, outputs=is_running)
|
| 170 |
|
| 171 |
+
# Core engine transmission loop linked up with ONLY the stream, mode, and run-state
|
| 172 |
input_stream.stream(
|
| 173 |
fn=process_video_stream,
|
| 174 |
+
inputs=[input_stream, mode_dropdown, is_running],
|
| 175 |
outputs=[output_stream],
|
| 176 |
trigger_mode="always_last", # Drops intermediate backlog frames when backend is busy
|
| 177 |
concurrency_limit=1 # Ensures only one frame flies over the network at a time
|
backend/backend.py
CHANGED
|
@@ -30,7 +30,7 @@ app = modal.App("flux-klein-voxel-backend", image=image)
|
|
| 30 |
# 🏎️ 1. THE DEMO PIPELINE (FALLBACK ROUTE)
|
| 31 |
# ==============================================================================
|
| 32 |
@app.function()
|
| 33 |
-
def demo_stream_frame(img_bytes: bytes
|
| 34 |
"""Fallback route structurally aligned to match the frontend signature."""
|
| 35 |
from PIL import Image, ImageDraw
|
| 36 |
|
|
@@ -102,10 +102,14 @@ class VoxelModel:
|
|
| 102 |
print("Warmup complete. Ready for real-time requests!")
|
| 103 |
|
| 104 |
@modal.method()
|
| 105 |
-
def process_frame(self, img_bytes: bytes
|
| 106 |
from PIL import Image
|
| 107 |
import torch
|
| 108 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
input_image = Image.open(io.BytesIO(img_bytes)).convert("RGB")
|
| 110 |
input_image = input_image.resize((512, 512))
|
| 111 |
|
|
|
|
| 30 |
# 🏎️ 1. THE DEMO PIPELINE (FALLBACK ROUTE)
|
| 31 |
# ==============================================================================
|
| 32 |
@app.function()
|
| 33 |
+
def demo_stream_frame(img_bytes: bytes) -> bytes:
|
| 34 |
"""Fallback route structurally aligned to match the frontend signature."""
|
| 35 |
from PIL import Image, ImageDraw
|
| 36 |
|
|
|
|
| 102 |
print("Warmup complete. Ready for real-time requests!")
|
| 103 |
|
| 104 |
@modal.method()
|
| 105 |
+
def process_frame(self, img_bytes: bytes) -> bytes:
|
| 106 |
from PIL import Image
|
| 107 |
import torch
|
| 108 |
|
| 109 |
+
# Hardcoded parameters that were previously passed from the UI
|
| 110 |
+
prompt = "isometric 3d minecraft block voxel style, high resolution, volumetric lighting"
|
| 111 |
+
strength = 0.45
|
| 112 |
+
|
| 113 |
input_image = Image.open(io.BytesIO(img_bytes)).convert("RGB")
|
| 114 |
input_image = input_image.resize((512, 512))
|
| 115 |
|