AnimeOverlord commited on
Commit
b97cf3f
·
1 Parent(s): 3a269bf

still initial commit

Browse files
Files changed (2) hide show
  1. app.py +10 -27
  2. backend/backend.py +6 -2
app.py CHANGED
@@ -61,14 +61,14 @@ def _offline_frame(frame: np.ndarray, message: str) -> np.ndarray:
61
  return out
62
 
63
 
64
- def _run_voxel_backend(frame: np.ndarray, prompt: str, strength: float) -> np.ndarray:
65
- """Encodes and ships raw image bytes along with UI parameters to the Modal worker."""
66
  success, encoded = cv2.imencode(".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), 85])
67
  if not success:
68
  return frame
69
  try:
70
- # Match the 3 required arguments expected by the VoxelModel class on Modal
71
- processed_bytes = voxel_backend.remote(encoded.tobytes(), prompt, strength)
72
  result = cv2.imdecode(np.frombuffer(processed_bytes, dtype=np.uint8), cv2.IMREAD_COLOR)
73
  return result if result is not None else frame
74
  except Exception as err:
@@ -81,9 +81,9 @@ def _run_voxel_backend(frame: np.ndarray, prompt: str, strength: float) -> np.nd
81
  # ── Core Stream Handler ─────────────────────────────────────────────────────
82
  frame_counter = 0
83
 
84
- def process_video_stream(frame: np.ndarray, mode: str, is_running: bool, prompt: str, strength: float) -> np.ndarray:
85
  """
86
- Accepts incoming frame from the webcam, pipeline settings, execution state, and prompt config.
87
  """
88
  global frame_counter
89
  if frame is None:
@@ -104,8 +104,8 @@ def process_video_stream(frame: np.ndarray, mode: str, is_running: bool, prompt:
104
  if frame_counter % 15 == 0:
105
  print(f"🚀 [LIVE PIPELINE] Transmitting frames. Dispatched {frame_counter} payloads to Modal.")
106
 
107
- # Process via the single-image pipeline, passing down prompt strings and context values
108
- processed = _run_voxel_backend(frame, prompt, strength)
109
 
110
  # Mode A: Full view rendering
111
  if mode == "Minecraft Filter":
@@ -154,23 +154,6 @@ with gr.Blocks(title="⛏️ Minecraft Spatial Voxel Filter") as demo:
154
  label="🎯 Pipeline Mode",
155
  interactive=True,
156
  )
157
-
158
- # Added controls to dynamically configure the FLUX diffusion backend
159
- prompt_input = gr.Textbox(
160
- value="isometric 3d minecraft block voxel style, high resolution, volumetric lighting",
161
- label="✨ Generation Prompt",
162
- lines=2,
163
- interactive=True
164
- )
165
-
166
- strength_slider = gr.Slider(
167
- minimum=0.1,
168
- maximum=1.0,
169
- value=0.45,
170
- step=0.05,
171
- label="🎛️ Image Transformation Strength",
172
- interactive=True
173
- )
174
 
175
  with gr.Row():
176
  start_btn = gr.Button("🚀 Start Processing", variant="primary")
@@ -185,10 +168,10 @@ with gr.Blocks(title="⛏️ Minecraft Spatial Voxel Filter") as demo:
185
  start_btn.click(fn=lambda: True, inputs=None, outputs=is_running)
186
  stop_btn.click(fn=lambda: False, inputs=None, outputs=is_running)
187
 
188
- # Core engine transmission loop linked up with the new UI input arguments
189
  input_stream.stream(
190
  fn=process_video_stream,
191
- inputs=[input_stream, mode_dropdown, is_running, prompt_input, strength_slider],
192
  outputs=[output_stream],
193
  trigger_mode="always_last", # Drops intermediate backlog frames when backend is busy
194
  concurrency_limit=1 # Ensures only one frame flies over the network at a time
 
61
  return out
62
 
63
 
64
+ def _run_voxel_backend(frame: np.ndarray) -> np.ndarray:
65
+ """Encodes and ships ONLY raw image bytes to the Modal worker."""
66
  success, encoded = cv2.imencode(".jpg", frame, [int(cv2.IMWRITE_JPEG_QUALITY), 85])
67
  if not success:
68
  return frame
69
  try:
70
+ # Pushing ONLY the camera feed byte data across the network
71
+ processed_bytes = voxel_backend.remote(encoded.tobytes())
72
  result = cv2.imdecode(np.frombuffer(processed_bytes, dtype=np.uint8), cv2.IMREAD_COLOR)
73
  return result if result is not None else frame
74
  except Exception as err:
 
81
  # ── Core Stream Handler ─────────────────────────────────────────────────────
82
  frame_counter = 0
83
 
84
+ def process_video_stream(frame: np.ndarray, mode: str, is_running: bool) -> np.ndarray:
85
  """
86
+ Accepts incoming frame from the webcam, pipeline settings, and execution state.
87
  """
88
  global frame_counter
89
  if frame is None:
 
104
  if frame_counter % 15 == 0:
105
  print(f"🚀 [LIVE PIPELINE] Transmitting frames. Dispatched {frame_counter} payloads to Modal.")
106
 
107
+ # Process via the single-image pipeline
108
+ processed = _run_voxel_backend(frame)
109
 
110
  # Mode A: Full view rendering
111
  if mode == "Minecraft Filter":
 
154
  label="🎯 Pipeline Mode",
155
  interactive=True,
156
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
157
 
158
  with gr.Row():
159
  start_btn = gr.Button("🚀 Start Processing", variant="primary")
 
168
  start_btn.click(fn=lambda: True, inputs=None, outputs=is_running)
169
  stop_btn.click(fn=lambda: False, inputs=None, outputs=is_running)
170
 
171
+ # Core engine transmission loop linked up with ONLY the stream, mode, and run-state
172
  input_stream.stream(
173
  fn=process_video_stream,
174
+ inputs=[input_stream, mode_dropdown, is_running],
175
  outputs=[output_stream],
176
  trigger_mode="always_last", # Drops intermediate backlog frames when backend is busy
177
  concurrency_limit=1 # Ensures only one frame flies over the network at a time
backend/backend.py CHANGED
@@ -30,7 +30,7 @@ app = modal.App("flux-klein-voxel-backend", image=image)
30
  # 🏎️ 1. THE DEMO PIPELINE (FALLBACK ROUTE)
31
  # ==============================================================================
32
  @app.function()
33
- def demo_stream_frame(img_bytes: bytes, prompt: str, strength: float) -> bytes:
34
  """Fallback route structurally aligned to match the frontend signature."""
35
  from PIL import Image, ImageDraw
36
 
@@ -102,10 +102,14 @@ class VoxelModel:
102
  print("Warmup complete. Ready for real-time requests!")
103
 
104
  @modal.method()
105
- def process_frame(self, img_bytes: bytes, prompt: str, strength: float) -> bytes:
106
  from PIL import Image
107
  import torch
108
 
 
 
 
 
109
  input_image = Image.open(io.BytesIO(img_bytes)).convert("RGB")
110
  input_image = input_image.resize((512, 512))
111
 
 
30
  # 🏎️ 1. THE DEMO PIPELINE (FALLBACK ROUTE)
31
  # ==============================================================================
32
  @app.function()
33
+ def demo_stream_frame(img_bytes: bytes) -> bytes:
34
  """Fallback route structurally aligned to match the frontend signature."""
35
  from PIL import Image, ImageDraw
36
 
 
102
  print("Warmup complete. Ready for real-time requests!")
103
 
104
  @modal.method()
105
+ def process_frame(self, img_bytes: bytes) -> bytes:
106
  from PIL import Image
107
  import torch
108
 
109
+ # Hardcoded parameters that were previously passed from the UI
110
+ prompt = "isometric 3d minecraft block voxel style, high resolution, volumetric lighting"
111
+ strength = 0.45
112
+
113
  input_image = Image.open(io.BytesIO(img_bytes)).convert("RGB")
114
  input_image = input_image.resize((512, 512))
115