John2J commited on
Commit
66d970f
·
verified ·
1 Parent(s): b09f0cf

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +602 -300
app.py CHANGED
@@ -1,25 +1,41 @@
1
  import os
2
  import sys
3
- import subprocess
4
  import tempfile
5
- import shutil
6
  from pathlib import Path
7
 
8
- # ---------------------------------------------------------
 
 
 
 
9
  # Configuration
10
- # ---------------------------------------------------------
11
 
12
  REPO_DIR = Path("/tmp/EraserDiT")
 
13
 
14
- # Enable faster Hugging Face downloads when available
15
  os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1")
 
 
 
 
 
 
 
 
 
 
 
 
 
16
 
17
- # ---------------------------------------------------------
18
- # Clone EraserDiT source
19
- # ---------------------------------------------------------
20
 
21
- if not REPO_DIR.exists():
22
  print("Cloning EraserDiT...")
 
23
  subprocess.run(
24
  [
25
  "git",
@@ -32,35 +48,23 @@ if not REPO_DIR.exists():
32
  check=True,
33
  )
34
 
35
- sys.path.insert(0, str(REPO_DIR))
36
 
37
- # ---------------------------------------------------------
38
- # Imports from EraserDiT
39
- # ---------------------------------------------------------
40
 
41
- import torch
42
- import gradio as gr
43
- import spaces
44
 
45
- from utils.common import GlobalValues
46
- from utils.pre import VideoInpaintPre
47
- from utils.inference_utils import init, inference_batch
48
- from utils.post import post_stream_normalized
49
- from utils.post_pkg import FFmpegWriter
50
 
51
- import decord
52
- import ffmpeg
53
- import datetime
54
- import math
55
 
 
56
 
57
- # ---------------------------------------------------------
58
- # EraserDiT configuration
59
- # ---------------------------------------------------------
60
-
61
- GlobalValues.DEBUG = False
62
 
63
- MODEL_ID = "jieeliu/EraserDiT"
 
 
64
 
65
  NEGATIVE_PROMPT = (
66
  "Colorful color tone, overexposure, static, blurry details, "
@@ -72,17 +76,10 @@ NEGATIVE_PROMPT = (
72
  "the background, walking backwards, no noise"
73
  )
74
 
75
- DEVICE = torch.device("cuda")
76
-
77
- # Original model uses bfloat16
78
- WEIGHT_DTYPE = torch.bfloat16
79
-
80
- pipeline = None
81
-
82
 
83
- # ---------------------------------------------------------
84
- # Load model once
85
- # ---------------------------------------------------------
86
 
87
  def load_model():
88
  global pipeline
@@ -90,129 +87,311 @@ def load_model():
90
  if pipeline is not None:
91
  return pipeline
92
 
 
 
93
  if not torch.cuda.is_available():
94
  raise RuntimeError("CUDA GPU is required.")
95
 
96
- print("========================================")
 
97
  print("Loading EraserDiT")
98
- print("GPU:", torch.cuda.get_device_name(0))
99
- print("VRAM:",
100
- round(torch.cuda.get_device_properties(0).total_memory / 1024**3, 2),
101
- "GB")
102
- print("========================================")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
 
104
  pipeline = init(
105
- DEVICE,
106
- WEIGHT_DTYPE,
107
  pre_dir=MODEL_ID,
108
  )
109
 
 
 
110
  return pipeline
111
 
112
 
113
- # ---------------------------------------------------------
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114
  # Main inference
115
- # ---------------------------------------------------------
116
 
117
  @spaces.GPU(duration=1200)
118
- def process_video(video_path, mask_path, prompt, progress=gr.Progress()):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
119
  if video_path is None:
120
- raise gr.Error("Please upload an input video.")
 
 
121
 
122
  if mask_path is None:
123
- raise gr.Error("Please upload a mask video.")
 
 
124
 
125
  if not prompt or not prompt.strip():
126
- raise gr.Error("Please provide a description/prompt for the video.")
 
 
127
 
128
  video_path = str(video_path)
129
  mask_path = str(mask_path)
130
 
131
- # Make sure files exist
132
  if not os.path.isfile(video_path):
133
- raise gr.Error(f"Input video does not exist: {video_path}")
 
 
134
 
135
  if not os.path.isfile(mask_path):
136
- raise gr.Error(f"Mask video does not exist: {mask_path}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
137
 
138
  print()
139
- print("========================================")
140
- print("EraserDiT inference")
141
- print("Video:", video_path)
142
- print("Mask :", mask_path)
143
- print("Prompt:", prompt)
144
- print("========================================")
145
 
146
- # -----------------------------------------------------
147
- # Validate video
148
- # -----------------------------------------------------
 
149
 
150
- try:
151
- video_info = ffmpeg.probe(video_path)
152
- mask_info = ffmpeg.probe(mask_path)
 
153
 
154
- video_stream = next(
155
- s for s in video_info["streams"]
156
- if s.get("codec_type") == "video"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
157
  )
158
 
159
- mask_stream = next(
160
- s for s in mask_info["streams"]
161
- if s.get("codec_type") == "video"
 
 
162
  )
163
 
164
- width = int(video_stream["width"])
165
- height = int(video_stream["height"])
 
166
 
167
- mask_width = int(mask_stream["width"])
168
- mask_height = int(mask_stream["height"])
169
-
170
- if width != mask_width or height != mask_height:
171
- raise gr.Error(
172
- f"Video and mask resolution must match.\n"
173
- f"Video: {width}x{height}\n"
174
- f"Mask: {mask_width}x{mask_height}"
175
- )
176
 
177
- # EraserDiT's normal path handles resolutions up to
178
- # approximately 1920x1088 without bbox cropping.
179
- if width * height > 1920 * 1088:
180
- raise gr.Error(
181
- "This Space currently expects video resolution up to "
182
- "1920x1088. Larger videos require EraserDiT's bbox "
183
- "cropping workflow."
184
- )
185
 
186
- print(f"Resolution: {width}x{height}")
 
187
 
188
- except gr.Error:
189
- raise
190
 
191
  except Exception as e:
192
- raise gr.Error(f"Could not inspect video: {e}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
193
 
194
- # -----------------------------------------------------
195
  # Load model
196
- # -----------------------------------------------------
197
 
198
  pipe = load_model()
199
 
200
- # -----------------------------------------------------
201
- # Temporary output directory
202
- # -----------------------------------------------------
203
 
204
- output_dir = Path(tempfile.mkdtemp(prefix="eraserdit_"))
 
 
 
205
 
206
- video_name = Path(video_path).stem
 
 
207
 
208
- output_path = output_dir / f"{video_name}_eraserdit.mp4"
209
 
210
- # -----------------------------------------------------
211
- # Preprocessor
212
- # -----------------------------------------------------
213
 
214
  preprocessor = VideoInpaintPre(
215
- device=DEVICE,
216
  align_h=32,
217
  align_w=32,
218
  ksize=(9, 9),
@@ -222,62 +401,68 @@ def process_video(video_path, mask_path, prompt, progress=gr.Progress()):
222
  crop_flag=False,
223
  )
224
 
225
- # -----------------------------------------------------
226
- # Video information
227
- # -----------------------------------------------------
228
 
229
- org_video_info = ffmpeg.probe(video_path)
 
 
230
 
231
- video_stream = next(
232
- s for s in org_video_info["streams"]
233
- if s.get("codec_type") == "video"
 
234
  )
235
 
236
- fps_string = video_stream.get("r_frame_rate", "30/1")
237
 
238
- try:
239
- fps_num, fps_den = map(int, fps_string.split("/"))
240
- fps = fps_num / fps_den
241
- except Exception:
242
- fps = 30.0
243
 
244
- bitrate = video_stream.get("bit_rate")
 
 
 
245
 
246
- if bitrate:
247
- try:
248
- bitrate_m = max(1, int(bitrate) // 1_000_000)
249
- except Exception:
250
- bitrate_m = 10
251
- else:
252
- bitrate_m = 10
253
 
254
- # -----------------------------------------------------
255
- # Processing
256
- # -----------------------------------------------------
257
 
258
- video_save_writer = None
259
  pre_video_shift = None
260
 
 
 
261
  current_batch = 0
262
 
263
- # The model processes 121-frame temporal chunks
264
- # with 9-frame overlap.
265
- total_frames = None
 
 
 
 
 
 
266
 
267
  try:
268
- probe_reader = decord.VideoReader(
269
- video_path,
270
- ctx=decord.cpu(0),
271
- )
272
- total_frames = len(probe_reader)
273
- del probe_reader
274
- except Exception:
275
- pass
276
 
277
- while True:
278
 
279
- video_ori, mask_ori, fps_loaded, _, _ = (
280
- preprocessor.load_videos(
 
 
 
 
 
 
 
 
 
281
  video_path=video_path,
282
  mask_path=mask_path,
283
  bbox_path=None,
@@ -285,190 +470,291 @@ def process_video(video_path, mask_path, prompt, progress=gr.Progress()):
285
  sample_rate=1,
286
  batch_idx=current_batch,
287
  )
288
- )
289
 
290
- if video_ori is None:
291
- break
 
292
 
293
- video_input, mask_input, _ = preprocessor(
294
- video_ori,
295
- mask_ori,
296
- batch_idx=current_batch,
297
- format="nhwc",
298
- )
299
 
300
- input_shape = preprocessor.TranslateShape(
301
- video_input.shape,
302
- src="nchw",
303
- dst="nhwc",
304
- )
 
 
 
 
 
 
 
 
 
305
 
306
- # -------------------------------------------------
307
- # Initialize writer
308
- # -------------------------------------------------
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
309
 
310
- if video_save_writer is None:
 
 
 
311
 
312
- video_save_writer = FFmpegWriter(
313
- path=str(output_path),
314
- width=video_ori.shape[2],
315
- height=video_ori.shape[1],
316
- fps=fps_loaded,
317
- bitrate=f"{bitrate_m}M",
 
 
 
 
 
 
 
318
  )
319
 
320
- # -------------------------------------------------
321
- # Temporal overlap
322
- # -------------------------------------------------
323
 
324
- if current_batch == 0:
 
 
 
 
325
 
326
- masks_zero_shift = torch.zeros(
327
- (
328
- math.ceil(preprocessor.shift_alpha / 8),
329
- mask_input.shape[1],
330
- mask_input.shape[2],
331
- mask_input.shape[3],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
332
  ),
333
- dtype=mask_input.dtype,
334
  )
335
 
336
- else:
 
 
337
 
338
- video_input = torch.cat(
339
- [
340
- pre_video_shift,
341
- video_input,
342
- ]
 
343
  )
344
 
345
- mask_input = torch.cat(
346
- [
347
- masks_zero_shift,
348
- mask_input,
349
- ]
350
  )
351
 
352
- print(
353
- f"Processing batch {current_batch} "
354
- f"({video_ori.shape[0]} source frames)"
355
- )
356
 
357
- # -------------------------------------------------
358
- # Model inference
359
- # -------------------------------------------------
360
-
361
- output_frames = inference_batch(
362
- videos=video_input,
363
- masks_input=mask_input,
364
- prompt=prompt,
365
- negative_prompt=NEGATIVE_PROMPT,
366
- pipeline=pipe,
367
- generator=None,
368
- device=DEVICE,
369
- weight_dtype=WEIGHT_DTYPE,
370
- )
371
 
372
- # Save overlap frames for next batch
373
- pre_video_shift = (
374
- output_frames[
375
- -preprocessor.shift_alpha:
376
- ].cpu()
377
- )
378
 
379
- # -------------------------------------------------
380
- # Write output
381
- # -------------------------------------------------
382
-
383
- if current_batch == 0:
384
-
385
- post_stream_normalized(
386
- output_frames=output_frames,
387
- ori_shape=video_ori.shape,
388
- model_video_shape=input_shape,
389
- writer=video_save_writer,
390
- crop_flag=False,
391
- videos_input_ori=None,
392
- video_ori=video_ori,
393
- mask_ori=mask_ori,
394
- output_bbox=None,
395
- write_to=True,
396
- )
397
 
398
- else:
399
-
400
- post_stream_normalized(
401
- output_frames=output_frames[
402
- preprocessor.shift_alpha:
403
- ],
404
- ori_shape=video_ori.shape,
405
- model_video_shape=input_shape,
406
- writer=video_save_writer,
407
- crop_flag=False,
408
- videos_input_ori=None,
409
- video_ori=video_ori,
410
- mask_ori=mask_ori,
411
- output_bbox=None,
412
- write_to=True,
413
- )
414
 
415
- current_batch += 1
416
 
417
- # -------------------------------------------------
418
- # Progress
419
- # -------------------------------------------------
 
 
420
 
421
- if total_frames:
422
 
423
- processed = min(
424
- current_batch * (121 - preprocessor.shift_alpha),
425
- total_frames,
426
- )
427
 
428
- progress(
429
- processed / total_frames,
430
- desc=f"Processing video: {processed}/{total_frames} frames",
431
- )
432
 
433
- # -----------------------------------------------------
434
- # Close writer
435
- # -----------------------------------------------------
 
436
 
437
- if video_save_writer is not None:
438
- video_save_writer.Close()
439
- video_save_writer = None
440
 
441
- if not output_path.exists():
442
- raise gr.Error("EraserDiT did not produce an output video.")
 
 
 
 
 
 
 
443
 
444
- print("Finished:", output_path)
 
 
 
 
 
 
445
 
446
  return str(output_path)
447
 
448
 
449
- # ---------------------------------------------------------
450
  # Gradio UI
451
- # ---------------------------------------------------------
452
 
453
  with gr.Blocks(
454
- title="EraserDiT Video Inpainting"
455
  ) as demo:
456
 
457
  gr.Markdown(
458
  """
459
- # EraserDiT Video Inpainting
460
 
461
- Upload a video and a corresponding mask video to remove an object
462
- using **EraserDiT**.
463
 
464
- The video and mask must have:
 
465
 
466
- - The same resolution
467
- - The same number of frames
468
- - The same frame rate
469
- - MP4-compatible video encoding
470
 
471
- The mask should contain the area that should be removed.
 
 
 
 
472
  """
473
  )
474
 
@@ -489,15 +775,22 @@ The mask should contain the area that should be removed.
489
  )
490
 
491
  prompt = gr.Textbox(
492
- label="Prompt",
493
- value="A natural continuation of the surrounding video scene.",
494
- placeholder="Describe the scene after removing the masked object...",
495
- lines=3,
 
 
 
 
 
 
496
  )
497
 
498
- submit = gr.Button(
499
  "Run EraserDiT",
500
  variant="primary",
 
501
  )
502
 
503
  with gr.Column():
@@ -507,32 +800,41 @@ The mask should contain the area that should be removed.
507
  interactive=False,
508
  )
509
 
510
- submit.click(
511
  fn=process_video,
512
  inputs=[
513
  input_video,
514
  input_mask,
515
  prompt,
516
  ],
517
- outputs=output_video,
 
 
518
  )
519
 
520
  gr.Markdown(
521
  """
522
- ### Notes
 
 
523
 
524
- EraserDiT is a large video diffusion model. The official project
525
- reports **over 60 GB VRAM for 2K video**, so a high-memory GPU is
526
- recommended.
527
 
528
- Model: `jieeliu/EraserDiT`
 
529
  """
530
  )
531
 
532
 
 
 
 
 
533
  if __name__ == "__main__":
 
534
  demo.queue(
535
  max_size=1,
 
536
  ).launch(
537
  show_error=True,
538
  )
 
1
  import os
2
  import sys
 
3
  import tempfile
4
+ import subprocess
5
  from pathlib import Path
6
 
7
+ import gradio as gr
8
+ import spaces
9
+
10
+
11
+ # ============================================================
12
  # Configuration
13
+ # ============================================================
14
 
15
  REPO_DIR = Path("/tmp/EraserDiT")
16
+ MODEL_ID = "jieeliu/EraserDiT"
17
 
 
18
  os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1")
19
+ os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
20
+
21
+ # EraserDiT expects a large amount of VRAM.
22
+ # Keep CUDA memory allocation reasonably flexible.
23
+ os.environ.setdefault(
24
+ "PYTORCH_CUDA_ALLOC_CONF",
25
+ "expandable_segments:True"
26
+ )
27
+
28
+
29
+ # ============================================================
30
+ # Clone EraserDiT
31
+ # ============================================================
32
 
33
+ def setup_repo():
34
+ if REPO_DIR.exists():
35
+ return
36
 
 
37
  print("Cloning EraserDiT...")
38
+
39
  subprocess.run(
40
  [
41
  "git",
 
48
  check=True,
49
  )
50
 
 
51
 
52
+ setup_repo()
 
 
53
 
54
+ if str(REPO_DIR) not in sys.path:
55
+ sys.path.insert(0, str(REPO_DIR))
 
56
 
 
 
 
 
 
57
 
58
+ # ============================================================
59
+ # Global model
60
+ # ============================================================
 
61
 
62
+ pipeline = None
63
 
 
 
 
 
 
64
 
65
+ # ============================================================
66
+ # Negative prompt
67
+ # ============================================================
68
 
69
  NEGATIVE_PROMPT = (
70
  "Colorful color tone, overexposure, static, blurry details, "
 
76
  "the background, walking backwards, no noise"
77
  )
78
 
 
 
 
 
 
 
 
79
 
80
+ # ============================================================
81
+ # Model loading
82
+ # ============================================================
83
 
84
  def load_model():
85
  global pipeline
 
87
  if pipeline is not None:
88
  return pipeline
89
 
90
+ import torch
91
+
92
  if not torch.cuda.is_available():
93
  raise RuntimeError("CUDA GPU is required.")
94
 
95
+ print()
96
+ print("=" * 60)
97
  print("Loading EraserDiT")
98
+ print("=" * 60)
99
+
100
+ print(
101
+ "GPU:",
102
+ torch.cuda.get_device_name(0)
103
+ )
104
+
105
+ props = torch.cuda.get_device_properties(0)
106
+
107
+ print(
108
+ "VRAM:",
109
+ round(props.total_memory / (1024 ** 3), 2),
110
+ "GB"
111
+ )
112
+
113
+ print(
114
+ "Torch:",
115
+ torch.__version__
116
+ )
117
+
118
+ print("=" * 60)
119
+
120
+ # Import these only when the GPU job actually starts.
121
+ #
122
+ # This prevents Transformers / TorchAO / EraserDiT from
123
+ # being unnecessarily initialized during Space startup.
124
+ from utils.common import GlobalValues
125
+ from utils.inference_utils import init
126
+
127
+ GlobalValues.DEBUG = False
128
+
129
+ device = torch.device("cuda")
130
+ weight_dtype = torch.bfloat16
131
 
132
  pipeline = init(
133
+ device=device,
134
+ weight_dtype=weight_dtype,
135
  pre_dir=MODEL_ID,
136
  )
137
 
138
+ print("EraserDiT loaded successfully.")
139
+
140
  return pipeline
141
 
142
 
143
+ # ============================================================
144
+ # Video information
145
+ # ============================================================
146
+
147
+ def get_video_info(path):
148
+ import ffmpeg
149
+
150
+ info = ffmpeg.probe(path)
151
+
152
+ video_stream = next(
153
+ (
154
+ stream
155
+ for stream in info["streams"]
156
+ if stream.get("codec_type") == "video"
157
+ ),
158
+ None,
159
+ )
160
+
161
+ if video_stream is None:
162
+ raise ValueError("No video stream found.")
163
+
164
+ width = int(video_stream["width"])
165
+ height = int(video_stream["height"])
166
+
167
+ frame_count = None
168
+
169
+ nb_frames = video_stream.get("nb_frames")
170
+
171
+ if nb_frames:
172
+ try:
173
+ frame_count = int(nb_frames)
174
+ except Exception:
175
+ pass
176
+
177
+ fps_string = video_stream.get("r_frame_rate", "30/1")
178
+
179
+ try:
180
+ numerator, denominator = fps_string.split("/")
181
+ fps = float(numerator) / float(denominator)
182
+ except Exception:
183
+ fps = 30.0
184
+
185
+ bitrate = video_stream.get("bit_rate")
186
+
187
+ try:
188
+ bitrate_mbps = max(
189
+ 1,
190
+ int(bitrate) // 1_000_000
191
+ )
192
+ except Exception:
193
+ bitrate_mbps = 10
194
+
195
+ return {
196
+ "width": width,
197
+ "height": height,
198
+ "fps": fps,
199
+ "frame_count": frame_count,
200
+ "bitrate_mbps": bitrate_mbps,
201
+ }
202
+
203
+
204
+ # ============================================================
205
  # Main inference
206
+ # ============================================================
207
 
208
  @spaces.GPU(duration=1200)
209
+ def process_video(
210
+ video_path,
211
+ mask_path,
212
+ prompt,
213
+ progress=gr.Progress(),
214
+ ):
215
+
216
+ import math
217
+ import torch
218
+ import decord
219
+
220
+ # --------------------------------------------------------
221
+ # Validate inputs
222
+ # --------------------------------------------------------
223
+
224
  if video_path is None:
225
+ raise gr.Error(
226
+ "Please upload an input video."
227
+ )
228
 
229
  if mask_path is None:
230
+ raise gr.Error(
231
+ "Please upload a mask video."
232
+ )
233
 
234
  if not prompt or not prompt.strip():
235
+ raise gr.Error(
236
+ "Please enter a prompt describing the video scene."
237
+ )
238
 
239
  video_path = str(video_path)
240
  mask_path = str(mask_path)
241
 
 
242
  if not os.path.isfile(video_path):
243
+ raise gr.Error(
244
+ f"Input video does not exist: {video_path}"
245
+ )
246
 
247
  if not os.path.isfile(mask_path):
248
+ raise gr.Error(
249
+ f"Mask video does not exist: {mask_path}"
250
+ )
251
+
252
+ # --------------------------------------------------------
253
+ # Video information
254
+ # --------------------------------------------------------
255
+
256
+ try:
257
+ video_info = get_video_info(video_path)
258
+ mask_info = get_video_info(mask_path)
259
+ except Exception as e:
260
+ raise gr.Error(
261
+ f"Could not inspect videos: {e}"
262
+ )
263
+
264
+ width = video_info["width"]
265
+ height = video_info["height"]
266
+
267
+ mask_width = mask_info["width"]
268
+ mask_height = mask_info["height"]
269
 
270
  print()
271
+ print("=" * 60)
272
+ print("Input")
273
+ print("=" * 60)
 
 
 
274
 
275
+ print(
276
+ f"Video: {width}x{height} "
277
+ f"@ {video_info['fps']:.3f} FPS"
278
+ )
279
 
280
+ print(
281
+ f"Mask : {mask_width}x{mask_height} "
282
+ f"@ {mask_info['fps']:.3f} FPS"
283
+ )
284
 
285
+ # --------------------------------------------------------
286
+ # Resolution check
287
+ # --------------------------------------------------------
288
+ #
289
+ # The upstream inference.py switches to a bbox crop path
290
+ # above 1920x1088.
291
+ #
292
+ # Current upstream pre.py has:
293
+ #
294
+ # bbox_video(..., inference_idx)
295
+ #
296
+ # while the current inference.py calls it without that
297
+ # argument. Since this Space does not provide a bbox CSV,
298
+ # explicitly reject oversized input instead of invoking
299
+ # that incompatible path.
300
+ # --------------------------------------------------------
301
+
302
+ if width * height > 1920 * 1088:
303
+ raise gr.Error(
304
+ "Videos larger than 1920x1088 are not supported "
305
+ "by this Space version because EraserDiT's "
306
+ "large-resolution bbox workflow requires a bbox "
307
+ "CSV. Please provide a video at 1920x1088 or below."
308
  )
309
 
310
+ if mask_width != width or mask_height != height:
311
+ raise gr.Error(
312
+ "Video and mask must have exactly the same resolution.\n\n"
313
+ f"Video: {width}x{height}\n"
314
+ f"Mask: {mask_width}x{mask_height}"
315
  )
316
 
317
+ # --------------------------------------------------------
318
+ # Open readers to validate frame count
319
+ # --------------------------------------------------------
320
 
321
+ try:
322
+ video_reader_check = decord.VideoReader(
323
+ video_path,
324
+ ctx=decord.cpu(0),
325
+ )
 
 
 
 
326
 
327
+ mask_reader_check = decord.VideoReader(
328
+ mask_path,
329
+ ctx=decord.cpu(0),
330
+ )
 
 
 
 
331
 
332
+ video_frames = len(video_reader_check)
333
+ mask_frames = len(mask_reader_check)
334
 
335
+ del video_reader_check
336
+ del mask_reader_check
337
 
338
  except Exception as e:
339
+ raise gr.Error(
340
+ f"Could not read video: {e}"
341
+ )
342
+
343
+ if video_frames != mask_frames:
344
+ raise gr.Error(
345
+ "Video and mask must contain exactly the same "
346
+ "number of frames.\n\n"
347
+ f"Video frames: {video_frames}\n"
348
+ f"Mask frames: {mask_frames}"
349
+ )
350
+
351
+ print(
352
+ f"Frames: {video_frames}"
353
+ )
354
+
355
+ # --------------------------------------------------------
356
+ # FPS check
357
+ # --------------------------------------------------------
358
+
359
+ fps_difference = abs(
360
+ video_info["fps"] - mask_info["fps"]
361
+ )
362
+
363
+ if fps_difference > 0.01:
364
+ raise gr.Error(
365
+ "Video and mask must have the same frame rate.\n\n"
366
+ f"Video FPS: {video_info['fps']}\n"
367
+ f"Mask FPS: {mask_info['fps']}"
368
+ )
369
 
370
+ # --------------------------------------------------------
371
  # Load model
372
+ # --------------------------------------------------------
373
 
374
  pipe = load_model()
375
 
376
+ # --------------------------------------------------------
377
+ # Import remaining EraserDiT components
378
+ # --------------------------------------------------------
379
 
380
+ from utils.pre import VideoInpaintPre
381
+ from utils.inference_utils import inference_batch
382
+ from utils.post import post_stream_normalized
383
+ from utils.post_pkg import FFmpegWriter
384
 
385
+ # --------------------------------------------------------
386
+ # Configuration matching upstream inference.py
387
+ # --------------------------------------------------------
388
 
389
+ device = torch.device("cuda")
390
 
391
+ weight_dtype = torch.bfloat16
 
 
392
 
393
  preprocessor = VideoInpaintPre(
394
+ device=device,
395
  align_h=32,
396
  align_w=32,
397
  ksize=(9, 9),
 
401
  crop_flag=False,
402
  )
403
 
404
+ generator = None
 
 
405
 
406
+ # --------------------------------------------------------
407
+ # Temporary output
408
+ # --------------------------------------------------------
409
 
410
+ output_dir = Path(
411
+ tempfile.mkdtemp(
412
+ prefix="eraserdit_"
413
+ )
414
  )
415
 
416
+ input_name = Path(video_path).stem
417
 
418
+ output_path = output_dir / (
419
+ f"{input_name}_eraserdit.mp4"
420
+ )
 
 
421
 
422
+ print()
423
+ print("=" * 60)
424
+ print("Output")
425
+ print("=" * 60)
426
 
427
+ print(
428
+ output_path
429
+ )
 
 
 
 
430
 
431
+ # --------------------------------------------------------
432
+ # Processing state
433
+ # --------------------------------------------------------
434
 
 
435
  pre_video_shift = None
436
 
437
+ video_save_writer = None
438
+
439
  current_batch = 0
440
 
441
+ # The upstream model uses:
442
+ #
443
+ # TEMP_INFER_LEN = 121
444
+ # shift_alpha = 9
445
+ #
446
+ # Subsequent batches therefore overlap by 9 frames.
447
+ #
448
+ # See upstream inference.py.
449
+ # --------------------------------------------------------
450
 
451
  try:
 
 
 
 
 
 
 
 
452
 
453
+ while True:
454
 
455
+ # ------------------------------------------------
456
+ # Load next temporal batch
457
+ # ------------------------------------------------
458
+
459
+ (
460
+ video_ori,
461
+ mask_ori,
462
+ fps,
463
+ videos_input_ori,
464
+ masks_input_ori,
465
+ ) = preprocessor.load_videos(
466
  video_path=video_path,
467
  mask_path=mask_path,
468
  bbox_path=None,
 
470
  sample_rate=1,
471
  batch_idx=current_batch,
472
  )
 
473
 
474
+ # ------------------------------------------------
475
+ # End of video
476
+ # ------------------------------------------------
477
 
478
+ if video_ori is None:
479
+ break
 
 
 
 
480
 
481
+ # ------------------------------------------------
482
+ # Prepare input
483
+ # ------------------------------------------------
484
+
485
+ (
486
+ video_input,
487
+ mask_input,
488
+ _,
489
+ ) = preprocessor(
490
+ video_ori,
491
+ mask_ori,
492
+ batch_idx=current_batch,
493
+ format="nhwc",
494
+ )
495
 
496
+ input_shape = (
497
+ preprocessor.TranslateShape(
498
+ video_input.shape,
499
+ src="nchw",
500
+ dst="nhwc",
501
+ )
502
+ )
503
+
504
+ # ------------------------------------------------
505
+ # Create output writer
506
+ # ------------------------------------------------
507
+
508
+ if video_save_writer is None:
509
+
510
+ video_save_writer = FFmpegWriter(
511
+ path=str(output_path),
512
+ width=video_ori.shape[2],
513
+ height=video_ori.shape[1],
514
+ fps=fps,
515
+ bitrate=(
516
+ f"{video_info['bitrate_mbps']}M"
517
+ ),
518
+ )
519
+
520
+ # ------------------------------------------------
521
+ # Temporal mask shift
522
+ # ------------------------------------------------
523
+
524
+ if current_batch == 0:
525
+
526
+ masks_zero_shift = torch.zeros(
527
+ (
528
+ math.ceil(
529
+ preprocessor.shift_alpha / 8
530
+ ),
531
+ mask_input.shape[1],
532
+ mask_input.shape[2],
533
+ mask_input.shape[3],
534
+ ),
535
+ dtype=mask_input.dtype,
536
+ )
537
+
538
+ else:
539
+
540
+ video_input = torch.cat(
541
+ [
542
+ pre_video_shift,
543
+ video_input,
544
+ ],
545
+ dim=0,
546
+ )
547
+
548
+ mask_input = torch.cat(
549
+ [
550
+ masks_zero_shift,
551
+ mask_input,
552
+ ],
553
+ dim=0,
554
+ )
555
+
556
+ print()
557
+ print(
558
+ f"Batch {current_batch}: "
559
+ f"{video_ori.shape[0]} source frames"
560
+ )
561
 
562
+ print(
563
+ "Model input:",
564
+ tuple(video_input.shape)
565
+ )
566
 
567
+ # ------------------------------------------------
568
+ # EraserDiT inference
569
+ # ------------------------------------------------
570
+
571
+ output_frames = inference_batch(
572
+ videos=video_input,
573
+ masks_input=mask_input,
574
+ prompt=prompt.strip(),
575
+ negative_prompt=NEGATIVE_PROMPT,
576
+ pipeline=pipe,
577
+ generator=generator,
578
+ device=device,
579
+ weight_dtype=weight_dtype,
580
  )
581
 
582
+ # ------------------------------------------------
583
+ # Save overlap for next batch
584
+ # ------------------------------------------------
585
 
586
+ pre_video_shift = (
587
+ output_frames[
588
+ -preprocessor.shift_alpha:
589
+ ].cpu()
590
+ )
591
 
592
+ # ------------------------------------------------
593
+ # Write output
594
+ # ------------------------------------------------
595
+
596
+ if current_batch == 0:
597
+
598
+ post_stream_normalized(
599
+ output_frames=output_frames,
600
+ ori_shape=video_ori.shape,
601
+ model_video_shape=input_shape,
602
+ crop_flag=False,
603
+ videos_input_ori=None,
604
+ video_ori=video_ori,
605
+ mask_ori=mask_ori,
606
+ output_bbox=None,
607
+ writer=video_save_writer,
608
+ write_to=True,
609
+ )
610
+
611
+ else:
612
+
613
+ post_stream_normalized(
614
+ output_frames=(
615
+ output_frames[
616
+ preprocessor.shift_alpha:
617
+ ]
618
+ ),
619
+ ori_shape=video_ori.shape,
620
+ model_video_shape=input_shape,
621
+ crop_flag=False,
622
+ videos_input_ori=None,
623
+ video_ori=video_ori,
624
+ mask_ori=mask_ori,
625
+ output_bbox=None,
626
+ writer=video_save_writer,
627
+ write_to=True,
628
+ )
629
+
630
+ # ------------------------------------------------
631
+ # Progress
632
+ # ------------------------------------------------
633
+
634
+ current_batch += 1
635
+
636
+ # First batch contributes up to 121 frames.
637
+ # Every following batch contributes 112 frames.
638
+ processed_frames = min(
639
+ 121
640
+ + max(
641
+ 0,
642
+ current_batch - 1
643
+ ) * (
644
+ 121
645
+ - preprocessor.shift_alpha
646
  ),
647
+ video_frames,
648
  )
649
 
650
+ fraction = (
651
+ processed_frames / video_frames
652
+ )
653
 
654
+ progress(
655
+ fraction,
656
+ desc=(
657
+ f"Processing "
658
+ f"{processed_frames}/{video_frames} frames"
659
+ ),
660
  )
661
 
662
+ print(
663
+ f"Progress: "
664
+ f"{processed_frames}/{video_frames}"
 
 
665
  )
666
 
667
+ # ------------------------------------------------
668
+ # Release temporary GPU tensors
669
+ # ------------------------------------------------
 
670
 
671
+ del output_frames
672
+ del video_input
673
+ del mask_input
674
+ del video_ori
675
+ del mask_ori
 
 
 
 
 
 
 
 
 
676
 
677
+ # Don't aggressively empty CUDA cache every batch.
678
+ #
679
+ # EraserDiT is a very large model and repeatedly
680
+ # flushing the allocator can hurt performance.
 
 
681
 
682
+ # ----------------------------------------------------
683
+ # Close output
684
+ # ----------------------------------------------------
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
685
 
686
+ if video_save_writer is not None:
687
+ video_save_writer.Close()
688
+ video_save_writer = None
 
 
 
 
 
 
 
 
 
 
 
 
 
689
 
690
+ except Exception:
691
 
692
+ if video_save_writer is not None:
693
+ try:
694
+ video_save_writer.Close()
695
+ except Exception:
696
+ pass
697
 
698
+ video_save_writer = None
699
 
700
+ raise
 
 
 
701
 
702
+ # --------------------------------------------------------
703
+ # Validate output
704
+ # --------------------------------------------------------
 
705
 
706
+ if not output_path.exists():
707
+ raise gr.Error(
708
+ "EraserDiT finished without producing an output video."
709
+ )
710
 
711
+ output_size = output_path.stat().st_size
 
 
712
 
713
+ if output_size <= 0:
714
+ raise gr.Error(
715
+ "The generated output video is empty."
716
+ )
717
+
718
+ print()
719
+ print("=" * 60)
720
+ print("Finished")
721
+ print("=" * 60)
722
 
723
+ print(
724
+ f"Output: {output_path}"
725
+ )
726
+
727
+ print(
728
+ f"Size: {output_size / (1024 * 1024):.2f} MB"
729
+ )
730
 
731
  return str(output_path)
732
 
733
 
734
+ # ============================================================
735
  # Gradio UI
736
+ # ============================================================
737
 
738
  with gr.Blocks(
739
+ title="EraserDiT Video Object Removal"
740
  ) as demo:
741
 
742
  gr.Markdown(
743
  """
744
+ # EraserDiT Video Object Removal
745
 
746
+ Upload your **original video** and a corresponding **mask video**.
 
747
 
748
+ The masked area will be removed and EraserDiT will generate new
749
+ content consistent with the surrounding scene.
750
 
751
+ ### Input requirements
 
 
 
752
 
753
+ - Video and mask must have the **same resolution**
754
+ - Video and mask must have the **same number of frames**
755
+ - Video and mask must have the **same FPS**
756
+ - Maximum supported resolution: **1920×1088**
757
+ - A high-VRAM GPU is required
758
  """
759
  )
760
 
 
775
  )
776
 
777
  prompt = gr.Textbox(
778
+ label="Scene Description",
779
+ placeholder=(
780
+ "Describe what the scene should look like "
781
+ "after removing the masked object."
782
+ ),
783
+ value=(
784
+ "A natural continuation of the "
785
+ "surrounding video scene."
786
+ ),
787
+ lines=4,
788
  )
789
 
790
+ run_button = gr.Button(
791
  "Run EraserDiT",
792
  variant="primary",
793
+ size="lg",
794
  )
795
 
796
  with gr.Column():
 
800
  interactive=False,
801
  )
802
 
803
+ run_button.click(
804
  fn=process_video,
805
  inputs=[
806
  input_video,
807
  input_mask,
808
  prompt,
809
  ],
810
+ outputs=[
811
+ output_video,
812
+ ],
813
  )
814
 
815
  gr.Markdown(
816
  """
817
+ ### Example prompt
818
+
819
+ If you're removing a person from a street:
820
 
821
+ `A city street with buildings, cars and pedestrians in the background.`
 
 
822
 
823
+ The prompt should describe the **scene/content**, not the object
824
+ being removed.
825
  """
826
  )
827
 
828
 
829
+ # ============================================================
830
+ # Launch
831
+ # ============================================================
832
+
833
  if __name__ == "__main__":
834
+
835
  demo.queue(
836
  max_size=1,
837
+ default_concurrency_limit=1,
838
  ).launch(
839
  show_error=True,
840
  )