diffcve-2607 / app.py
multimodalart's picture
multimodalart HF Staff
Cap frames at 10 to match measured 90s GPU budget
6e379ba verified
Raw History Blame Contribute Delete
16.4 kB
import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import spaces # noqa: E402 (must precede torch)
import json # noqa: E402
import math # noqa: E402
import random # noqa: E402
import shutil # noqa: E402
import subprocess # noqa: E402
import sys # noqa: E402
import tempfile # noqa: E402
from pathlib import Path # noqa: E402
import numpy as np # noqa: E402
import torch # noqa: E402
from PIL import Image # noqa: E402
# torch 2.6+ defaults torch.load to weights_only=True, which rejects the
# pickled numpy objects inside the authors' checkpoints.
_orig_torch_load = torch.load
torch.load = lambda *a, **k: _orig_torch_load(*a, **{**k, "weights_only": k.get("weights_only", False)})
# torch 2.9+ removed torch.onnx.is_in_onnx_export(); vendored code still calls it.
if not hasattr(torch.onnx, "is_in_onnx_export"):
torch.onnx.is_in_onnx_export = lambda: False
import gradio as gr # noqa: E402
from huggingface_hub import hf_hub_download # noqa: E402
from omegaconf import OmegaConf # noqa: E402
APP_ROOT = Path(__file__).resolve().parent
sys.path.insert(0, str(APP_ROOT))
from codec.processor import VideoStream # noqa: E402
from diffcve.utils import instantiate_from_config, load_checkpoint # noqa: E402
from scripts.color_correction import adaptive_instance_normalization, wavelet_reconstruction # noqa: E402
from scripts.stream_adapter import iter_segments # noqa: E402
from scripts.tiling import build_patch_grid, merge_patch_grid # noqa: E402
MODEL_REPO = "wqmaker/DiffCVE"
CHECKPOINT = "single_step.ckpt"
TEXT_CONDITIONS = "text_conditions.pt"
CODEC_DIR = APP_ROOT / "codec"
EXAMPLES_DIR = APP_ROOT / "examples"
# The Hub's git upload does not preserve the executable bit, so re-assert it
# for the bundled JM codec binaries (idempotent).
for _binary in (CODEC_DIR / "encoder", CODEC_DIR / "decoder"):
_binary.chmod(0o755)
SEGMENT_SIZE = 5 # the checkpoint is trained on 5-frame segments
DEFAULT_MAX_FRAMES = 10
MAX_MAX_FRAMES = 10 # 10 frames measured 76.8s on ZeroGPU; more would exceed the GPU budget
MAX_DIM = 720
DEFAULT_DURATION_S = 90 # measured 76.8s for 10 frames on ZeroGPU + ~15% margin
FFMPEG = shutil.which("ffmpeg") or "ffmpeg"
FFPROBE = shutil.which("ffprobe") or "ffprobe"
def seed_everything(seed):
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
def probe_video(path):
"""Return (width, height, fps, frames) of a video via ffprobe."""
out = subprocess.run(
[FFPROBE, "-v", "error", "-select_streams", "v:0",
"-show_entries", "stream=width,height,r_frame_rate,nb_frames",
"-show_entries", "format=duration",
"-of", "json", str(path)],
capture_output=True, text=True, check=True).stdout
info = json.loads(out)
stream = info["streams"][0]
width, height = int(stream["width"]), int(stream["height"])
num, den = stream["r_frame_rate"].split("/")
fps = float(num) / float(den)
duration = float(info.get("format", {}).get("duration") or 0)
frames = int(stream.get("nb_frames") or 0) or int(round(duration * fps))
return width, height, fps, frames
def encode_h264(yuv_path, bitstream_path, width, height, frames, fps, qp):
"""Encode raw yuv420p to constant-QP H.264 with the authors' JM encoder."""
cmd = [
str(CODEC_DIR / "encoder"), "-d", str(CODEC_DIR / "encoder.cfg"),
"-p", f"InputFile={yuv_path}",
"-p", f"OutputFile={bitstream_path}",
"-p", f"SourceWidth={width}",
"-p", f"SourceHeight={height}",
"-p", f"FramesToBeEncoded={frames}",
"-p", f"FrameRate={round(fps)}",
"-p", "ProfileIDC=66", "-p", "LevelIDC=50",
"-p", f"QPISlice={qp}", "-p", f"QPPSlice={qp}",
"-p", "NumberReferenceFrames=5", "-p", "SearchRange=32",
"-p", "IntraPeriod=0", "-p", "IDRPeriod=0", "-p", "NumberBFrames=0",
"-p", "YUVFormat=1", "-p", "SourceBitDepthLuma=8", "-p", "SourceBitDepthChroma=8",
"-p", "ReconFile=/dev/null", "-p", "TraceFile=/dev/null",
"-p", "LeakyBucketParamFile=/dev/null", "-p", "StatsFile=/dev/null",
]
result = subprocess.run(cmd, capture_output=True, text=True, timeout=600)
if result.returncode != 0 or not Path(bitstream_path).is_file():
raise RuntimeError(f"H.264 encoding failed: {result.stderr[-2000:]}")
def frames_to_mp4(frames_dir, out_path, fps):
"""Mux a directory of PNG frames into an mp4."""
subprocess.run(
[FFMPEG, "-y", "-nostdin", "-hide_banner", "-loglevel", "error",
"-framerate", str(fps), "-i", str(frames_dir / "f%05d.png"),
"-c:v", "libx264", "-pix_fmt", "yuv420p", "-crf", "18", str(out_path)],
capture_output=True, text=True, check=True)
@torch.no_grad()
def enhance_segment(model, images, residues, motions, qp, seed, color_correction):
"""Enhance one 5-frame segment — mirrors the reference scripts/video_inference.py."""
height, width = images.shape[-2:]
if height < 720:
rows, cols = 1, 1
pad_h, pad_w = (-height) % 64, (-width) % 64
elif height < 1080:
rows, cols = 1, 2
pad_h, pad_w = (-height) % 64, 0
else:
rows, cols = (2, 3) if height < 1600 else (3, 4)
pad_h = pad_w = 0
if pad_h or pad_w:
images, residues, motions = [
torch.nn.functional.pad(t, (0, pad_w, 0, pad_h), mode="reflect")
for t in [images, residues, motions]]
patches, metadata = build_patch_grid(*images.shape[-2:], rows, cols)
vae = model.vae
outputs = []
for hs, ws in patches:
seed_everything(seed)
image_patch = images[:, :, hs, ws]
posterior, encoder_features = vae.encode(image_patch)
latent = model.sample_latent(posterior)
prior, prior_features = model.prior_encoder(residues[:, :, hs, ws],
motions[:, :, hs, ws],
return_features=True)
condition = model.text_conditions[qp].to(model.device)
samples = model.sample(cond=condition, struct_cond=latent, prior=prior,
x_T=latent, timesteps=1)
qp_tensor = torch.full((images.shape[0],), qp,
dtype=torch.float32, device=model.device) / 51.0
output = vae.decode(samples / model.scale_factor, encoder_features,
prior_features, qp_tensor)
outputs.append(output.float().cpu())
del posterior, encoder_features, latent, prior, prior_features
del condition, samples, qp_tensor, output
output = merge_patch_grid(images.shape, patches, outputs, metadata,
device="cpu").to(model.device)
if color_correction == "adain":
output = adaptive_instance_normalization(output, images)
elif color_correction == "wavelet":
output = wavelet_reconstruction(output, images)
output = torch.clamp((output + 1.0) / 2.0, 0.0, 1.0)
if not torch.isfinite(output).all():
raise RuntimeError("Inference produced non-finite image values")
return output[:, :, :height, :width].cpu().numpy().transpose(0, 2, 3, 1) * 255
print("Loading DiffCVE single-step model …", flush=True)
text_conditions_path = hf_hub_download(MODEL_REPO, TEXT_CONDITIONS)
checkpoint_path = hf_hub_download(MODEL_REPO, CHECKPOINT)
config = OmegaConf.load(str(APP_ROOT / "configs" / "single_step.yaml"))
params = config.model.params
params.text_conditions_path = str(text_conditions_path)
params.cond_stage_config = None
params.text_prompts = None
params.text_condition_qps = list(range(52)) # interpolate cached anchors to any QP 0..51
model = load_checkpoint(instantiate_from_config(config.model), str(checkpoint_path)).eval().to("cuda")
model.denoiser = model.denoiser.merge_and_unload()
model.eval().requires_grad_(False).to("cuda")
model.register_schedule(timesteps=1000, linear_start=0.00085, linear_end=0.012)
model.to("cuda")
model.vae.decoder.fusion_w = 1.0
_missing = {47} - set(model.text_conditions.keys())
if _missing:
raise RuntimeError(f"Missing text conditions for QPs: {sorted(_missing)}")
print("DiffCVE model ready.", flush=True)
@spaces.GPU(duration=DEFAULT_DURATION_S)
def enhance(video, qp: int = 37, max_frames: int = DEFAULT_MAX_FRAMES, seed: int = 42,
color_correction: str = "adain",
progress: gr.Progress = gr.Progress(track_tqdm=False)):
"""Enhance a compressed video with DiffCVE.
The uploaded video is first H.264-compressed at the chosen quantization
parameter (QP), then restored with the diffusion enhancer using H.264
coding priors (motion vectors + residuals) and QP-conditioned text prompts.
Args:
video: Input video file (any common container; no audio required).
qp: H.264 quantization parameter (0-51); higher means worse quality.
max_frames: How many frames to enhance (each frame costs GPU time).
seed: Random seed for latent sampling.
color_correction: Post-hoc color alignment to the compressed input
("adain", "wavelet", or "nofix").
Returns:
(enhanced video, compressed input video, status text)
"""
qp = int(qp)
seed = int(seed)
max_frames = int(max_frames)
if not video:
raise gr.Error("Please upload a video first.")
if not 0 <= qp <= 51:
raise gr.Error("QP must be between 0 and 51.")
if max_frames < 1:
max_frames = DEFAULT_MAX_FRAMES
max_frames = min(max_frames, MAX_MAX_FRAMES)
src = Path(video)
width, height, fps, total_frames = probe_video(src)
if total_frames < 1:
raise gr.Error("Could not read any video frames.")
fps = min(max(fps, 1.0), 60.0)
frames = min(max_frames, total_frames)
scale = None
if max(width, height) > MAX_DIM:
scale = MAX_DIM / max(width, height)
if scale is not None:
width = int(math.floor(width * scale / 16)) * 16
height = int(math.floor(height * scale / 16)) * 16
width += width % 2
height += height % 2
if width < 16 or height < 16:
raise gr.Error("Video is too small after resizing.")
with tempfile.TemporaryDirectory(prefix="diffcve-") as work_str:
work = Path(work_str)
yuv_path = work / "input.yuv"
# 1. Decode + scale + trim the upload to raw yuv420p (the JM encoder's input).
cmd = [FFMPEG, "-y", "-nostdin", "-hide_banner", "-loglevel", "error",
"-i", str(src),
"-frames:v", str(frames),
"-vf", (f"scale={width}:{height}" if scale is not None else "null"),
"-pix_fmt", "yuv420p", "-f", "rawvideo", str(yuv_path)]
subprocess.run(cmd, capture_output=True, text=True, check=True)
if not yuv_path.is_file() or yuv_path.stat().st_size < width * height * 3 // 2:
raise gr.Error("Failed to decode the uploaded video.")
# 2. Compress with the authors' JM encoder at the chosen constant QP.
bitstream = work / "input.264"
encode_h264(yuv_path, bitstream, width, height, frames, fps, qp)
# 3. Stream-decode with coding priors and enhance segment by segment.
decoded_dir, enhanced_dir = work / "decoded", work / "enhanced"
decoded_dir.mkdir()
enhanced_dir.mkdir()
written = 0
n_segments = math.ceil(frames / SEGMENT_SIZE)
with VideoStream(bitstream, decoder=CODEC_DIR / "decoder", ffmpeg=FFMPEG,
fps=fps, expected_qp=qp, prefetch=5, prior_prefetch=32) as vid:
seg_i = 0
for segment in iter_segments(vid, SEGMENT_SIZE, frames):
images, residues, motions = segment.to_torch(model.device)
seg_i += 1
progress((seg_i, n_segments),
desc=f"Enhancing segment {seg_i}/{n_segments}")
with torch.autocast("cuda"):
out = enhance_segment(model, images, residues, motions,
segment.qp, seed, color_correction)
for i, frame in enumerate(segment.frames):
Image.fromarray(out[i].astype(np.uint8)).save(enhanced_dir / f"f{written:05d}.png")
Image.fromarray(frame.rgb).save(decoded_dir / f"f{written:05d}.png")
written += 1
del images, residues, motions, out
if not vid.completed:
raise RuntimeError("Bitstream decoding did not complete validation")
if written == 0:
raise gr.Error("Enhancement produced no frames.")
# 4. Mux both videos and move them out of the temp dir before it is wiped.
final_dir = Path(tempfile.mkdtemp(prefix="diffcve-out-"))
enhanced_mp4 = final_dir / f"enhanced_qp{qp}.mp4"
decoded_mp4 = final_dir / f"compressed_qp{qp}.mp4"
frames_to_mp4(enhanced_dir, enhanced_mp4, fps)
frames_to_mp4(decoded_dir, decoded_mp4, fps)
status = (f"Enhanced {written} frames at {width}×{height} @ {fps:.2f} fps, "
f"QP {qp} · seed {seed} · color fix: {color_correction}")
return str(enhanced_mp4), str(decoded_mp4), status
EXAMPLES = [
[str(EXAMPLES_DIR / "ocean_waves_qp37.mp4"), 37],
[str(EXAMPLES_DIR / "cat_qp47.mp4"), 47],
[str(EXAMPLES_DIR / "jellyfish_qp32.mp4"), 32],
]
CSS = """
#col-container { max-width: 1200px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""
with gr.Blocks(title="DiffCVE — Compressed Video Enhancement") as demo:
gr.Markdown(
"""
# 🎬 DiffCVE: Diffusion-based Compressed Video Enhancement
Upload a video, pick an H.264 quantization parameter (QP), and the
single-step distilled DiffCVE model restores it — using the bitstream's
own coding priors (motion vectors + prediction residuals), QP-conditioned
degradation prompts, and a coding-prior-weighted VAE decoder (CPWF).
*Paper: [DiffCVE: Diffusion-based Compressed Video Enhancement](https://huggingface.co/papers/2607.07195)
· Code: [wqmaker/DiffCVE](https://github.com/wqmaker/DiffCVE)
· Model: [wqmaker/DiffCVE](https://huggingface.co/wqmaker/DiffCVE)*
"""
)
with gr.Row():
with gr.Column():
video_in = gr.Video(label="Input video", sources=["upload"])
qp_slider = gr.Slider(0, 51, value=37, step=1, label="QP (compression severity)",
info="Higher QP → stronger blocking artifacts to remove")
enhance_btn = gr.Button("Enhance video", variant="primary")
with gr.Column():
video_out = gr.Video(label="Enhanced video (DiffCVE)")
video_cmp = gr.Video(label="Compressed input (QP applied)")
status_text = gr.Textbox(label="Status", interactive=False)
with gr.Accordion("Advanced options", open=False):
with gr.Row():
frames_slider = gr.Slider(1, MAX_MAX_FRAMES, value=DEFAULT_MAX_FRAMES, step=1,
label="Frames to enhance",
info="Each frame costs GPU time; 10 frames ≈ 80 s")
seed_number = gr.Number(value=42, label="Seed", precision=0)
color_dropdown = gr.Dropdown(["adain", "wavelet", "nofix"], value="adain",
label="Color correction",
info="Aligns output colors with the compressed input")
gr.Markdown(
"""
**Tip:** videos larger than 720 px on either edge are downscaled first.
Keep clips short (≤ 10 frames) — diffusion restoration of each
5-frame segment is a heavy forward pass.
"""
)
gr.Examples(
examples=EXAMPLES,
inputs=[video_in, qp_slider],
outputs=[video_out, video_cmp, status_text],
fn=enhance,
cache_examples=True,
cache_mode="lazy",
)
enhance_btn.click(
fn=enhance,
inputs=[video_in, qp_slider, frames_slider, seed_number, color_dropdown],
outputs=[video_out, video_cmp, status_text],
api_name="enhance",
)
demo.launch(mcp_server=True, theme=gr.themes.Citrus(), css=CSS)