Spaces:
Running on Zero
Running on Zero
Download app.py from hugging-apps/diffcve-2607: direct link, hf CLI and curl.
- Browser
- Download file 16.4 kB
-
https://huggingface.co/spaces/hugging-apps/diffcve-2607/resolve/main/app.py
- Command line
-
hf download hf://spaces/hugging-apps/diffcve-2607/app.py
-
curl -L -o app.py https://huggingface.co/spaces/hugging-apps/diffcve-2607/resolve/main/app.py
16.4 kB
| import os | |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") | |
| import spaces # noqa: E402 (must precede torch) | |
| import json # noqa: E402 | |
| import math # noqa: E402 | |
| import random # noqa: E402 | |
| import shutil # noqa: E402 | |
| import subprocess # noqa: E402 | |
| import sys # noqa: E402 | |
| import tempfile # noqa: E402 | |
| from pathlib import Path # noqa: E402 | |
| import numpy as np # noqa: E402 | |
| import torch # noqa: E402 | |
| from PIL import Image # noqa: E402 | |
| # torch 2.6+ defaults torch.load to weights_only=True, which rejects the | |
| # pickled numpy objects inside the authors' checkpoints. | |
| _orig_torch_load = torch.load | |
| torch.load = lambda *a, **k: _orig_torch_load(*a, **{**k, "weights_only": k.get("weights_only", False)}) | |
| # torch 2.9+ removed torch.onnx.is_in_onnx_export(); vendored code still calls it. | |
| if not hasattr(torch.onnx, "is_in_onnx_export"): | |
| torch.onnx.is_in_onnx_export = lambda: False | |
| import gradio as gr # noqa: E402 | |
| from huggingface_hub import hf_hub_download # noqa: E402 | |
| from omegaconf import OmegaConf # noqa: E402 | |
| APP_ROOT = Path(__file__).resolve().parent | |
| sys.path.insert(0, str(APP_ROOT)) | |
| from codec.processor import VideoStream # noqa: E402 | |
| from diffcve.utils import instantiate_from_config, load_checkpoint # noqa: E402 | |
| from scripts.color_correction import adaptive_instance_normalization, wavelet_reconstruction # noqa: E402 | |
| from scripts.stream_adapter import iter_segments # noqa: E402 | |
| from scripts.tiling import build_patch_grid, merge_patch_grid # noqa: E402 | |
| MODEL_REPO = "wqmaker/DiffCVE" | |
| CHECKPOINT = "single_step.ckpt" | |
| TEXT_CONDITIONS = "text_conditions.pt" | |
| CODEC_DIR = APP_ROOT / "codec" | |
| EXAMPLES_DIR = APP_ROOT / "examples" | |
| # The Hub's git upload does not preserve the executable bit, so re-assert it | |
| # for the bundled JM codec binaries (idempotent). | |
| for _binary in (CODEC_DIR / "encoder", CODEC_DIR / "decoder"): | |
| _binary.chmod(0o755) | |
| SEGMENT_SIZE = 5 # the checkpoint is trained on 5-frame segments | |
| DEFAULT_MAX_FRAMES = 10 | |
| MAX_MAX_FRAMES = 10 # 10 frames measured 76.8s on ZeroGPU; more would exceed the GPU budget | |
| MAX_DIM = 720 | |
| DEFAULT_DURATION_S = 90 # measured 76.8s for 10 frames on ZeroGPU + ~15% margin | |
| FFMPEG = shutil.which("ffmpeg") or "ffmpeg" | |
| FFPROBE = shutil.which("ffprobe") or "ffprobe" | |
| def seed_everything(seed): | |
| random.seed(seed) | |
| np.random.seed(seed) | |
| torch.manual_seed(seed) | |
| torch.cuda.manual_seed_all(seed) | |
| def probe_video(path): | |
| """Return (width, height, fps, frames) of a video via ffprobe.""" | |
| out = subprocess.run( | |
| [FFPROBE, "-v", "error", "-select_streams", "v:0", | |
| "-show_entries", "stream=width,height,r_frame_rate,nb_frames", | |
| "-show_entries", "format=duration", | |
| "-of", "json", str(path)], | |
| capture_output=True, text=True, check=True).stdout | |
| info = json.loads(out) | |
| stream = info["streams"][0] | |
| width, height = int(stream["width"]), int(stream["height"]) | |
| num, den = stream["r_frame_rate"].split("/") | |
| fps = float(num) / float(den) | |
| duration = float(info.get("format", {}).get("duration") or 0) | |
| frames = int(stream.get("nb_frames") or 0) or int(round(duration * fps)) | |
| return width, height, fps, frames | |
| def encode_h264(yuv_path, bitstream_path, width, height, frames, fps, qp): | |
| """Encode raw yuv420p to constant-QP H.264 with the authors' JM encoder.""" | |
| cmd = [ | |
| str(CODEC_DIR / "encoder"), "-d", str(CODEC_DIR / "encoder.cfg"), | |
| "-p", f"InputFile={yuv_path}", | |
| "-p", f"OutputFile={bitstream_path}", | |
| "-p", f"SourceWidth={width}", | |
| "-p", f"SourceHeight={height}", | |
| "-p", f"FramesToBeEncoded={frames}", | |
| "-p", f"FrameRate={round(fps)}", | |
| "-p", "ProfileIDC=66", "-p", "LevelIDC=50", | |
| "-p", f"QPISlice={qp}", "-p", f"QPPSlice={qp}", | |
| "-p", "NumberReferenceFrames=5", "-p", "SearchRange=32", | |
| "-p", "IntraPeriod=0", "-p", "IDRPeriod=0", "-p", "NumberBFrames=0", | |
| "-p", "YUVFormat=1", "-p", "SourceBitDepthLuma=8", "-p", "SourceBitDepthChroma=8", | |
| "-p", "ReconFile=/dev/null", "-p", "TraceFile=/dev/null", | |
| "-p", "LeakyBucketParamFile=/dev/null", "-p", "StatsFile=/dev/null", | |
| ] | |
| result = subprocess.run(cmd, capture_output=True, text=True, timeout=600) | |
| if result.returncode != 0 or not Path(bitstream_path).is_file(): | |
| raise RuntimeError(f"H.264 encoding failed: {result.stderr[-2000:]}") | |
| def frames_to_mp4(frames_dir, out_path, fps): | |
| """Mux a directory of PNG frames into an mp4.""" | |
| subprocess.run( | |
| [FFMPEG, "-y", "-nostdin", "-hide_banner", "-loglevel", "error", | |
| "-framerate", str(fps), "-i", str(frames_dir / "f%05d.png"), | |
| "-c:v", "libx264", "-pix_fmt", "yuv420p", "-crf", "18", str(out_path)], | |
| capture_output=True, text=True, check=True) | |
| def enhance_segment(model, images, residues, motions, qp, seed, color_correction): | |
| """Enhance one 5-frame segment — mirrors the reference scripts/video_inference.py.""" | |
| height, width = images.shape[-2:] | |
| if height < 720: | |
| rows, cols = 1, 1 | |
| pad_h, pad_w = (-height) % 64, (-width) % 64 | |
| elif height < 1080: | |
| rows, cols = 1, 2 | |
| pad_h, pad_w = (-height) % 64, 0 | |
| else: | |
| rows, cols = (2, 3) if height < 1600 else (3, 4) | |
| pad_h = pad_w = 0 | |
| if pad_h or pad_w: | |
| images, residues, motions = [ | |
| torch.nn.functional.pad(t, (0, pad_w, 0, pad_h), mode="reflect") | |
| for t in [images, residues, motions]] | |
| patches, metadata = build_patch_grid(*images.shape[-2:], rows, cols) | |
| vae = model.vae | |
| outputs = [] | |
| for hs, ws in patches: | |
| seed_everything(seed) | |
| image_patch = images[:, :, hs, ws] | |
| posterior, encoder_features = vae.encode(image_patch) | |
| latent = model.sample_latent(posterior) | |
| prior, prior_features = model.prior_encoder(residues[:, :, hs, ws], | |
| motions[:, :, hs, ws], | |
| return_features=True) | |
| condition = model.text_conditions[qp].to(model.device) | |
| samples = model.sample(cond=condition, struct_cond=latent, prior=prior, | |
| x_T=latent, timesteps=1) | |
| qp_tensor = torch.full((images.shape[0],), qp, | |
| dtype=torch.float32, device=model.device) / 51.0 | |
| output = vae.decode(samples / model.scale_factor, encoder_features, | |
| prior_features, qp_tensor) | |
| outputs.append(output.float().cpu()) | |
| del posterior, encoder_features, latent, prior, prior_features | |
| del condition, samples, qp_tensor, output | |
| output = merge_patch_grid(images.shape, patches, outputs, metadata, | |
| device="cpu").to(model.device) | |
| if color_correction == "adain": | |
| output = adaptive_instance_normalization(output, images) | |
| elif color_correction == "wavelet": | |
| output = wavelet_reconstruction(output, images) | |
| output = torch.clamp((output + 1.0) / 2.0, 0.0, 1.0) | |
| if not torch.isfinite(output).all(): | |
| raise RuntimeError("Inference produced non-finite image values") | |
| return output[:, :, :height, :width].cpu().numpy().transpose(0, 2, 3, 1) * 255 | |
| print("Loading DiffCVE single-step model …", flush=True) | |
| text_conditions_path = hf_hub_download(MODEL_REPO, TEXT_CONDITIONS) | |
| checkpoint_path = hf_hub_download(MODEL_REPO, CHECKPOINT) | |
| config = OmegaConf.load(str(APP_ROOT / "configs" / "single_step.yaml")) | |
| params = config.model.params | |
| params.text_conditions_path = str(text_conditions_path) | |
| params.cond_stage_config = None | |
| params.text_prompts = None | |
| params.text_condition_qps = list(range(52)) # interpolate cached anchors to any QP 0..51 | |
| model = load_checkpoint(instantiate_from_config(config.model), str(checkpoint_path)).eval().to("cuda") | |
| model.denoiser = model.denoiser.merge_and_unload() | |
| model.eval().requires_grad_(False).to("cuda") | |
| model.register_schedule(timesteps=1000, linear_start=0.00085, linear_end=0.012) | |
| model.to("cuda") | |
| model.vae.decoder.fusion_w = 1.0 | |
| _missing = {47} - set(model.text_conditions.keys()) | |
| if _missing: | |
| raise RuntimeError(f"Missing text conditions for QPs: {sorted(_missing)}") | |
| print("DiffCVE model ready.", flush=True) | |
| def enhance(video, qp: int = 37, max_frames: int = DEFAULT_MAX_FRAMES, seed: int = 42, | |
| color_correction: str = "adain", | |
| progress: gr.Progress = gr.Progress(track_tqdm=False)): | |
| """Enhance a compressed video with DiffCVE. | |
| The uploaded video is first H.264-compressed at the chosen quantization | |
| parameter (QP), then restored with the diffusion enhancer using H.264 | |
| coding priors (motion vectors + residuals) and QP-conditioned text prompts. | |
| Args: | |
| video: Input video file (any common container; no audio required). | |
| qp: H.264 quantization parameter (0-51); higher means worse quality. | |
| max_frames: How many frames to enhance (each frame costs GPU time). | |
| seed: Random seed for latent sampling. | |
| color_correction: Post-hoc color alignment to the compressed input | |
| ("adain", "wavelet", or "nofix"). | |
| Returns: | |
| (enhanced video, compressed input video, status text) | |
| """ | |
| qp = int(qp) | |
| seed = int(seed) | |
| max_frames = int(max_frames) | |
| if not video: | |
| raise gr.Error("Please upload a video first.") | |
| if not 0 <= qp <= 51: | |
| raise gr.Error("QP must be between 0 and 51.") | |
| if max_frames < 1: | |
| max_frames = DEFAULT_MAX_FRAMES | |
| max_frames = min(max_frames, MAX_MAX_FRAMES) | |
| src = Path(video) | |
| width, height, fps, total_frames = probe_video(src) | |
| if total_frames < 1: | |
| raise gr.Error("Could not read any video frames.") | |
| fps = min(max(fps, 1.0), 60.0) | |
| frames = min(max_frames, total_frames) | |
| scale = None | |
| if max(width, height) > MAX_DIM: | |
| scale = MAX_DIM / max(width, height) | |
| if scale is not None: | |
| width = int(math.floor(width * scale / 16)) * 16 | |
| height = int(math.floor(height * scale / 16)) * 16 | |
| width += width % 2 | |
| height += height % 2 | |
| if width < 16 or height < 16: | |
| raise gr.Error("Video is too small after resizing.") | |
| with tempfile.TemporaryDirectory(prefix="diffcve-") as work_str: | |
| work = Path(work_str) | |
| yuv_path = work / "input.yuv" | |
| # 1. Decode + scale + trim the upload to raw yuv420p (the JM encoder's input). | |
| cmd = [FFMPEG, "-y", "-nostdin", "-hide_banner", "-loglevel", "error", | |
| "-i", str(src), | |
| "-frames:v", str(frames), | |
| "-vf", (f"scale={width}:{height}" if scale is not None else "null"), | |
| "-pix_fmt", "yuv420p", "-f", "rawvideo", str(yuv_path)] | |
| subprocess.run(cmd, capture_output=True, text=True, check=True) | |
| if not yuv_path.is_file() or yuv_path.stat().st_size < width * height * 3 // 2: | |
| raise gr.Error("Failed to decode the uploaded video.") | |
| # 2. Compress with the authors' JM encoder at the chosen constant QP. | |
| bitstream = work / "input.264" | |
| encode_h264(yuv_path, bitstream, width, height, frames, fps, qp) | |
| # 3. Stream-decode with coding priors and enhance segment by segment. | |
| decoded_dir, enhanced_dir = work / "decoded", work / "enhanced" | |
| decoded_dir.mkdir() | |
| enhanced_dir.mkdir() | |
| written = 0 | |
| n_segments = math.ceil(frames / SEGMENT_SIZE) | |
| with VideoStream(bitstream, decoder=CODEC_DIR / "decoder", ffmpeg=FFMPEG, | |
| fps=fps, expected_qp=qp, prefetch=5, prior_prefetch=32) as vid: | |
| seg_i = 0 | |
| for segment in iter_segments(vid, SEGMENT_SIZE, frames): | |
| images, residues, motions = segment.to_torch(model.device) | |
| seg_i += 1 | |
| progress((seg_i, n_segments), | |
| desc=f"Enhancing segment {seg_i}/{n_segments}") | |
| with torch.autocast("cuda"): | |
| out = enhance_segment(model, images, residues, motions, | |
| segment.qp, seed, color_correction) | |
| for i, frame in enumerate(segment.frames): | |
| Image.fromarray(out[i].astype(np.uint8)).save(enhanced_dir / f"f{written:05d}.png") | |
| Image.fromarray(frame.rgb).save(decoded_dir / f"f{written:05d}.png") | |
| written += 1 | |
| del images, residues, motions, out | |
| if not vid.completed: | |
| raise RuntimeError("Bitstream decoding did not complete validation") | |
| if written == 0: | |
| raise gr.Error("Enhancement produced no frames.") | |
| # 4. Mux both videos and move them out of the temp dir before it is wiped. | |
| final_dir = Path(tempfile.mkdtemp(prefix="diffcve-out-")) | |
| enhanced_mp4 = final_dir / f"enhanced_qp{qp}.mp4" | |
| decoded_mp4 = final_dir / f"compressed_qp{qp}.mp4" | |
| frames_to_mp4(enhanced_dir, enhanced_mp4, fps) | |
| frames_to_mp4(decoded_dir, decoded_mp4, fps) | |
| status = (f"Enhanced {written} frames at {width}×{height} @ {fps:.2f} fps, " | |
| f"QP {qp} · seed {seed} · color fix: {color_correction}") | |
| return str(enhanced_mp4), str(decoded_mp4), status | |
| EXAMPLES = [ | |
| [str(EXAMPLES_DIR / "ocean_waves_qp37.mp4"), 37], | |
| [str(EXAMPLES_DIR / "cat_qp47.mp4"), 47], | |
| [str(EXAMPLES_DIR / "jellyfish_qp32.mp4"), 32], | |
| ] | |
| CSS = """ | |
| #col-container { max-width: 1200px; margin: 0 auto; } | |
| .dark .gradio-container { color: var(--body-text-color); } | |
| """ | |
| with gr.Blocks(title="DiffCVE — Compressed Video Enhancement") as demo: | |
| gr.Markdown( | |
| """ | |
| # 🎬 DiffCVE: Diffusion-based Compressed Video Enhancement | |
| Upload a video, pick an H.264 quantization parameter (QP), and the | |
| single-step distilled DiffCVE model restores it — using the bitstream's | |
| own coding priors (motion vectors + prediction residuals), QP-conditioned | |
| degradation prompts, and a coding-prior-weighted VAE decoder (CPWF). | |
| *Paper: [DiffCVE: Diffusion-based Compressed Video Enhancement](https://huggingface.co/papers/2607.07195) | |
| · Code: [wqmaker/DiffCVE](https://github.com/wqmaker/DiffCVE) | |
| · Model: [wqmaker/DiffCVE](https://huggingface.co/wqmaker/DiffCVE)* | |
| """ | |
| ) | |
| with gr.Row(): | |
| with gr.Column(): | |
| video_in = gr.Video(label="Input video", sources=["upload"]) | |
| qp_slider = gr.Slider(0, 51, value=37, step=1, label="QP (compression severity)", | |
| info="Higher QP → stronger blocking artifacts to remove") | |
| enhance_btn = gr.Button("Enhance video", variant="primary") | |
| with gr.Column(): | |
| video_out = gr.Video(label="Enhanced video (DiffCVE)") | |
| video_cmp = gr.Video(label="Compressed input (QP applied)") | |
| status_text = gr.Textbox(label="Status", interactive=False) | |
| with gr.Accordion("Advanced options", open=False): | |
| with gr.Row(): | |
| frames_slider = gr.Slider(1, MAX_MAX_FRAMES, value=DEFAULT_MAX_FRAMES, step=1, | |
| label="Frames to enhance", | |
| info="Each frame costs GPU time; 10 frames ≈ 80 s") | |
| seed_number = gr.Number(value=42, label="Seed", precision=0) | |
| color_dropdown = gr.Dropdown(["adain", "wavelet", "nofix"], value="adain", | |
| label="Color correction", | |
| info="Aligns output colors with the compressed input") | |
| gr.Markdown( | |
| """ | |
| **Tip:** videos larger than 720 px on either edge are downscaled first. | |
| Keep clips short (≤ 10 frames) — diffusion restoration of each | |
| 5-frame segment is a heavy forward pass. | |
| """ | |
| ) | |
| gr.Examples( | |
| examples=EXAMPLES, | |
| inputs=[video_in, qp_slider], | |
| outputs=[video_out, video_cmp, status_text], | |
| fn=enhance, | |
| cache_examples=True, | |
| cache_mode="lazy", | |
| ) | |
| enhance_btn.click( | |
| fn=enhance, | |
| inputs=[video_in, qp_slider, frames_slider, seed_number, color_dropdown], | |
| outputs=[video_out, video_cmp, status_text], | |
| api_name="enhance", | |
| ) | |
| demo.launch(mcp_server=True, theme=gr.themes.Citrus(), css=CSS) | |