import os os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import spaces # noqa: E402 (must precede torch) import json # noqa: E402 import math # noqa: E402 import random # noqa: E402 import shutil # noqa: E402 import subprocess # noqa: E402 import sys # noqa: E402 import tempfile # noqa: E402 from pathlib import Path # noqa: E402 import numpy as np # noqa: E402 import torch # noqa: E402 from PIL import Image # noqa: E402 # torch 2.6+ defaults torch.load to weights_only=True, which rejects the # pickled numpy objects inside the authors' checkpoints. _orig_torch_load = torch.load torch.load = lambda *a, **k: _orig_torch_load(*a, **{**k, "weights_only": k.get("weights_only", False)}) # torch 2.9+ removed torch.onnx.is_in_onnx_export(); vendored code still calls it. if not hasattr(torch.onnx, "is_in_onnx_export"): torch.onnx.is_in_onnx_export = lambda: False import gradio as gr # noqa: E402 from huggingface_hub import hf_hub_download # noqa: E402 from omegaconf import OmegaConf # noqa: E402 APP_ROOT = Path(__file__).resolve().parent sys.path.insert(0, str(APP_ROOT)) from codec.processor import VideoStream # noqa: E402 from diffcve.utils import instantiate_from_config, load_checkpoint # noqa: E402 from scripts.color_correction import adaptive_instance_normalization, wavelet_reconstruction # noqa: E402 from scripts.stream_adapter import iter_segments # noqa: E402 from scripts.tiling import build_patch_grid, merge_patch_grid # noqa: E402 MODEL_REPO = "wqmaker/DiffCVE" CHECKPOINT = "single_step.ckpt" TEXT_CONDITIONS = "text_conditions.pt" CODEC_DIR = APP_ROOT / "codec" EXAMPLES_DIR = APP_ROOT / "examples" # The Hub's git upload does not preserve the executable bit, so re-assert it # for the bundled JM codec binaries (idempotent). for _binary in (CODEC_DIR / "encoder", CODEC_DIR / "decoder"): _binary.chmod(0o755) SEGMENT_SIZE = 5 # the checkpoint is trained on 5-frame segments DEFAULT_MAX_FRAMES = 10 MAX_MAX_FRAMES = 10 # 10 frames measured 76.8s on ZeroGPU; more would exceed the GPU budget MAX_DIM = 720 DEFAULT_DURATION_S = 90 # measured 76.8s for 10 frames on ZeroGPU + ~15% margin FFMPEG = shutil.which("ffmpeg") or "ffmpeg" FFPROBE = shutil.which("ffprobe") or "ffprobe" def seed_everything(seed): random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) def probe_video(path): """Return (width, height, fps, frames) of a video via ffprobe.""" out = subprocess.run( [FFPROBE, "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height,r_frame_rate,nb_frames", "-show_entries", "format=duration", "-of", "json", str(path)], capture_output=True, text=True, check=True).stdout info = json.loads(out) stream = info["streams"][0] width, height = int(stream["width"]), int(stream["height"]) num, den = stream["r_frame_rate"].split("/") fps = float(num) / float(den) duration = float(info.get("format", {}).get("duration") or 0) frames = int(stream.get("nb_frames") or 0) or int(round(duration * fps)) return width, height, fps, frames def encode_h264(yuv_path, bitstream_path, width, height, frames, fps, qp): """Encode raw yuv420p to constant-QP H.264 with the authors' JM encoder.""" cmd = [ str(CODEC_DIR / "encoder"), "-d", str(CODEC_DIR / "encoder.cfg"), "-p", f"InputFile={yuv_path}", "-p", f"OutputFile={bitstream_path}", "-p", f"SourceWidth={width}", "-p", f"SourceHeight={height}", "-p", f"FramesToBeEncoded={frames}", "-p", f"FrameRate={round(fps)}", "-p", "ProfileIDC=66", "-p", "LevelIDC=50", "-p", f"QPISlice={qp}", "-p", f"QPPSlice={qp}", "-p", "NumberReferenceFrames=5", "-p", "SearchRange=32", "-p", "IntraPeriod=0", "-p", "IDRPeriod=0", "-p", "NumberBFrames=0", "-p", "YUVFormat=1", "-p", "SourceBitDepthLuma=8", "-p", "SourceBitDepthChroma=8", "-p", "ReconFile=/dev/null", "-p", "TraceFile=/dev/null", "-p", "LeakyBucketParamFile=/dev/null", "-p", "StatsFile=/dev/null", ] result = subprocess.run(cmd, capture_output=True, text=True, timeout=600) if result.returncode != 0 or not Path(bitstream_path).is_file(): raise RuntimeError(f"H.264 encoding failed: {result.stderr[-2000:]}") def frames_to_mp4(frames_dir, out_path, fps): """Mux a directory of PNG frames into an mp4.""" subprocess.run( [FFMPEG, "-y", "-nostdin", "-hide_banner", "-loglevel", "error", "-framerate", str(fps), "-i", str(frames_dir / "f%05d.png"), "-c:v", "libx264", "-pix_fmt", "yuv420p", "-crf", "18", str(out_path)], capture_output=True, text=True, check=True) @torch.no_grad() def enhance_segment(model, images, residues, motions, qp, seed, color_correction): """Enhance one 5-frame segment — mirrors the reference scripts/video_inference.py.""" height, width = images.shape[-2:] if height < 720: rows, cols = 1, 1 pad_h, pad_w = (-height) % 64, (-width) % 64 elif height < 1080: rows, cols = 1, 2 pad_h, pad_w = (-height) % 64, 0 else: rows, cols = (2, 3) if height < 1600 else (3, 4) pad_h = pad_w = 0 if pad_h or pad_w: images, residues, motions = [ torch.nn.functional.pad(t, (0, pad_w, 0, pad_h), mode="reflect") for t in [images, residues, motions]] patches, metadata = build_patch_grid(*images.shape[-2:], rows, cols) vae = model.vae outputs = [] for hs, ws in patches: seed_everything(seed) image_patch = images[:, :, hs, ws] posterior, encoder_features = vae.encode(image_patch) latent = model.sample_latent(posterior) prior, prior_features = model.prior_encoder(residues[:, :, hs, ws], motions[:, :, hs, ws], return_features=True) condition = model.text_conditions[qp].to(model.device) samples = model.sample(cond=condition, struct_cond=latent, prior=prior, x_T=latent, timesteps=1) qp_tensor = torch.full((images.shape[0],), qp, dtype=torch.float32, device=model.device) / 51.0 output = vae.decode(samples / model.scale_factor, encoder_features, prior_features, qp_tensor) outputs.append(output.float().cpu()) del posterior, encoder_features, latent, prior, prior_features del condition, samples, qp_tensor, output output = merge_patch_grid(images.shape, patches, outputs, metadata, device="cpu").to(model.device) if color_correction == "adain": output = adaptive_instance_normalization(output, images) elif color_correction == "wavelet": output = wavelet_reconstruction(output, images) output = torch.clamp((output + 1.0) / 2.0, 0.0, 1.0) if not torch.isfinite(output).all(): raise RuntimeError("Inference produced non-finite image values") return output[:, :, :height, :width].cpu().numpy().transpose(0, 2, 3, 1) * 255 print("Loading DiffCVE single-step model …", flush=True) text_conditions_path = hf_hub_download(MODEL_REPO, TEXT_CONDITIONS) checkpoint_path = hf_hub_download(MODEL_REPO, CHECKPOINT) config = OmegaConf.load(str(APP_ROOT / "configs" / "single_step.yaml")) params = config.model.params params.text_conditions_path = str(text_conditions_path) params.cond_stage_config = None params.text_prompts = None params.text_condition_qps = list(range(52)) # interpolate cached anchors to any QP 0..51 model = load_checkpoint(instantiate_from_config(config.model), str(checkpoint_path)).eval().to("cuda") model.denoiser = model.denoiser.merge_and_unload() model.eval().requires_grad_(False).to("cuda") model.register_schedule(timesteps=1000, linear_start=0.00085, linear_end=0.012) model.to("cuda") model.vae.decoder.fusion_w = 1.0 _missing = {47} - set(model.text_conditions.keys()) if _missing: raise RuntimeError(f"Missing text conditions for QPs: {sorted(_missing)}") print("DiffCVE model ready.", flush=True) @spaces.GPU(duration=DEFAULT_DURATION_S) def enhance(video, qp: int = 37, max_frames: int = DEFAULT_MAX_FRAMES, seed: int = 42, color_correction: str = "adain", progress: gr.Progress = gr.Progress(track_tqdm=False)): """Enhance a compressed video with DiffCVE. The uploaded video is first H.264-compressed at the chosen quantization parameter (QP), then restored with the diffusion enhancer using H.264 coding priors (motion vectors + residuals) and QP-conditioned text prompts. Args: video: Input video file (any common container; no audio required). qp: H.264 quantization parameter (0-51); higher means worse quality. max_frames: How many frames to enhance (each frame costs GPU time). seed: Random seed for latent sampling. color_correction: Post-hoc color alignment to the compressed input ("adain", "wavelet", or "nofix"). Returns: (enhanced video, compressed input video, status text) """ qp = int(qp) seed = int(seed) max_frames = int(max_frames) if not video: raise gr.Error("Please upload a video first.") if not 0 <= qp <= 51: raise gr.Error("QP must be between 0 and 51.") if max_frames < 1: max_frames = DEFAULT_MAX_FRAMES max_frames = min(max_frames, MAX_MAX_FRAMES) src = Path(video) width, height, fps, total_frames = probe_video(src) if total_frames < 1: raise gr.Error("Could not read any video frames.") fps = min(max(fps, 1.0), 60.0) frames = min(max_frames, total_frames) scale = None if max(width, height) > MAX_DIM: scale = MAX_DIM / max(width, height) if scale is not None: width = int(math.floor(width * scale / 16)) * 16 height = int(math.floor(height * scale / 16)) * 16 width += width % 2 height += height % 2 if width < 16 or height < 16: raise gr.Error("Video is too small after resizing.") with tempfile.TemporaryDirectory(prefix="diffcve-") as work_str: work = Path(work_str) yuv_path = work / "input.yuv" # 1. Decode + scale + trim the upload to raw yuv420p (the JM encoder's input). cmd = [FFMPEG, "-y", "-nostdin", "-hide_banner", "-loglevel", "error", "-i", str(src), "-frames:v", str(frames), "-vf", (f"scale={width}:{height}" if scale is not None else "null"), "-pix_fmt", "yuv420p", "-f", "rawvideo", str(yuv_path)] subprocess.run(cmd, capture_output=True, text=True, check=True) if not yuv_path.is_file() or yuv_path.stat().st_size < width * height * 3 // 2: raise gr.Error("Failed to decode the uploaded video.") # 2. Compress with the authors' JM encoder at the chosen constant QP. bitstream = work / "input.264" encode_h264(yuv_path, bitstream, width, height, frames, fps, qp) # 3. Stream-decode with coding priors and enhance segment by segment. decoded_dir, enhanced_dir = work / "decoded", work / "enhanced" decoded_dir.mkdir() enhanced_dir.mkdir() written = 0 n_segments = math.ceil(frames / SEGMENT_SIZE) with VideoStream(bitstream, decoder=CODEC_DIR / "decoder", ffmpeg=FFMPEG, fps=fps, expected_qp=qp, prefetch=5, prior_prefetch=32) as vid: seg_i = 0 for segment in iter_segments(vid, SEGMENT_SIZE, frames): images, residues, motions = segment.to_torch(model.device) seg_i += 1 progress((seg_i, n_segments), desc=f"Enhancing segment {seg_i}/{n_segments}") with torch.autocast("cuda"): out = enhance_segment(model, images, residues, motions, segment.qp, seed, color_correction) for i, frame in enumerate(segment.frames): Image.fromarray(out[i].astype(np.uint8)).save(enhanced_dir / f"f{written:05d}.png") Image.fromarray(frame.rgb).save(decoded_dir / f"f{written:05d}.png") written += 1 del images, residues, motions, out if not vid.completed: raise RuntimeError("Bitstream decoding did not complete validation") if written == 0: raise gr.Error("Enhancement produced no frames.") # 4. Mux both videos and move them out of the temp dir before it is wiped. final_dir = Path(tempfile.mkdtemp(prefix="diffcve-out-")) enhanced_mp4 = final_dir / f"enhanced_qp{qp}.mp4" decoded_mp4 = final_dir / f"compressed_qp{qp}.mp4" frames_to_mp4(enhanced_dir, enhanced_mp4, fps) frames_to_mp4(decoded_dir, decoded_mp4, fps) status = (f"Enhanced {written} frames at {width}×{height} @ {fps:.2f} fps, " f"QP {qp} · seed {seed} · color fix: {color_correction}") return str(enhanced_mp4), str(decoded_mp4), status EXAMPLES = [ [str(EXAMPLES_DIR / "ocean_waves_qp37.mp4"), 37], [str(EXAMPLES_DIR / "cat_qp47.mp4"), 47], [str(EXAMPLES_DIR / "jellyfish_qp32.mp4"), 32], ] CSS = """ #col-container { max-width: 1200px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } """ with gr.Blocks(title="DiffCVE — Compressed Video Enhancement") as demo: gr.Markdown( """ # 🎬 DiffCVE: Diffusion-based Compressed Video Enhancement Upload a video, pick an H.264 quantization parameter (QP), and the single-step distilled DiffCVE model restores it — using the bitstream's own coding priors (motion vectors + prediction residuals), QP-conditioned degradation prompts, and a coding-prior-weighted VAE decoder (CPWF). *Paper: [DiffCVE: Diffusion-based Compressed Video Enhancement](https://huggingface.co/papers/2607.07195) · Code: [wqmaker/DiffCVE](https://github.com/wqmaker/DiffCVE) · Model: [wqmaker/DiffCVE](https://huggingface.co/wqmaker/DiffCVE)* """ ) with gr.Row(): with gr.Column(): video_in = gr.Video(label="Input video", sources=["upload"]) qp_slider = gr.Slider(0, 51, value=37, step=1, label="QP (compression severity)", info="Higher QP → stronger blocking artifacts to remove") enhance_btn = gr.Button("Enhance video", variant="primary") with gr.Column(): video_out = gr.Video(label="Enhanced video (DiffCVE)") video_cmp = gr.Video(label="Compressed input (QP applied)") status_text = gr.Textbox(label="Status", interactive=False) with gr.Accordion("Advanced options", open=False): with gr.Row(): frames_slider = gr.Slider(1, MAX_MAX_FRAMES, value=DEFAULT_MAX_FRAMES, step=1, label="Frames to enhance", info="Each frame costs GPU time; 10 frames ≈ 80 s") seed_number = gr.Number(value=42, label="Seed", precision=0) color_dropdown = gr.Dropdown(["adain", "wavelet", "nofix"], value="adain", label="Color correction", info="Aligns output colors with the compressed input") gr.Markdown( """ **Tip:** videos larger than 720 px on either edge are downscaled first. Keep clips short (≤ 10 frames) — diffusion restoration of each 5-frame segment is a heavy forward pass. """ ) gr.Examples( examples=EXAMPLES, inputs=[video_in, qp_slider], outputs=[video_out, video_cmp, status_text], fn=enhance, cache_examples=True, cache_mode="lazy", ) enhance_btn.click( fn=enhance, inputs=[video_in, qp_slider, frames_slider, seed_number, color_dropdown], outputs=[video_out, video_cmp, status_text], api_name="enhance", ) demo.launch(mcp_server=True, theme=gr.themes.Citrus(), css=CSS)