#!/usr/bin/env python3 """Gradio demo for InstructAV2AV: source video + instruction -> edited video.""" from __future__ import annotations import argparse import gc import logging import os import sys import uuid from collections import OrderedDict from pathlib import Path from threading import Lock from typing import Any, Callable REPO_ROOT = Path(__file__).resolve().parents[1] if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) # Some upstream Ovi modules resolve their default config relative to cwd. os.chdir(REPO_ROOT) import gradio as gr import torch from omegaconf import OmegaConf from ovi.distributed_comms.parallel_states import initialize_sequence_parallel_state from ovi.utils.av_edit_data import ( get_video_info, load_audio_array, load_video_array, snap_num_frames, to_audio_tensor, to_video_tensor, ) from ovi.utils.io_utils import save_video DEFAULT_CONFIG = REPO_ROOT / "ovi/configs/inference/inference_av_edit.yaml" DEFAULT_MODEL_DIR = REPO_ROOT / "ckpts/InstructAV2AV" DEFAULT_OUTPUT_DIR = REPO_ROOT / "outputs/demo" MODEL_SPECS = OrderedDict( { "general": { "label": "General Edit", "description": "Flexible editing of appearance, scenes, actions, speech, and sound.", "example": "Make the horse dark brown with a white saddle.", }, "insertion": { "label": "Content Insertion", "description": "Add a an object to the source video.", "example": "Add a dark vintage sedan driving from the right to the left.", }, "removal": { "label": "Content Removal", "description": "Remove an object from the video.", "example": "Remove the chipmunk standing on the stone surface among the peanuts.", }, "clone_id": { "label": "Identity Cloning", "description": "Preserve a person's visual identity during editing.", "example": "Keep the person‘s appearance, change the timbre to a man, and change the spoken words to I understand, but I think we need to consider..", }, "clone_voice": { "label": "Voice Cloning", "description": "Preserve the speaker's timbre during editing.", "example": "Keep the timbre, change ..., and change the spoken words to I came here to tell you that you should to go..", }, "clone_id_voice": { "label": "Identity + Voice Cloning", "description": "Preserve both visual identity and timbre.", "example": "Keep the person’s identity and change the spoken words to This is more than just art, it’s a statement..", }, } ) CSS = """ .gradio-container { max-width: 1200px !important; margin: 0 auto !important; padding: 24px 20px 32px !important; font-family: Arial, Helvetica, sans-serif !important; } #page-header { margin-bottom: 18px !important; } #page-header h1 { margin-bottom: 4px !important; font-size: 1.7rem !important; } #page-header p { margin: 0 !important; color: var(--body-text-color-subdued); } #video-row { gap: 18px !important; align-items: start !important; } #source-column, #result-column { gap: 8px !important; min-width: 0 !important; } #source-video, #result-video { height: auto !important; margin: 0 !important; aspect-ratio: 16 / 9; } #source-video [data-testid="video"], #result-video [data-testid="video"] { width: 100% !important; height: auto !important; aspect-ratio: 16 / 9; overflow: hidden; } #source-video video, #result-video video { width: 100% !important; height: 100% !important; object-fit: contain !important; background: #000 !important; border-radius: 4px !important; } #settings-panel { margin-top: 18px !important; padding: 16px !important; border: 1px solid var(--border-color-primary) !important; border-radius: 6px !important; box-shadow: none !important; } #settings-title { margin: 0 0 6px !important; } #settings-title h2 { margin: 0 !important; font-size: 1.1rem !important; } #settings-row { align-items: start !important; gap: 14px !important; } #model-help { margin: 0 !important; padding: 2px !important; color: var(--body-text-color-subdued); font-size: 0.83rem; } #model-help .prose { padding: 0 !important; } #model-help p { margin: 0 0 4px !important; } #model-help p:last-child { margin-bottom: 0 !important; } #advanced-settings { margin-top: 6px !important; } #action-row { gap: 10px !important; justify-content: flex-end !important; } #generate-button { min-height: 40px; border-radius: 4px; } #clear-button { min-height: 40px; border-radius: 4px; } @media (max-width: 760px) { .gradio-container { padding: 10px !important; } #video-row { flex-direction: column !important; } #source-column, #result-column { width: 100% !important; } #video-row, #settings-row { gap: 10px !important; } #settings-panel { padding: 12px !important; } } """ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--config-file", default=str(DEFAULT_CONFIG)) parser.add_argument("--model-dir", default=str(DEFAULT_MODEL_DIR)) parser.add_argument("--ckpt-dir", default=None, help="Base Ovi checkpoint directory override.") parser.add_argument("--output-dir", default=str(DEFAULT_OUTPUT_DIR)) parser.add_argument("--device", type=int, default=0) parser.add_argument("--server-name", default="127.0.0.1") parser.add_argument("--server-port", type=int, default=7860) parser.add_argument("--share", action="store_true") parser.add_argument("--inbrowser", action="store_true") parser.add_argument("--no-cpu-offload", action="store_true") parser.add_argument("--max-queue-size", type=int, default=8) return parser.parse_args() def resolve_path(value: str | Path, base: Path = REPO_ROOT) -> Path: path = Path(value).expanduser() if not path.is_absolute(): path = base / path return path.resolve() def discover_checkpoints(model_dir: Path) -> dict[str, Path]: checkpoints = {key: model_dir / f"{key}.safetensors" for key in MODEL_SPECS} missing = [str(path) for path in checkpoints.values() if not path.is_file()] if missing: raise FileNotFoundError( "Missing InstructAV2AV checkpoints:\n" + "\n".join(f"- {path}" for path in missing) ) return checkpoints def uploaded_path(value: Any) -> Path: if value is None: raise gr.Error("Please upload a source video first.") if isinstance(value, (str, Path)): path = Path(value) elif isinstance(value, dict): raw_path = value.get("path") or value.get("name") if not raw_path: raise gr.Error("The uploaded video could not be read.") path = Path(raw_path) else: raise gr.Error(f"Unsupported video input type: {type(value).__name__}") path = path.expanduser().resolve() if not path.is_file(): raise gr.Error(f"The uploaded video does not exist: {path}") return path class DemoRuntime: """Keep shared modules alive and swap only the selected fusion checkpoint.""" def __init__( self, args: argparse.Namespace, checkpoint_resolver: Callable[[str, Any], Path] | None = None, ): self.args = args self.config_path = resolve_path(args.config_file) self.model_dir = resolve_path(args.model_dir) self.output_dir = resolve_path(args.output_dir) self.checkpoint_resolver = checkpoint_resolver self.checkpoints = ( {} if checkpoint_resolver is not None else discover_checkpoints(self.model_dir) ) self.config = self._load_config() self.engine: Any | None = None self.active_model: str | None = None self.lock = Lock() self.output_dir.mkdir(parents=True, exist_ok=True) def _load_config(self): if not self.config_path.is_file(): raise FileNotFoundError(f"Inference config not found: {self.config_path}") config = OmegaConf.load(self.config_path) ckpt_dir = self.args.ckpt_dir or config.get("ckpt_dir", "./ckpts") config.ckpt_dir = str(resolve_path(ckpt_dir)) config.av2av_edit = True config.has_video = True config.has_audio = True config.mode = "t2v" config.sp_size = 1 config.cpu_offload = not self.args.no_cpu_offload return config def _validate_cuda(self) -> None: if not torch.cuda.is_available(): raise RuntimeError("InstructAV2AV inference requires a CUDA GPU.") device_count = torch.cuda.device_count() if self.args.device < 0 or self.args.device >= device_count: raise RuntimeError( f"CUDA device {self.args.device} is unavailable; " f"found {device_count} GPU(s)." ) def get_engine(self, model_key: str, progress: gr.Progress) -> Any: if model_key not in MODEL_SPECS: raise ValueError(f"Unknown model: {model_key}") self._validate_cuda() # Keep the web page startup light; import the large model stack on first use. from ovi.ovi_fusion_engine import OviFusionEngine from ovi.utils.model_loading_utils import load_fusion_checkpoint if self.checkpoint_resolver is None: checkpoint = self.checkpoints[model_key] else: checkpoint = Path(self.checkpoint_resolver(model_key, progress)).resolve() if not checkpoint.is_file(): raise FileNotFoundError(f"Editing checkpoint not found: {checkpoint}") if self.engine is None: progress(0.08, desc=f"Loading {MODEL_SPECS[model_key]['label']} model") torch.cuda.set_device(self.args.device) initialize_sequence_parallel_state(1) config = OmegaConf.create(OmegaConf.to_container(self.config, resolve=True)) config.finetune_path = str(checkpoint) self.engine = OviFusionEngine( config=config, device=self.args.device, target_dtype=torch.bfloat16, ).eval() self.active_model = model_key elif self.active_model != model_key: progress(0.08, desc=f"Switching to {MODEL_SPECS[model_key]['label']}…") # All six checkpoints share one architecture, so the T5 and VAEs stay loaded. self.active_model = None self.engine.model = self.engine.model.to("cpu") torch.cuda.empty_cache() load_fusion_checkpoint(self.engine.model, str(checkpoint), from_meta=False) if not self.engine.cpu_offload: self.engine.model = self.engine.model.to(device=self.args.device) self.engine.eval() self.active_model = model_key gc.collect() return self.engine def generate( self, video_value: Any, instruction: str, model_key: str, seed: float, sample_steps: float, video_guidance_scale: float, audio_guidance_scale: float, progress: gr.Progress = gr.Progress(), ) -> str: video_path = uploaded_path(video_value) instruction = (instruction or "").strip() if not instruction: raise gr.Error("Please enter an editing instruction.") if model_key not in MODEL_SPECS: raise gr.Error("Please select a valid model.") sample_steps = int(sample_steps) seed = int(seed) if not 1 <= sample_steps <= 100: raise gr.Error("Sampling steps must be between 1 and 100.") try: with self.lock: engine = self.get_engine(model_key, progress) progress(0.18, desc="Preparing the source video and audio…") fps, total_frames = get_video_info(video_path) configured_frames = int(self.config.get("num_frames", total_frames)) num_frames = snap_num_frames(min(total_frames, configured_frames)) frame_size = list(self.config.get("video_frame_height_width", [704, 1280])) video, _ = load_video_array( video_path, num_frames=num_frames, height=int(frame_size[0]), width=int(frame_size[1]), max_pixels=int(frame_size[0]) * int(frame_size[1]), ) sample_rate = int(self.config.get("audio_sample_rate", 16000)) audio_samples = max(1, round(num_frames / fps * sample_rate)) try: audio = load_audio_array( video_path, sample_rate=sample_rate, num_samples=audio_samples, ) except Exception as exc: raise RuntimeError( "The video's audio track could not be read. " "Please upload a video that contains audio." ) from exc input_video = to_video_tensor(video, engine.device, engine.target_dtype) input_audio = to_audio_tensor(audio, engine.device) del video, audio progress(0.28, desc="Generating the edit. This may take several minutes…") torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) generated = engine.generate( text_prompt=instruction, image_path=None, video_frame_height_width=frame_size, seed=seed, solver_name=str(self.config.get("solver_name", "unipc")), sample_steps=sample_steps, shift=float(self.config.get("shift", 5.0)), video_guidance_scale=float(video_guidance_scale), audio_guidance_scale=float(audio_guidance_scale), slg_layer=int(self.config.get("slg_layer", 11)), video_negative_prompt=str(self.config.get("video_negative_prompt", "")), audio_negative_prompt=str(self.config.get("audio_negative_prompt", "")), input_video=input_video, input_audio=input_audio, ) del input_video, input_audio if generated is None: raise RuntimeError( "Generation failed. Check the terminal log for details." ) generated_video, generated_audio, _ = generated if generated_video is None or generated_audio is None: raise RuntimeError("The model returned an incomplete audio-video result.") progress(0.92, desc="Encoding the edited video…") output_path = self.output_dir / f"{model_key}_{uuid.uuid4().hex[:12]}.mp4" save_video( str(output_path), generated_video, generated_audio, sample_rate=sample_rate, fps=fps, ) del generated_video, generated_audio, generated gc.collect() torch.cuda.empty_cache() progress(1.0, desc="Edit complete") return str(output_path) except gr.Error: raise except Exception as exc: logging.exception("InstructAV2AV demo generation failed") raise gr.Error(str(exc)) from exc def warmup( self, model_key: str = "general", progress: gr.Progress = gr.Progress(), ) -> str: """Initialize the default engine before the first edit request.""" try: with self.lock: self.get_engine(model_key, progress) return f"✅ {MODEL_SPECS[model_key]['label']} is ready on ZeroGPU." except Exception as exc: logging.exception("InstructAV2AV demo warmup failed") return f"⚠️ Model warmup did not finish: {exc}" def model_help(model_key: str) -> str: spec = MODEL_SPECS.get(model_key, MODEL_SPECS["general"]) return f"{spec['description']} \n**Example:** `{spec['example']}`" def build_demo( runtime: DemoRuntime, generate_fn: Callable[..., str] | None = None, warmup_fn: Callable[..., str] | None = None, zero_gpu_fn: Callable[..., str] | None = None, ) -> gr.Blocks: choices = [(spec["label"], key) for key, spec in MODEL_SPECS.items()] config = runtime.config with gr.Blocks(title="InstructAV2AV Demo") as demo: warmup_operation = gr.State("warmup") generate_operation = gr.State("generate") gr.Markdown( "# InstructAV2AV\nUpload a video with audio, choose an editing type, provide the edit instruction, and generate the result. The default General model is prepared when the Space starts.", elem_id="page-header", ) model_status = gr.Markdown( "⏳ Preparing the default General model…", elem_id="model-status", ) with gr.Row(equal_height=True, elem_id="video-row"): with gr.Column(scale=1, min_width=0, elem_id="source-column"): source_video = gr.Video( label="Source video", sources=["upload"], include_audio=True, elem_id="source-video", elem_classes=["video-card"], ) with gr.Column(scale=1, min_width=0, elem_id="result-column"): result_video = gr.Video( label="Edited video", format="mp4", interactive=False, elem_id="result-video", elem_classes=["video-card"], ) with gr.Group(elem_id="settings-panel"): gr.Markdown("## Edit settings", elem_id="settings-title") with gr.Row(elem_id="settings-row"): with gr.Column(scale=1, min_width=260): model_choice = gr.Dropdown( choices=choices, value="general", label="Editing type", allow_custom_value=False, ) model_description = gr.Markdown( model_help("general"), elem_id="model-help" ) instruction = gr.Textbox( label="Editing instruction", placeholder="Example: Change the man into a young woman with brown hair, wearing a gray blazer, and saying, I really think we should give it another chance..", info=( "For speech editing, wrap the spoken text with and ." ), lines=4, max_lines=8, scale=2, min_width=320, ) with gr.Accordion("Advanced settings", open=False, elem_id="advanced-settings"): with gr.Row(): seed = gr.Number( value=int(config.get("seed", 103)), label="Seed", precision=0, ) sample_steps = gr.Slider( minimum=1, maximum=100, value=int(config.get("sample_steps", 50)), step=1, label="Sampling steps", ) with gr.Row(): video_guidance = gr.Slider( minimum=0, maximum=10, value=float(config.get("video_guidance_scale", 4.0)), step=0.1, label="Video guidance", ) audio_guidance = gr.Slider( minimum=0, maximum=10, value=float(config.get("audio_guidance_scale", 3.0)), step=0.1, label="Audio guidance", ) with gr.Row(elem_id="action-row"): clear_button = gr.Button( "Clear", variant="secondary", elem_id="clear-button" ) generate_button = gr.Button( "Generate edit", variant="primary", elem_id="generate-button", ) model_choice.change( model_help, inputs=model_choice, outputs=model_description, queue=False, ) generation_inputs = [ source_video, instruction, model_choice, seed, sample_steps, video_guidance, audio_guidance, ] generate_button.click( zero_gpu_fn or generate_fn or runtime.generate, inputs=( [generate_operation, *generation_inputs] if zero_gpu_fn is not None else generation_inputs ), outputs=result_video, concurrency_limit=1, concurrency_id="instructav2av-generation", api_name="edit_video", ) clear_button.click( lambda: ( None, "", None, ), outputs=[source_video, instruction, result_video], queue=False, ) if zero_gpu_fn is not None: demo.load( zero_gpu_fn, inputs=[warmup_operation, *generation_inputs], outputs=model_status, concurrency_limit=1, concurrency_id="instructav2av-generation", ) elif warmup_fn is not None: demo.load( warmup_fn, outputs=model_status, concurrency_limit=1, concurrency_id="instructav2av-generation", ) return demo def main() -> None: args = parse_args() logging.basicConfig( level=logging.INFO, format="[%(asctime)s] %(levelname)s: %(message)s", ) runtime = DemoRuntime(args) demo = build_demo(runtime) demo.queue(max_size=args.max_queue_size, default_concurrency_limit=1) demo.launch( server_name=args.server_name, server_port=args.server_port, share=args.share, inbrowser=args.inbrowser, allowed_paths=[str(runtime.output_dir)], show_error=True, theme=gr.themes.Default(), css=CSS, ) if __name__ == "__main__": main()