| """Modified from https://github.com/guoyww/AnimateDiff/blob/main/app.py |
| """ |
| import os |
| import random |
|
|
| import cv2 |
| import gradio as gr |
| import numpy as np |
| import torch |
| from PIL import Image |
| from safetensors import safe_open |
|
|
| from ..data.bucket_sampler import ASPECT_RATIO_512, get_closest_ratio |
| from ..models import (AutoencoderKLCogVideoX, CogVideoXTransformer3DModel, |
| T5EncoderModel, T5Tokenizer) |
| from ..pipeline import (CogVideoXFunControlPipeline, |
| CogVideoXFunInpaintPipeline, CogVideoXFunPipeline) |
| from ..utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, |
| convert_weight_dtype_wrapper) |
| from ..utils.lora_utils import merge_lora, unmerge_lora |
| from ..utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, timer, |
| get_video_to_video_latent, save_videos_grid) |
| from .controller import (Fun_Controller, Fun_Controller_Client, |
| all_cheduler_dict, css, ddpm_scheduler_dict, |
| flow_scheduler_dict, gradio_version, |
| gradio_version_is_above_4) |
| from .ui import (create_cfg_and_seedbox, |
| create_fake_finetune_models_checkpoints, |
| create_fake_height_width, create_fake_model_checkpoints, |
| create_fake_model_type, create_finetune_models_checkpoints, |
| create_generation_method, |
| create_generation_methods_and_video_length, |
| create_height_width, create_model_checkpoints, |
| create_model_type, create_prompts, create_samplers, |
| create_ui_outputs) |
| from ..dist import set_multi_gpus_devices, shard_model |
|
|
|
|
| class CogVideoXFunController(Fun_Controller): |
| def update_diffusion_transformer(self, diffusion_transformer_dropdown): |
| print(f"Update diffusion transformer: {diffusion_transformer_dropdown}") |
| self.diffusion_transformer_dropdown = diffusion_transformer_dropdown |
| if diffusion_transformer_dropdown == "none": |
| return gr.update() |
| self.vae = AutoencoderKLCogVideoX.from_pretrained( |
| diffusion_transformer_dropdown, |
| subfolder="vae", |
| ).to(self.weight_dtype) |
|
|
| |
| self.transformer = CogVideoXTransformer3DModel.from_pretrained( |
| diffusion_transformer_dropdown, |
| subfolder="transformer", |
| low_cpu_mem_usage=True, |
| ).to(self.weight_dtype) |
| |
| |
| tokenizer = T5Tokenizer.from_pretrained( |
| diffusion_transformer_dropdown, subfolder="tokenizer" |
| ) |
| text_encoder = T5EncoderModel.from_pretrained( |
| diffusion_transformer_dropdown, subfolder="text_encoder", torch_dtype=self.weight_dtype |
| ) |
| |
| |
| if self.model_type == "Inpaint": |
| if self.transformer.config.in_channels != self.vae.config.latent_channels: |
| self.pipeline = CogVideoXFunInpaintPipeline( |
| tokenizer=tokenizer, |
| text_encoder=text_encoder, |
| vae=self.vae, |
| transformer=self.transformer, |
| scheduler=self.scheduler_dict[list(self.scheduler_dict.keys())[0]].from_pretrained(diffusion_transformer_dropdown, subfolder="scheduler"), |
| ) |
| else: |
| self.pipeline = CogVideoXFunPipeline( |
| tokenizer=tokenizer, |
| text_encoder=text_encoder, |
| vae=self.vae, |
| transformer=self.transformer, |
| scheduler=self.scheduler_dict[list(self.scheduler_dict.keys())[0]].from_pretrained(diffusion_transformer_dropdown, subfolder="scheduler"), |
| ) |
| else: |
| self.pipeline = CogVideoXFunControlPipeline( |
| diffusion_transformer_dropdown, |
| vae=self.vae, |
| transformer=self.transformer, |
| scheduler=self.scheduler_dict[list(self.scheduler_dict.keys())[0]].from_pretrained(diffusion_transformer_dropdown, subfolder="scheduler"), |
| torch_dtype=self.weight_dtype |
| ) |
|
|
| if self.ulysses_degree > 1 or self.ring_degree > 1: |
| from functools import partial |
| self.transformer.enable_multi_gpus_inference() |
| if self.fsdp_dit: |
| shard_fn = partial(shard_model, device_id=self.device, param_dtype=self.weight_dtype) |
| self.pipeline.transformer = shard_fn(self.pipeline.transformer) |
| print("Add FSDP DIT") |
| if self.fsdp_text_encoder: |
| shard_fn = partial(shard_model, device_id=self.device, param_dtype=self.weight_dtype) |
| self.pipeline.text_encoder = shard_fn(self.pipeline.text_encoder) |
| print("Add FSDP TEXT ENCODER") |
|
|
| if self.compile_dit: |
| for i in range(len(self.pipeline.transformer.transformer_blocks)): |
| self.pipeline.transformer.transformer_blocks[i] = torch.compile(self.pipeline.transformer.transformer_blocks[i]) |
| print("Add Compile") |
|
|
| if self.GPU_memory_mode == "sequential_cpu_offload": |
| self.pipeline.enable_sequential_cpu_offload(device=self.device) |
| elif self.GPU_memory_mode == "model_cpu_offload_and_qfloat8": |
| convert_model_weight_to_float8(self.pipeline.transformer, exclude_module_name=[], device=self.device) |
| convert_weight_dtype_wrapper(self.pipeline.transformer, self.weight_dtype) |
| self.pipeline.enable_model_cpu_offload(device=self.device) |
| elif self.GPU_memory_mode == "model_cpu_offload": |
| self.pipeline.enable_model_cpu_offload(device=self.device) |
| elif self.GPU_memory_mode == "model_full_load_and_qfloat8": |
| convert_model_weight_to_float8(self.pipeline.transformer, exclude_module_name=[], device=self.device) |
| convert_weight_dtype_wrapper(self.pipeline.transformer, self.weight_dtype) |
| self.pipeline.to(self.device) |
| else: |
| self.pipeline.to(self.device) |
| print("Update diffusion transformer done") |
| return gr.update() |
|
|
| @timer |
| def generate( |
| self, |
| diffusion_transformer_dropdown, |
| base_model_dropdown, |
| lora_model_dropdown, |
| lora_alpha_slider, |
| prompt_textbox, |
| negative_prompt_textbox, |
| sampler_dropdown, |
| sample_step_slider, |
| resize_method, |
| width_slider, |
| height_slider, |
| base_resolution, |
| generation_method, |
| length_slider, |
| overlap_video_length, |
| partial_video_length, |
| cfg_scale_slider, |
| start_image, |
| end_image, |
| validation_video, |
| validation_video_mask, |
| control_video, |
| denoise_strength, |
| seed_textbox, |
| ref_image = None, |
| enable_teacache = None, |
| teacache_threshold = None, |
| num_skip_start_steps = None, |
| teacache_offload = None, |
| cfg_skip_ratio = None, |
| enable_riflex = None, |
| riflex_k = None, |
| base_model_2_dropdown=None, |
| lora_model_2_dropdown=None, |
| fps = None, |
| is_api = False, |
| ): |
| self.clear_cache() |
|
|
| print(f"Input checking.") |
| _, comment = self.input_check( |
| resize_method, generation_method, start_image, end_image, validation_video,control_video, is_api |
| ) |
| print(f"Input checking down") |
| if comment != "OK": |
| return "", comment |
| is_image = True if generation_method == "Image Generation" else False |
|
|
| if self.base_model_path != base_model_dropdown: |
| self.update_base_model(base_model_dropdown) |
|
|
| if self.lora_model_path != lora_model_dropdown: |
| self.update_lora_model(lora_model_dropdown) |
|
|
| print(f"Load scheduler.") |
| self.pipeline.scheduler = self.scheduler_dict[sampler_dropdown].from_config(self.pipeline.scheduler.config) |
| print(f"Load scheduler down.") |
|
|
| if resize_method == "Resize according to Reference": |
| print(f"Calculate height and width according to Reference.") |
| height_slider, width_slider = self.get_height_width_from_reference( |
| base_resolution, start_image, validation_video, control_video, |
| ) |
|
|
| if self.lora_model_path != "none": |
| print(f"Merge Lora.") |
| self.pipeline = merge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) |
| print(f"Merge Lora done.") |
|
|
| if fps is None: |
| fps = 8 |
|
|
| print(f"Generate seed.") |
| if int(seed_textbox) != -1 and seed_textbox != "": torch.manual_seed(int(seed_textbox)) |
| else: seed_textbox = np.random.randint(0, 1e10) |
| generator = torch.Generator(device=self.device).manual_seed(int(seed_textbox)) |
| print(f"Generate seed done.") |
| |
| try: |
| print(f"Generation.") |
| if self.model_type == "Inpaint": |
| if self.transformer.config.in_channels != self.vae.config.latent_channels: |
| if generation_method == "Long Video Generation": |
| if validation_video is not None: |
| raise gr.Error(f"Video to Video is not Support Long Video Generation now.") |
| init_frames = 0 |
| last_frames = init_frames + partial_video_length |
| while init_frames < length_slider: |
| if last_frames >= length_slider: |
| _partial_video_length = length_slider - init_frames |
| _partial_video_length = int((_partial_video_length - 1) // self.vae.config.temporal_compression_ratio * self.vae.config.temporal_compression_ratio) + 1 |
| |
| if _partial_video_length <= 0: |
| break |
| else: |
| _partial_video_length = partial_video_length |
|
|
| if last_frames >= length_slider: |
| input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=_partial_video_length, sample_size=(height_slider, width_slider)) |
| else: |
| input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, None, video_length=_partial_video_length, sample_size=(height_slider, width_slider)) |
|
|
| with torch.no_grad(): |
| sample = self.pipeline( |
| prompt_textbox, |
| negative_prompt = negative_prompt_textbox, |
| num_inference_steps = sample_step_slider, |
| guidance_scale = cfg_scale_slider, |
| width = width_slider, |
| height = height_slider, |
| num_frames = _partial_video_length, |
| generator = generator, |
|
|
| video = input_video, |
| mask_video = input_video_mask, |
| strength = 1, |
| ).videos |
| |
| if init_frames != 0: |
| mix_ratio = torch.from_numpy( |
| np.array([float(_index) / float(overlap_video_length) for _index in range(overlap_video_length)], np.float32) |
| ).unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) |
| |
| new_sample[:, :, -overlap_video_length:] = new_sample[:, :, -overlap_video_length:] * (1 - mix_ratio) + \ |
| sample[:, :, :overlap_video_length] * mix_ratio |
| new_sample = torch.cat([new_sample, sample[:, :, overlap_video_length:]], dim = 2) |
|
|
| sample = new_sample |
| else: |
| new_sample = sample |
|
|
| if last_frames >= length_slider: |
| break |
|
|
| start_image = [ |
| Image.fromarray( |
| (sample[0, :, _index].transpose(0, 1).transpose(1, 2) * 255).numpy().astype(np.uint8) |
| ) for _index in range(-overlap_video_length, 0) |
| ] |
|
|
| init_frames = init_frames + _partial_video_length - overlap_video_length |
| last_frames = init_frames + _partial_video_length |
| else: |
| if validation_video is not None: |
| input_video, input_video_mask, ref_image, clip_image = get_video_to_video_latent(validation_video, length_slider if not is_image else 1, sample_size=(height_slider, width_slider), validation_video_mask=validation_video_mask, fps=fps) |
| strength = denoise_strength |
| else: |
| input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, length_slider if not is_image else 1, sample_size=(height_slider, width_slider)) |
| strength = 1 |
|
|
| sample = self.pipeline( |
| prompt_textbox, |
| negative_prompt = negative_prompt_textbox, |
| num_inference_steps = sample_step_slider, |
| guidance_scale = cfg_scale_slider, |
| width = width_slider, |
| height = height_slider, |
| num_frames = length_slider if not is_image else 1, |
| generator = generator, |
|
|
| video = input_video, |
| mask_video = input_video_mask, |
| strength = strength, |
| ).videos |
| else: |
| sample = self.pipeline( |
| prompt_textbox, |
| negative_prompt = negative_prompt_textbox, |
| num_inference_steps = sample_step_slider, |
| guidance_scale = cfg_scale_slider, |
| width = width_slider, |
| height = height_slider, |
| num_frames = length_slider if not is_image else 1, |
| generator = generator |
| ).videos |
| else: |
| input_video, input_video_mask, ref_image, clip_image = get_video_to_video_latent(control_video, length_slider if not is_image else 1, sample_size=(height_slider, width_slider), fps=fps) |
|
|
| sample = self.pipeline( |
| prompt_textbox, |
| negative_prompt = negative_prompt_textbox, |
| num_inference_steps = sample_step_slider, |
| guidance_scale = cfg_scale_slider, |
| width = width_slider, |
| height = height_slider, |
| num_frames = length_slider if not is_image else 1, |
| generator = generator, |
|
|
| control_video = input_video, |
| ).videos |
| except Exception as e: |
| self.auto_model_clear_cache(self.pipeline.transformer) |
| self.auto_model_clear_cache(self.pipeline.text_encoder) |
| self.auto_model_clear_cache(self.pipeline.vae) |
| self.clear_cache() |
| |
| print(f"Error. error information is {str(e)}") |
| if self.lora_model_path != "none": |
| self.pipeline = unmerge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) |
| if is_api: |
| return "", f"Error. error information is {str(e)}" |
| else: |
| return gr.update(), gr.update(), f"Error. error information is {str(e)}" |
|
|
| self.clear_cache() |
| |
| if self.lora_model_path != "none": |
| print(f"Unmerge Lora.") |
| self.pipeline = unmerge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) |
| print(f"Unmerge Lora done.") |
|
|
| print(f"Saving outputs.") |
| save_sample_path = self.save_outputs( |
| is_image, length_slider, sample, fps=fps |
| ) |
| print(f"Saving outputs done.") |
|
|
| if is_image or length_slider == 1: |
| if is_api: |
| return save_sample_path, "Success" |
| else: |
| if gradio_version_is_above_4: |
| return gr.Image(value=save_sample_path, visible=True), gr.Video(value=None, visible=False), "Success" |
| else: |
| return gr.Image.update(value=save_sample_path, visible=True), gr.Video.update(value=None, visible=False), "Success" |
| else: |
| if is_api: |
| return save_sample_path, "Success" |
| else: |
| if gradio_version_is_above_4: |
| return gr.Image(visible=False, value=None), gr.Video(value=save_sample_path, visible=True), "Success" |
| else: |
| return gr.Image.update(visible=False, value=None), gr.Video.update(value=save_sample_path, visible=True), "Success" |
|
|
| CogVideoXFunController_Host = CogVideoXFunController |
| CogVideoXFunController_Client = Fun_Controller_Client |
|
|
| def ui(GPU_memory_mode, scheduler_dict, compile_dit, weight_dtype, savedir_sample=None): |
| controller = CogVideoXFunController( |
| GPU_memory_mode, scheduler_dict, model_name=None, model_type="Inpaint", |
| compile_dit=compile_dit, |
| weight_dtype=weight_dtype, savedir_sample=savedir_sample, |
| ) |
|
|
| with gr.Blocks(css=css) as demo: |
| gr.Markdown( |
| """ |
| # CogVideoX-Fun: |
| |
| A CogVideoX with more flexible generation conditions, capable of producing videos of different resolutions, around 6 seconds, and fps 8 (frames 1 to 49), as well as image generated videos. |
| |
| [Github](https://github.com/aigc-apps/CogVideoX-Fun/) |
| """ |
| ) |
| with gr.Column(variant="panel"): |
| model_type = create_model_type(visible=True) |
| diffusion_transformer_dropdown, diffusion_transformer_refresh_button = \ |
| create_model_checkpoints(controller, visible=True) |
| base_model_dropdown, lora_model_dropdown, lora_alpha_slider, personalized_refresh_button = \ |
| create_finetune_models_checkpoints(controller, visible=True) |
|
|
| with gr.Column(variant="panel"): |
| prompt_textbox, negative_prompt_textbox = create_prompts() |
|
|
| with gr.Row(): |
| with gr.Column(): |
| sampler_dropdown, sample_step_slider = create_samplers(controller) |
|
|
| resize_method, width_slider, height_slider, base_resolution = create_height_width( |
| default_height = 384, default_width = 672, maximum_height = 1344, |
| maximum_width = 1344, |
| ) |
| gr.Markdown( |
| """ |
| V1.0 and V1.1 support up to 49 frames of video generation, while V1.5 supports up to 85 frames. |
| (V1.0和V1.1支持最大49帧视频生成，V1.5支持最大85帧视频生成。) |
| """ |
| ) |
| generation_method, length_slider, overlap_video_length, partial_video_length = \ |
| create_generation_methods_and_video_length( |
| ["Video Generation", "Image Generation", "Long Video Generation"], |
| default_video_length=49, |
| maximum_video_length=85, |
| ) |
| image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( |
| ["Text to Video (文本到视频)", "Image to Video (图片到视频)", "Video to Video (视频到视频)", "Video Control (视频控制)"], prompt_textbox |
| ) |
| cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) |
|
|
| generate_button = gr.Button(value="Generate (生成)", variant='primary') |
| |
| result_image, result_video, infer_progress = create_ui_outputs() |
|
|
| model_type.change( |
| fn=controller.update_model_type, |
| inputs=[model_type], |
| outputs=[] |
| ) |
|
|
| def upload_generation_method(generation_method): |
| if generation_method == "Video Generation": |
| return [gr.update(visible=True, maximum=85, value=49, interactive=True), gr.update(visible=False), gr.update(visible=False)] |
| elif generation_method == "Image Generation": |
| return [gr.update(minimum=1, maximum=1, value=1, interactive=False), gr.update(visible=False), gr.update(visible=False)] |
| else: |
| return [gr.update(visible=True, maximum=1344), gr.update(visible=True), gr.update(visible=True)] |
| generation_method.change( |
| upload_generation_method, generation_method, [length_slider, overlap_video_length, partial_video_length] |
| ) |
|
|
| def upload_source_method(source_method): |
| if source_method == "Text to Video (文本到视频)": |
| return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] |
| elif source_method == "Image to Video (图片到视频)": |
| return [gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None), gr.update(value=None)] |
| elif source_method == "Video to Video (视频到视频)": |
| return [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(), gr.update(), gr.update(value=None)] |
| else: |
| return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update()] |
| source_method.change( |
| upload_source_method, source_method, [ |
| image_to_video_col, video_to_video_col, control_video_col, start_image, end_image, |
| validation_video, validation_video_mask, control_video |
| ] |
| ) |
|
|
| def upload_resize_method(resize_method): |
| if resize_method == "Generate by": |
| return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] |
| else: |
| return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] |
| resize_method.change( |
| upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] |
| ) |
|
|
| generate_button.click( |
| fn=controller.generate, |
| inputs=[ |
| diffusion_transformer_dropdown, |
| base_model_dropdown, |
| lora_model_dropdown, |
| lora_alpha_slider, |
| prompt_textbox, |
| negative_prompt_textbox, |
| sampler_dropdown, |
| sample_step_slider, |
| resize_method, |
| width_slider, |
| height_slider, |
| base_resolution, |
| generation_method, |
| length_slider, |
| overlap_video_length, |
| partial_video_length, |
| cfg_scale_slider, |
| start_image, |
| end_image, |
| validation_video, |
| validation_video_mask, |
| control_video, |
| denoise_strength, |
| seed_textbox, |
| ], |
| outputs=[result_image, result_video, infer_progress] |
| ) |
| return demo, controller |
|
|
| def ui_host(GPU_memory_mode, scheduler_dict, model_name, model_type, compile_dit, weight_dtype, savedir_sample=None): |
| controller = CogVideoXFunController_Host( |
| GPU_memory_mode, scheduler_dict, model_name=model_name, model_type=model_type, |
| compile_dit=compile_dit, |
| weight_dtype=weight_dtype, savedir_sample=savedir_sample, |
| ) |
|
|
| with gr.Blocks(css=css) as demo: |
| gr.Markdown( |
| """ |
| # CogVideoX-Fun |
| |
| A CogVideoX with more flexible generation conditions, capable of producing videos of different resolutions, around 6 seconds, and fps 8 (frames 1 to 49), as well as image generated videos. |
| |
| [Github](https://github.com/aigc-apps/CogVideoX-Fun/) |
| """ |
| ) |
| with gr.Column(variant="panel"): |
| model_type = create_fake_model_type(visible=False) |
| diffusion_transformer_dropdown = create_fake_model_checkpoints(model_name, visible=True) |
| base_model_dropdown, lora_model_dropdown, lora_alpha_slider = create_fake_finetune_models_checkpoints(visible=True) |
| |
| with gr.Column(variant="panel"): |
| prompt_textbox, negative_prompt_textbox = create_prompts() |
|
|
| with gr.Row(): |
| with gr.Column(): |
| sampler_dropdown, sample_step_slider = create_samplers(controller) |
|
|
| resize_method, width_slider, height_slider, base_resolution = create_height_width( |
| default_height = 384, default_width = 672, maximum_height = 1344, |
| maximum_width = 1344, |
| ) |
| gr.Markdown( |
| """ |
| V1.0 and V1.1 support up to 49 frames of video generation, while V1.5 supports up to 85 frames. |
| (V1.0和V1.1支持最大49帧视频生成，V1.5支持最大85帧视频生成。) |
| """ |
| ) |
| generation_method, length_slider, overlap_video_length, partial_video_length = \ |
| create_generation_methods_and_video_length( |
| ["Video Generation", "Image Generation"], |
| default_video_length=49, |
| maximum_video_length=85, |
| ) |
| image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( |
| ["Text to Video (文本到视频)", "Image to Video (图片到视频)", "Video to Video (视频到视频)", "Video Control (视频控制)"], prompt_textbox |
| ) |
| cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) |
|
|
| generate_button = gr.Button(value="Generate (生成)", variant='primary') |
|
|
| result_image, result_video, infer_progress = create_ui_outputs() |
|
|
| def upload_generation_method(generation_method): |
| if generation_method == "Video Generation": |
| return gr.update(visible=True, minimum=8, maximum=85, value=49, interactive=True) |
| elif generation_method == "Image Generation": |
| return gr.update(minimum=1, maximum=1, value=1, interactive=False) |
| generation_method.change( |
| upload_generation_method, generation_method, [length_slider] |
| ) |
|
|
| def upload_source_method(source_method): |
| if source_method == "Text to Video (文本到视频)": |
| return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] |
| elif source_method == "Image to Video (图片到视频)": |
| return [gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None), gr.update(value=None)] |
| elif source_method == "Video to Video (视频到视频)": |
| return [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(), gr.update(), gr.update(value=None)] |
| else: |
| return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update()] |
| source_method.change( |
| upload_source_method, source_method, [ |
| image_to_video_col, video_to_video_col, control_video_col, start_image, end_image, |
| validation_video, validation_video_mask, control_video |
| ] |
| ) |
|
|
| def upload_resize_method(resize_method): |
| if resize_method == "Generate by": |
| return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] |
| else: |
| return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] |
| resize_method.change( |
| upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] |
| ) |
|
|
| generate_button.click( |
| fn=controller.generate, |
| inputs=[ |
| diffusion_transformer_dropdown, |
| base_model_dropdown, |
| lora_model_dropdown, |
| lora_alpha_slider, |
| prompt_textbox, |
| negative_prompt_textbox, |
| sampler_dropdown, |
| sample_step_slider, |
| resize_method, |
| width_slider, |
| height_slider, |
| base_resolution, |
| generation_method, |
| length_slider, |
| overlap_video_length, |
| partial_video_length, |
| cfg_scale_slider, |
| start_image, |
| end_image, |
| validation_video, |
| validation_video_mask, |
| control_video, |
| denoise_strength, |
| seed_textbox, |
| ], |
| outputs=[result_image, result_video, infer_progress] |
| ) |
| return demo, controller |
|
|
| def ui_client(scheduler_dict, model_name, savedir_sample=None): |
| controller = CogVideoXFunController_Client(scheduler_dict, savedir_sample) |
|
|
| with gr.Blocks(css=css) as demo: |
| gr.Markdown( |
| """ |
| # CogVideoX-Fun |
| |
| A CogVideoX with more flexible generation conditions, capable of producing videos of different resolutions, around 6 seconds, and fps 8 (frames 1 to 49), as well as image generated videos. |
| |
| [Github](https://github.com/aigc-apps/CogVideoX-Fun/) |
| """ |
| ) |
| with gr.Column(variant="panel"): |
| diffusion_transformer_dropdown = create_fake_model_checkpoints(model_name, visible=True) |
| base_model_dropdown, lora_model_dropdown, lora_alpha_slider = create_fake_finetune_models_checkpoints(visible=True) |
| |
| with gr.Column(variant="panel"): |
| prompt_textbox, negative_prompt_textbox = create_prompts() |
|
|
| with gr.Row(): |
| with gr.Column(): |
| sampler_dropdown, sample_step_slider = create_samplers(controller, maximum_step=50) |
|
|
| resize_method, width_slider, height_slider, base_resolution = create_fake_height_width( |
| default_height = 384, default_width = 672, maximum_height = 1344, |
| maximum_width = 1344, |
| ) |
| gr.Markdown( |
| """ |
| V1.0 and V1.1 support up to 49 frames of video generation, while V1.5 supports up to 85 frames. |
| (V1.0和V1.1支持最大49帧视频生成，V1.5支持最大85帧视频生成。) |
| """ |
| ) |
| generation_method, length_slider, overlap_video_length, partial_video_length = \ |
| create_generation_methods_and_video_length( |
| ["Video Generation", "Image Generation"], |
| default_video_length=49, |
| maximum_video_length=85, |
| ) |
| image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( |
| ["Text to Video (文本到视频)", "Image to Video (图片到视频)", "Video to Video (视频到视频)"], prompt_textbox |
| ) |
|
|
| cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) |
|
|
| generate_button = gr.Button(value="Generate (生成)", variant='primary') |
|
|
| result_image, result_video, infer_progress = create_ui_outputs() |
|
|
| def upload_generation_method(generation_method): |
| if generation_method == "Video Generation": |
| return gr.update(visible=True, minimum=5, maximum=85, value=49, interactive=True) |
| elif generation_method == "Image Generation": |
| return gr.update(minimum=1, maximum=1, value=1, interactive=False) |
| generation_method.change( |
| upload_generation_method, generation_method, [length_slider] |
| ) |
|
|
| def upload_source_method(source_method): |
| if source_method == "Text to Video (文本到视频)": |
| return [gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] |
| elif source_method == "Image to Video (图片到视频)": |
| return [gr.update(visible=True), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None)] |
| else: |
| return [gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(), gr.update()] |
| source_method.change( |
| upload_source_method, source_method, [image_to_video_col, video_to_video_col, start_image, end_image, validation_video, validation_video_mask] |
| ) |
|
|
| def upload_resize_method(resize_method): |
| if resize_method == "Generate by": |
| return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] |
| else: |
| return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] |
| resize_method.change( |
| upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] |
| ) |
|
|
| generate_button.click( |
| fn=controller.generate, |
| inputs=[ |
| diffusion_transformer_dropdown, |
| base_model_dropdown, |
| lora_model_dropdown, |
| lora_alpha_slider, |
| prompt_textbox, |
| negative_prompt_textbox, |
| sampler_dropdown, |
| sample_step_slider, |
| resize_method, |
| width_slider, |
| height_slider, |
| base_resolution, |
| generation_method, |
| length_slider, |
| cfg_scale_slider, |
| start_image, |
| end_image, |
| validation_video, |
| validation_video_mask, |
| denoise_strength, |
| seed_textbox, |
| ], |
| outputs=[result_image, result_video, infer_progress] |
| ) |
| return demo, controller |
