| | import os |
| | import gradio as gr |
| | from sd_model_cfg import model_dict |
| | from app import process, process0, process1, process2, get_frame_count, cfg_to_input |
| |
|
| | DESCRIPTION = ''' |
| | ## Rerender A Video |
| | ### This space provides the function of key frame translation. Full code for full video translation will be released upon the publication of the paper. |
| | ### To avoid overload, we set limitations to the maximum frame number (8) and the maximum frame resolution (512x768). |
| | ### The running time of a video of size 512x640 is about 1 minute per keyframe under T4 GPU. |
| | ### How to use: |
| | 1. **Run 1st Key Frame**: only translate the first frame, so you can adjust the prompts/models/parameters to find your ideal output appearance before run the whole video. |
| | 2. **Run Key Frames**: translate all the key frames based on the settings of the first frame |
| | 3. **Run All**: **Run 1st Key Frame** and **Run Key Frames** |
| | 4. **Run Propagation**: propogate the key frames to other frames for full video translation. This part will be released upon the publication of the paper. |
| | ### Tips: |
| | 1. This method cannot handle large or quick motions where the optical flow is hard to estimate. **Videos with stable motions are preferred**. |
| | 2. Pixel-aware fusion may not work for large or quick motions. |
| | 3. Try different color-aware AdaIN settings and even unuse it to avoid color jittering. |
| | 4. `revAnimated_v11` model for non-photorealstic style, `realisticVisionV20_v20` model for photorealstic style. |
| | 5. To use your own SD/LoRA model, you may clone the space and specify your model with [sd_model_cfg.py](https://huggingface.co/spaces/Anonymous-sub/Rerender/blob/main/sd_model_cfg.py). |
| | 6. This method is based on the original SD model. You may need to [convert](https://github.com/huggingface/diffusers/blob/main/scripts/convert_diffusers_to_original_stable_diffusion.py) Diffuser/Automatic1111 models to the original one. |
| | |
| | **This code is for research purpose and non-commercial use only.** |
| | |
| | <a href="https://huggingface.co/spaces/Anonymous-sub/Rerender?duplicate=true" style="display: inline-block;margin-top: .5em;margin-right: .25em;" target="_blank"> |
| | <img style="margin-bottom: 0em;display: inline;margin-top: -.25em;" src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a> for no queue on your own hardware.</p> |
| | ''' |
| |
|
| | MAX_KEYFRAME = 100000000 |
| |
|
| | block = gr.Blocks().queue() |
| | with block: |
| | with gr.Row(): |
| | gr.Markdown(DESCRIPTION) |
| | with gr.Row(): |
| | with gr.Column(): |
| | input_path = gr.Video(label='Input Video', |
| | source='upload', |
| | format='mp4', |
| | visible=True) |
| | prompt = gr.Textbox(label='Prompt') |
| | seed = gr.Slider(label='Seed', |
| | minimum=0, |
| | maximum=2147483647, |
| | step=1, |
| | value=0, |
| | randomize=True) |
| | run_button = gr.Button(value='Run All') |
| | with gr.Row(): |
| | run_button1 = gr.Button(value='Run 1st Key Frame') |
| | run_button2 = gr.Button(value='Run Key Frames') |
| | run_button3 = gr.Button(value='Run Propagation') |
| | with gr.Accordion('Advanced options for the 1st frame translation', |
| | open=False): |
| | image_resolution = gr.Slider( |
| | label='Frame rsolution', |
| | minimum=256, |
| | maximum=512, |
| | value=512, |
| | step=64, |
| | info='To avoid overload, maximum 512') |
| | control_strength = gr.Slider(label='ControNet strength', |
| | minimum=0.0, |
| | maximum=2.0, |
| | value=1.0, |
| | step=0.01) |
| | x0_strength = gr.Slider( |
| | label='Denoising strength', |
| | minimum=0.00, |
| | maximum=1.05, |
| | value=0.75, |
| | step=0.05, |
| | info=('0: fully recover the input.' |
| | '1.05: fully rerender the input.')) |
| | color_preserve = gr.Checkbox( |
| | label='Preserve color', |
| | value=True, |
| | info='Keep the color of the input video') |
| | with gr.Row(): |
| | left_crop = gr.Slider(label='Left crop length', |
| | minimum=0, |
| | maximum=512, |
| | value=0, |
| | step=1) |
| | right_crop = gr.Slider(label='Right crop length', |
| | minimum=0, |
| | maximum=512, |
| | value=0, |
| | step=1) |
| | with gr.Row(): |
| | top_crop = gr.Slider(label='Top crop length', |
| | minimum=0, |
| | maximum=512, |
| | value=0, |
| | step=1) |
| | bottom_crop = gr.Slider(label='Bottom crop length', |
| | minimum=0, |
| | maximum=512, |
| | value=0, |
| | step=1) |
| | with gr.Row(): |
| | control_type = gr.Dropdown(['HED', 'canny'], |
| | label='Control type', |
| | value='HED') |
| | low_threshold = gr.Slider(label='Canny low threshold', |
| | minimum=1, |
| | maximum=255, |
| | value=100, |
| | step=1) |
| | high_threshold = gr.Slider(label='Canny high threshold', |
| | minimum=1, |
| | maximum=255, |
| | value=200, |
| | step=1) |
| | ddim_steps = gr.Slider(label='Steps', |
| | minimum=1, |
| | maximum=20, |
| | value=20, |
| | step=1, |
| | info='To avoid overload, maximum 20') |
| | scale = gr.Slider(label='CFG scale', |
| | minimum=0.1, |
| | maximum=30.0, |
| | value=7.5, |
| | step=0.1) |
| | sd_model_list = list(model_dict.keys()) |
| | sd_model = gr.Dropdown(sd_model_list, |
| | label='Base model', |
| | value='Stable Diffusion 1.5') |
| | a_prompt = gr.Textbox(label='Added prompt', |
| | value='best quality, extremely detailed') |
| | n_prompt = gr.Textbox( |
| | label='Negative prompt', |
| | value=('longbody, lowres, bad anatomy, bad hands, ' |
| | 'missing fingers, extra digit, fewer digits, ' |
| | 'cropped, worst quality, low quality')) |
| | with gr.Accordion('Advanced options for the key fame translation', |
| | open=False): |
| | interval = gr.Slider( |
| | label='Key frame frequency (K)', |
| | minimum=1, |
| | maximum=1, |
| | value=1, |
| | step=1, |
| | info='Uniformly sample the key frames every K frames') |
| | keyframe_count = gr.Slider( |
| | label='Number of key frames', |
| | minimum=1, |
| | maximum=1, |
| | value=1, |
| | step=1, |
| | info='To avoid overload, maximum 8 key frames') |
| |
|
| | use_constraints = gr.CheckboxGroup( |
| | [ |
| | 'shape-aware fusion', 'pixel-aware fusion', |
| | 'color-aware AdaIN' |
| | ], |
| | label='Select the cross-frame contraints to be used', |
| | value=[ |
| | 'shape-aware fusion', 'pixel-aware fusion', |
| | 'color-aware AdaIN' |
| | ]), |
| | with gr.Row(): |
| | cross_start = gr.Slider( |
| | label='Cross-frame attention start', |
| | minimum=0, |
| | maximum=1, |
| | value=0, |
| | step=0.05) |
| | cross_end = gr.Slider(label='Cross-frame attention end', |
| | minimum=0, |
| | maximum=1, |
| | value=1, |
| | step=0.05) |
| | style_update_freq = gr.Slider( |
| | label='Cross-frame attention update frequency', |
| | minimum=1, |
| | maximum=100, |
| | value=1, |
| | step=1, |
| | info= |
| | ('Update the key and value for ' |
| | 'cross-frame attention every N key frames (recommend N*K>=10)' |
| | )) |
| | with gr.Row(): |
| | warp_start = gr.Slider(label='Shape-aware fusion start', |
| | minimum=0, |
| | maximum=1, |
| | value=0, |
| | step=0.05) |
| | warp_end = gr.Slider(label='Shape-aware fusion end', |
| | minimum=0, |
| | maximum=1, |
| | value=0.1, |
| | step=0.05) |
| | with gr.Row(): |
| | mask_start = gr.Slider(label='Pixel-aware fusion start', |
| | minimum=0, |
| | maximum=1, |
| | value=0.5, |
| | step=0.05) |
| | mask_end = gr.Slider(label='Pixel-aware fusion end', |
| | minimum=0, |
| | maximum=1, |
| | value=0.8, |
| | step=0.05) |
| | with gr.Row(): |
| | ada_start = gr.Slider(label='Color-aware AdaIN start', |
| | minimum=0, |
| | maximum=1, |
| | value=0.8, |
| | step=0.05) |
| | ada_end = gr.Slider(label='Color-aware AdaIN end', |
| | minimum=0, |
| | maximum=1, |
| | value=1, |
| | step=0.05) |
| | mask_strength = gr.Slider(label='Pixel-aware fusion stength', |
| | minimum=0, |
| | maximum=1, |
| | value=0.5, |
| | step=0.01) |
| | inner_strength = gr.Slider( |
| | label='Pixel-aware fusion detail level', |
| | minimum=0.5, |
| | maximum=1, |
| | value=0.9, |
| | step=0.01, |
| | info='Use a low value to prevent artifacts') |
| | smooth_boundary = gr.Checkbox( |
| | label='Smooth fusion boundary', |
| | value=True, |
| | info='Select to prevent artifacts at boundary') |
| |
|
| | with gr.Accordion('Example configs', open=True): |
| | config_dir = 'config' |
| | config_list = os.listdir(config_dir) |
| | args_list = [] |
| | for config in config_list: |
| | try: |
| | config_path = os.path.join(config_dir, config) |
| | args = cfg_to_input(config_path) |
| | args_list.append(args) |
| | except FileNotFoundError: |
| | |
| | pass |
| |
|
| | ips = [ |
| | prompt, image_resolution, control_strength, color_preserve, |
| | left_crop, right_crop, top_crop, bottom_crop, control_type, |
| | low_threshold, high_threshold, ddim_steps, scale, seed, |
| | sd_model, a_prompt, n_prompt, interval, keyframe_count, |
| | x0_strength, use_constraints[0], cross_start, cross_end, |
| | style_update_freq, warp_start, warp_end, mask_start, |
| | mask_end, ada_start, ada_end, mask_strength, |
| | inner_strength, smooth_boundary |
| | ] |
| |
|
| | with gr.Column(): |
| | result_image = gr.Image(label='Output first frame', |
| | type='numpy', |
| | interactive=False) |
| | result_keyframe = gr.Video(label='Output key frame video', |
| | format='mp4', |
| | interactive=False) |
| | with gr.Row(): |
| | gr.Examples(examples=args_list, |
| | inputs=[input_path, *ips], |
| | fn=process0, |
| | outputs=[result_image, result_keyframe], |
| | cache_examples=True) |
| |
|
| | def input_uploaded(path): |
| | frame_count = get_frame_count(path) |
| | if frame_count <= 2: |
| | raise gr.Error('The input video is too short!' |
| | 'Please input another video.') |
| |
|
| | default_interval = min(10, frame_count - 2) |
| | max_keyframe = min((frame_count - 2) // default_interval, MAX_KEYFRAME) |
| |
|
| | global video_frame_count |
| | video_frame_count = frame_count |
| | global global_video_path |
| | global_video_path = path |
| |
|
| | return gr.Slider.update(value=default_interval, |
| | maximum=MAX_KEYFRAME), gr.Slider.update( |
| | value=max_keyframe, maximum=max_keyframe) |
| |
|
| | def input_changed(path): |
| | frame_count = get_frame_count(path) |
| | if frame_count <= 2: |
| | return gr.Slider.update(maximum=1), gr.Slider.update(maximum=1) |
| |
|
| | default_interval = min(10, frame_count - 2) |
| | max_keyframe = min((frame_count - 2) // default_interval, MAX_KEYFRAME) |
| |
|
| | global video_frame_count |
| | video_frame_count = frame_count |
| | global global_video_path |
| | global_video_path = path |
| |
|
| | return gr.Slider.update(maximum=max_keyframe), \ |
| | gr.Slider.update(maximum=max_keyframe) |
| |
|
| | def interval_changed(interval): |
| | global video_frame_count |
| | if video_frame_count is None: |
| | return gr.Slider.update() |
| |
|
| | max_keyframe = (video_frame_count - 2) // interval |
| |
|
| | return gr.Slider.update(value=max_keyframe, maximum=max_keyframe) |
| |
|
| | input_path.change(input_changed, input_path, [interval, keyframe_count]) |
| | input_path.upload(input_uploaded, input_path, [interval, keyframe_count]) |
| | interval.change(interval_changed, interval, keyframe_count) |
| |
|
| | run_button.click(fn=process, |
| | inputs=ips, |
| | outputs=[result_image, result_keyframe]) |
| | run_button1.click(fn=process1, inputs=ips, outputs=[result_image]) |
| | run_button2.click(fn=process2, inputs=ips, outputs=[result_keyframe]) |
| |
|
| | def process3(): |
| | raise gr.Error( |
| | "Coming Soon. Full code for full video translation will be " |
| | "released upon the publication of the paper.") |
| |
|
| | run_button3.click(fn=process3, outputs=[result_keyframe]) |
| |
|
| | block.queue(concurrency_count=1, max_size=20) |
| | block.launch(server_name='0.0.0.0') |
| |
|
