Update app.py

app.py

@@ -23,21 +23,24 @@ from huggingface_hub import hf_hub_download
 config_path = "model_config.yaml"
 config = OmegaConf.load(config_path)
 
-
 # Download model
 REPO_ID = 'VideoCrafter/t2v-version-1-1'
 filename_list = ['models/base_t2v/model.ckpt',
                 'models/videolora/lora_001_Loving_Vincent_style.ckpt',
                 'models/videolora/lora_002_frozenmovie_style.ckpt',
                 'models/videolora/lora_003_MakotoShinkaiYourName_style.ckpt',
-                'models/videolora/lora_004_coco_style.ckpt']
+                'models/videolora/lora_004_coco_style.ckpt',
+                'models/adapter_t2v_depth/adapter.pth']
+
 for filename in filename_list:
     if not os.path.exists(filename):
         hf_hub_download(repo_id=REPO_ID, filename=filename, local_dir='./', local_dir_use_symlinks=False)
             
+ckpt_path = 'models/base_t2v/model.ckpt'
 
+midas_path_url = 'https://github.com/isl-org/DPT/releases/download/1_0/dpt_hybrid-midas-501f0c75.pt'
 
-ckpt_path = 'models/base_t2v/model.ckpt'
+subprocess.run(shlex.split(f'wget {midas_path_url} -O models/adapter_t2v_depth/dpt_hybrid-midas.pt'))
 
 # # get model & sampler
 model, _, _ = load_model(config, ckpt_path, 
@@ -170,7 +173,13 @@ def save_results(videos,
     return os.path.join(save_subdir, f"{save_name}_{i:03d}.mp4")
 
 def save_results_control(batch_samples, batch_conds):
-    return 
+    save_subdir = os.path.join("videos")
+    os.makedirs(save_subdir, exist_ok=True)
+    
+    tensor_to_mp4(video=batch_conds.detach().cpu(), savepath=os.path.join(save_subdir, f'results_depth.mp4'), fps=10)
+    tensor_to_mp4(video=batch_samples.detach().cpu(), savepath=os.path.join(save_subdir, f'results_sample.mp4'), fps=10)
+    
+    return os.path.join(save_subdir, f'results_depth.mp4'), os.path.join(save_subdir, f'results_sample.mp4')
 
 def get_video(prompt, seed, ddim_steps):
     seed_everything(seed)
@@ -206,12 +215,12 @@ def get_video_lora(prompt, seed, ddim_steps, model_choice):
 
 def get_video_control(prompt, input_video, seed, ddim_steps):
     seed_everything(seed)
-    # h,w = 512//8, 512//8
-    # noise_shape = [1, model.channels, model.temporal_length,h,w]
-    # batch_samples, batch_conds = adapter_guided_synthesis(model, prompt,input_video,noise_shape, sampler=ddim_sampler, n_samples = 1,
-    #                       ddim_steps=ddim_steps
-    #                       )
-    # return save_results_control(batch_samples, batch_conds)
+    h,w = 512//8, 512//8
+    noise_shape = [1, model.channels, model_control.temporal_length,h,w]
+    batch_samples, batch_conds = adapter_guided_synthesis(model, prompt,input_video,noise_shape, sampler=ddim_sampler, n_samples = 1,
+                          ddim_steps=ddim_steps
+                          )
+    #return save_results_control(batch_samples, batch_conds)
     return input_video
 
 from gradio_t2v import create_demo as create_demo_basic