Update app.py

app.py

@@ -1,36 +1,186 @@
-import gradio as gr
 import os
 import shutil
-import ffmpeg
-from datetime import datetime
-from pathlib import Path
 import numpy as np
+import gradio as gr
+import torchaudio
+import soundfile as sf
+from pathlib import Path
+from datetime import datetime
+from scipy.io.wavfile import write as write_wav
+from transformers import Wav2Vec2ForCTC, Wav2Vec2Tokenizer
+from encodec.utils import convert_audio
+
+from src.bark.history_to_hash import history_to_hash
+from src.bark.npz_tools import save_npz
+from src.bark.FullGeneration import FullGeneration
+from src.utils.date import get_date_string
+from src.bark.get_audio_from_npz import get_audio_from_full_generation
+from bark_hubert_quantizer.hubert_manager import HuBERTManager
+from bark_hubert_quantizer.pre_kmeans_hubert import CustomHubert
+from bark_hubert_quantizer.customtokenizer import CustomTokenizer
+from bark import SAMPLE_RATE
+from bark.generation import SUPPORTED_LANGS, generate_text_semantic, generate_coarse, generate_fine, codec_decode
+
+import ffmpeg
 import cv2
 import torch
-import spaces
-
+from PIL import Image
 from diffusers import AutoencoderKL, DDIMScheduler
 from einops import repeat
 from omegaconf import OmegaConf
-from PIL import Image
 from torchvision import transforms
 from transformers import CLIPVisionModelWithProjection
-
 from src.models.pose_guider import PoseGuider
 from src.models.unet_2d_condition import UNet2DConditionModel
 from src.models.unet_3d import UNet3DConditionModel
 from src.pipelines.pipeline_pose2vid_long import Pose2VideoPipeline
 from src.utils.util import get_fps, read_frames, save_videos_grid, save_pil_imgs
-
 from src.audio_models.model import Audio2MeshModel
 from src.utils.audio_util import prepare_audio_feature
-from src.utils.mp_utils  import LMKExtractor
+from src.utils.mp_utils import LMKExtractor
 from src.utils.draw_util import FaceMeshVisualizer
 from src.utils.pose_util import project_points, project_points_with_trans, matrix_to_euler_and_translation, euler_and_translation_to_matrix
 from src.utils.crop_face_single import crop_face
 from src.audio2vid import get_headpose_temp, smooth_pose_seq
 from src.utils.frame_interpolation import init_frame_interpolation_model, batch_images_interpolation_tool
 
+hubert_model = None
+
+def _load_hubert_model(device):
+    hubert_path = HuBERTManager.make_sure_hubert_installed()
+    global hubert_model
+    if hubert_model is None:
+        hubert_model = CustomHubert(
+            checkpoint_path=hubert_path,
+            device=device,
+        )
+    return hubert_model
+
+def _get_semantic_vectors(hubert_model: CustomHubert, path_to_wav: str, device):
+    wav, sr = torchaudio.load(path_to_wav)
+    if wav.shape[0] == 2:
+        wav = wav.mean(0, keepdim=True)
+    wav = wav.to(device)
+    return hubert_model.forward(wav, input_sample_hz=sr)
+
+def get_semantic_vectors(path_to_wav: str, device):
+    hubert_model = _load_hubert_model(device)
+    return _get_semantic_vectors(hubert_model, path_to_wav, device)
+
+tokenizer = None
+
+def _load_tokenizer(
+    model: str = "quantifier_hubert_base_ls960_14.pth",
+    repo: str = "GitMylo/bark-voice-cloning",
+    force_reload: bool = False,
+    device="cpu",
+) -> CustomTokenizer:
+    tokenizer_path = HuBERTManager.make_sure_tokenizer_installed(
+        model=model,
+        repo=repo,
+        local_file=model,
+    )
+    global tokenizer
+    if tokenizer is None or force_reload:
+        tokenizer = CustomTokenizer.load_from_checkpoint(
+            tokenizer_path,
+            map_location=device,
+        )
+        tokenizer.load_state_dict(torch.load(tokenizer_path, map_location=device))
+    return tokenizer
+
+def get_semantic_tokens(semantic_vectors: torch.Tensor, device):
+    tokenizer = _load_tokenizer(device=device)
+    return tokenizer.get_token(semantic_vectors)
+
+def get_semantic_prompt(path_to_wav: str, device):
+    semantic_vectors = get_semantic_vectors(path_to_wav, device)
+    return get_semantic_tokens(semantic_vectors, device).cpu().numpy()
+
+def get_prompts(path_to_wav: str, use_gpu: bool):
+    device = "cuda" if use_gpu else "cpu"
+    semantic_prompt = get_semantic_prompt(path_to_wav, device)
+    fine_prompt, coarse_prompt = get_encodec_prompts(path_to_wav, use_gpu)
+    return FullGeneration(
+        semantic_prompt=semantic_prompt,
+        coarse_prompt=coarse_prompt,
+        fine_prompt=fine_prompt,
+    )
+
+def get_encodec_prompts(path_to_wav: str, use_gpu=True):
+    device = "cuda" if use_gpu else "cpu"
+    model = load_codec_model(use_gpu=use_gpu)
+    wav, sr = torchaudio.load(path_to_wav)
+    wav = convert_audio(wav, sr, model.sample_rate, model.channels)
+    wav = wav.unsqueeze(0).to(device)
+    model.to(device)
+
+    with torch.no_grad():
+        encoded_frames = model.encode(wav)
+
+    fine_prompt = (
+        torch.cat([encoded[0] for encoded in encoded_frames], dim=-1)
+        .squeeze()
+        .cpu()
+        .numpy()
+    )
+    coarse_prompt = fine_prompt[:2, :]
+    return fine_prompt, coarse_prompt
+
+def save_cloned_voice(full_generation: FullGeneration):
+    voice_name = f"voice_from_audio_{history_to_hash(full_generation)}"
+    filename = f"voices/{voice_name}.npz"
+    date = get_date_string()
+    metadata = generate_cloned_voice_metadata(full_generation, date)
+    save_npz(filename, full_generation, metadata)
+    return filename
+
+def generate_cloned_voice_metadata(full_generation, date):
+    return {
+        "_version": "0.0.1",
+        "_hash_version": "0.0.2",
+        "_type": "bark",
+        "hash": history_to_hash(full_generation),
+        "date": date,
+    }
+
+def generate_voice(wav_file: str, use_gpu: bool):
+    full_generation = get_prompts(wav_file, use_gpu)
+    filename = save_cloned_voice(full_generation)
+    return filename, get_audio_from_full_generation(full_generation)
+
+# 음성 합성을 위한 함수
+def synthesize_speech(text, input_audio):
+    semantic_tokens = generate_text_semantic(text)
+    coarse_tokens = generate_coarse(semantic_tokens)
+    fine_tokens = generate_fine(coarse_tokens)
+    synthesized_audio = codec_decode(fine_tokens)
+    if isinstance(synthesized_audio, torch.Tensor):
+        synthesized_audio = synthesized_audio.squeeze().cpu().numpy()
+    else:
+        synthesized_audio = synthesized_audio.squeeze()
+
+    # 입력 음성의 길이 가져오기
+    input_wav, input_sr = torchaudio.load(input_audio)
+    input_length = input_wav.shape[1] / input_sr
+
+    # 출력 음성을 입력 음성의 길이에 맞추기
+    output_length = synthesized_audio.shape[0] / SAMPLE_RATE
+    if output_length > input_length:
+        synthesized_audio = synthesized_audio[:int(input_length * SAMPLE_RATE)]
+    else:
+        padding = int((input_length - output_length) * SAMPLE_RATE)
+        synthesized_audio = np.pad(synthesized_audio, (0, padding), 'constant')
+
+    sf.write("synthesized_audio.wav", synthesized_audio, SAMPLE_RATE)
+    return "synthesized_audio.wav"
+
+# TTS 기능 함수
+def tts_function(input_audio, input_text):
+    synthesized_audio_path = synthesize_speech(input_text, input_audio)
+    return synthesized_audio_path
+
+# aniportrait 함수 정의
 config = OmegaConf.load('./configs/prompts/animation_audio.yaml')
 if config.weight_dtype == "fp16":
     weight_dtype = torch.float16
@@ -38,49 +188,28 @@ else:
     weight_dtype = torch.float32
     
 audio_infer_config = OmegaConf.load(config.audio_inference_config)
-# prepare model
 a2m_model = Audio2MeshModel(audio_infer_config['a2m_model'])
 a2m_model.load_state_dict(torch.load(audio_infer_config['pretrained_model']['a2m_ckpt'], map_location="cpu"), strict=False)
 a2m_model.cuda().eval()
 
-vae = AutoencoderKL.from_pretrained(
-    config.pretrained_vae_path,
-).to("cuda", dtype=weight_dtype)
+vae = AutoencoderKL.from_pretrained(config.pretrained_vae_path).to("cuda", dtype=weight_dtype)
 
-reference_unet = UNet2DConditionModel.from_pretrained(
-    config.pretrained_base_model_path,
-    subfolder="unet",
-).to(dtype=weight_dtype, device="cuda")
+reference_unet = UNet2DConditionModel.from_pretrained(config.pretrained_base_model_path, subfolder="unet").to(dtype=weight_dtype, device="cuda")
 
 inference_config_path = config.inference_config
 infer_config = OmegaConf.load(inference_config_path)
-denoising_unet = UNet3DConditionModel.from_pretrained_2d(
-    config.pretrained_base_model_path,
-    config.motion_module_path,
-    subfolder="unet",
-    unet_additional_kwargs=infer_config.unet_additional_kwargs,
-).to(dtype=weight_dtype, device="cuda")
+denoising_unet = UNet3DConditionModel.from_pretrained_2d(config.pretrained_base_model_path, config.motion_module_path, subfolder="unet", unet_additional_kwargs=infer_config.unet_additional_kwargs).to(dtype=weight_dtype, device="cuda")
 
-pose_guider = PoseGuider(noise_latent_channels=320, use_ca=True).to(device="cuda", dtype=weight_dtype) # not use cross attention
+pose_guider = PoseGuider(noise_latent_channels=320, use_ca=True).to(device="cuda", dtype=weight_dtype)
 
-image_enc = CLIPVisionModelWithProjection.from_pretrained(
-    config.image_encoder_path
-).to(dtype=weight_dtype, device="cuda")
+image_enc = CLIPVisionModelWithProjection.from_pretrained(config.image_encoder_path).to(dtype=weight_dtype, device="cuda")
 
 sched_kwargs = OmegaConf.to_container(infer_config.noise_scheduler_kwargs)
 scheduler = DDIMScheduler(**sched_kwargs)
 
-# load pretrained weights
-denoising_unet.load_state_dict(
-    torch.load(config.denoising_unet_path, map_location="cpu"),
-    strict=False,
-)
-reference_unet.load_state_dict(
-    torch.load(config.reference_unet_path, map_location="cpu"),
-)
-pose_guider.load_state_dict(
-    torch.load(config.pose_guider_path, map_location="cpu"),
-)
+denoising_unet.load_state_dict(torch.load(config.denoising_unet_path, map_location="cpu"), strict=False)
+reference_unet.load_state_dict(torch.load(config.reference_unet_path, map_location="cpu"))
+pose_guider.load_state_dict(torch.load(config.pose_guider_path, map_location="cpu"))
 
 pipe = Pose2VideoPipeline(
     vae=vae,
@@ -92,13 +221,10 @@ pipe = Pose2VideoPipeline(
 )
 pipe = pipe.to("cuda", dtype=weight_dtype)
 
-# lmk_extractor = LMKExtractor()
-# vis = FaceMeshVisualizer()
-
 frame_inter_model = init_frame_interpolation_model()
 
 @spaces.GPU
-def audio2video(input_audio, ref_img, headpose_video=None, size=512, steps=25, length=60, seed=42):
+def audio2video(input_audio, ref_img, headpose_video=None, size=512, steps=25, length=60, seed=42):   
     fps = 30
     cfg = 3.5
     fi_step = 3
@@ -138,7 +264,6 @@ def audio2video(input_audio, ref_img, headpose_video=None, size=512, steps=25, l
     sample['audio_feature'] = torch.from_numpy(sample['audio_feature']).float().cuda()
     sample['audio_feature'] = sample['audio_feature'].unsqueeze(0)
 
-    # inference
     pred = a2m_model.infer(sample['audio_feature'], sample['seq_len'])
     pred = pred.squeeze().detach().cpu().numpy()
     pred = pred.reshape(pred.shape[0], -1, 3)
@@ -151,7 +276,6 @@ def audio2video(input_audio, ref_img, headpose_video=None, size=512, steps=25, l
     mirrored_pose_seq = np.concatenate((pose_seq, pose_seq[-2:0:-1]), axis=0)
     cycled_pose_seq = np.tile(mirrored_pose_seq, (sample['seq_len'] // len(mirrored_pose_seq) + 1, 1))[:sample['seq_len']]
 
-    # project 3D mesh to 2D landmark
     projected_vertices = project_points(pred, face_result['trans_mat'], cycled_pose_seq, [height, width])
 
     pose_images = []
@@ -160,10 +284,10 @@ def audio2video(input_audio, ref_img, headpose_video=None, size=512, steps=25, l
         pose_images.append(lmk_img)
 
     pose_list = []
-    args_L = len(pose_images) if length==0 or length > len(pose_images) else length
+    args_L = len(pose_images) if length == 0 or length > len(pose_images) else length
     args_L = min(args_L, 90)
-    for pose_image_np in pose_images[: args_L : fi_step]:
-        pose_image_np = cv2.resize(pose_image_np,  (width, height))
+    for pose_image_np in pose_images[:args_L:fi_step]:
+        pose_image_np = cv2.resize(pose_image_np, (width, height))
         pose_list.append(pose_image_np)
     
     pose_list = np.array(pose_list)
@@ -182,7 +306,7 @@ def audio2video(input_audio, ref_img, headpose_video=None, size=512, steps=25, l
         generator=generator,
     ).videos
     
-    video = batch_images_interpolation_tool(video, frame_inter_model, inter_frames=fi_step-1)
+    video = batch_images_interpolation_tool(video, frame_inter_model, inter_frames=fi_step - 1)
 
     save_path = f"{save_dir}/{size}x{size}_{time_str}_noaudio.mp4"
     save_videos_grid(
@@ -199,6 +323,131 @@ def audio2video(input_audio, ref_img, headpose_video=None, size=512, steps=25, l
     
     return save_path.replace('_noaudio.mp4', '.mp4'), ref_image_pil
 
+@spaces.GPU
+def video2video(ref_img, source_video, size=512, steps=25, length=60, seed=42):
+    cfg = 3.5
+    fi_step = 3
+    
+    generator = torch.manual_seed(seed)
+    
+    lmk_extractor = LMKExtractor()
+    vis = FaceMeshVisualizer()
+
+    width, height = size, size
+
+    date_str = datetime.now().strftime("%Y%m%d")
+    time_str = datetime.now().strftime("%H%M")
+    save_dir_name = f"{time_str}--seed_{seed}-{size}x{size}"
+
+    save_dir = Path(f"v2v_output/{date_str}/{save_dir_name}")
+    while os.path.exists(save_dir):
+        save_dir = Path(f"v2v_output/{date_str}/{save_dir_name}_{np.random.randint(10000):04d}")
+    save_dir.mkdir(exist_ok=True, parents=True)
+    
+    ref_image_np = cv2.cvtColor(ref_img, cv2.COLOR_RGB2BGR)
+    ref_image_np = crop_face(ref_image_np, lmk_extractor)
+    if ref_image_np is None:
+        return None, Image.fromarray(ref_img)
+    
+    ref_image_np = cv2.resize(ref_image_np, (size, size))
+    ref_image_pil = Image.fromarray(cv2.cvtColor(ref_image_np, cv2.COLOR_BGR2RGB))
+    
+    face_result = lmk_extractor(ref_image_np)
+    if face_result is None: 
+        return None, ref_image_pil
+    
+    lmks = face_result['lmks'].astype(np.float32)
+    ref_pose = vis.draw_landmarks((ref_image_np.shape[1], ref_image_np.shape[0]), lmks, normed=True)
+
+    source_images = read_frames(source_video)
+    src_fps = get_fps(source_video)
+    pose_transform = transforms.Compose([transforms.Resize((height, width)), transforms.ToTensor()])
+    
+    step = 1
+    if src_fps == 60:
+        src_fps = 30
+        step = 2
+    
+    pose_trans_list = []
+    verts_list = []
+    bs_list = []
+    args_L = len(source_images) if length == 0 or length * step > len(source_images) else length * step
+    args_L = min(args_L, 90 * step)
+    for src_image_pil in source_images[:args_L:step*fi_step]:
+        src_img_np = cv2.cvtColor(np.array(src_image_pil), cv2.COLOR_RGB2BGR)
+        frame_height, frame_width, _ = src_img_np.shape
+        src_img_result = lmk_extractor(src_img_np)
+        if src_img_result is None:
+            break
+        pose_trans_list.append(src_img_result['trans_mat'])
+        verts_list.append(src_img_result['lmks3d'])
+        bs_list.append(src_img_result['bs'])
+
+    trans_mat_arr = np.array(pose_trans_list)
+    verts_arr = np.array(verts_list)
+    bs_arr = np.array(bs_list)
+    min_bs_idx = np.argmin(bs_arr.sum(1))
+    
+    pose_arr = np.zeros([trans_mat_arr.shape[0], 6])
+    for i in range(pose_arr.shape[0]):
+        euler_angles, translation_vector = matrix_to_euler_and_translation(trans_mat_arr[i])
+        pose_arr[i, :3] = euler_angles
+        pose_arr[i, 3:6] = translation_vector
+    
+    init_tran_vec = face_result['trans_mat'][:3, 3]
+    pose_arr[:, 3:6] = pose_arr[:, 3:6] - pose_arr[0, 3:6] + init_tran_vec
+    pose_arr_smooth = smooth_pose_seq(pose_arr, window_size=3)
+    pose_mat_smooth = [euler_and_translation_to_matrix(pose_arr_smooth[i][:3], pose_arr_smooth[i][3:6]) for i in range(pose_arr_smooth.shape[0])]    
+    pose_mat_smooth = np.array(pose_mat_smooth)
+
+    verts_arr = verts_arr - verts_arr[min_bs_idx] + face_result['lmks3d']
+    projected_vertices = project_points_with_trans(verts_arr, pose_mat_smooth, [frame_height, frame_width])
+    
+    pose_list = []
+    for i, verts in enumerate(projected_vertices):
+        lmk_img = vis.draw_landmarks((frame_width, frame_height), verts, normed=False)
+        pose_image_np = cv2.resize(lmk_img, (width, height))
+        pose_list.append(pose_image_np)
+    
+    pose_list = np.array(pose_list)
+    
+    video_length = len(pose_list)
+
+    video = pipe(
+        ref_image_pil,
+        pose_list,
+        ref_pose,
+        width,
+        height,
+        video_length,
+        steps,
+        cfg,
+        generator=generator,
+    ).videos
+    
+    video = batch_images_interpolation_tool(video, frame_inter_model, inter_frames=fi_step - 1)
+
+    save_path = f"{save_dir}/{size}x{size}_{time_str}_noaudio.mp4"
+    save_videos_grid(
+        video,
+        save_path,
+        n_rows=1,
+        fps=src_fps,
+    )
+    
+    audio_output = f'{save_dir}/audio_from_video.aac'
+    try:
+        ffmpeg.input(source_video).output(audio_output, acodec='copy').run()
+        stream = ffmpeg.input(save_path)
+        audio = ffmpeg.input(audio_output)
+        ffmpeg.output(stream.video, audio.audio, save_path.replace('_noaudio.mp4', '.mp4'), vcodec='copy', acodec='aac', shortest=None).run()
+    
+        os.remove(save_path)
+        os.remove(audio_output)
+    except:
+        shutil.move(save_path, save_path.replace('_noaudio.mp4', '.mp4'))
+    
+    return save_path.replace('_noaudio.mp4', '.mp4'), ref_image_pil
 
 ################# GUI ################
 
@@ -225,8 +474,8 @@ with gr.Blocks() as demo:
             with gr.Column():
                 with gr.Row():
                     a2v_input_audio = gr.Audio(sources=["upload", "microphone"], type="filepath", editable=True, label="Input audio", interactive=True)
-                    a2v_ref_img = gr.Image(label="Upload reference image", sources="upload")
-                    a2v_headpose_video = gr.Video(label="Option: upload head pose reference video", sources="upload")
+                    a2v_ref_img = gr.Image(label="Upload reference image", source="upload")
+                    a2v_headpose_video = gr.Video(label="Option: upload head pose reference video", source="upload")
 
                 with gr.Row():
                     a2v_size_slider = gr.Slider(minimum=256, maximum=512, step=8, value=384, label="Video size (-W & -H)")
@@ -236,48 +485,73 @@ with gr.Blocks() as demo:
                     a2v_length = gr.Slider(minimum=0, maximum=90, step=1, value=30, label="Length (-L)")
                     a2v_seed = gr.Number(value=42, label="Seed (--seed)")
                 
-                a2v_botton = gr.Button("Generate", variant="primary")
-            a2v_output_video = gr.PlayableVideo(label="Result", interactive=False)
+                a2v_button = gr.Button("Generate", variant="primary")
+            a2v_output_video = gr.Video(label="Result", interactive=False)
         
         gr.Examples(
             examples=[
                 ["configs/inference/audio/lyl.wav", "configs/inference/ref_images/Aragaki.png", None],
                 ["configs/inference/audio/lyl.wav", "configs/inference/ref_images/solo.png", None],
                 ["configs/inference/audio/lyl.wav", "configs/inference/ref_images/lyl.png", "configs/inference/head_pose_temp/pose_ref_video.mp4"],
-                ],
+            ],
             inputs=[a2v_input_audio, a2v_ref_img, a2v_headpose_video],
         )
             
     
-    with gr.Tab("TTS"):
+    with gr.Tab("Video2video"):
         with gr.Row():
             with gr.Column():
                 with gr.Row():
-                    tts_text_input = gr.Textbox(lines=5, label="Input text", placeholder="Enter text to synthesize...")
-                    tts_ref_img = gr.Image(label="Upload reference image", sources="upload")
+                    v2v_ref_img = gr.Image(label="Upload reference image", source="upload")
+                    v2v_source_video = gr.Video(label="Upload source video", source="upload")
                 
                 with gr.Row():
-                    tts_size_slider = gr.Slider(minimum=256, maximum=512, step=8, value=384, label="Video size (-W & -H)")
-                    tts_step_slider = gr.Slider(minimum=5, maximum=20, step=1, value=15, label="Steps (--steps)")
+                    v2v_size_slider = gr.Slider(minimum=256, maximum=512, step=8, value=384, label="Video size (-W & -H)")
+                    v2v_step_slider = gr.Slider(minimum=5, maximum=20, step=1, value=15, label="Steps (--steps)")
                 
                 with gr.Row():
-                    tts_length = gr.Slider(minimum=0, maximum=90, step=1, value=30, label="Length (-L)")
-                    tts_seed = gr.Number(value=42, label="Seed (--seed)")
+                    v2v_length = gr.Slider(minimum=0, maximum=90, step=1, value=30, label="Length (-L)")
+                    v2v_seed = gr.Number(value=42, label="Seed (--seed)")
                 
-                tts_button = gr.Button("Generate", variant="primary")
-            tts_output_video = gr.PlayableVideo(label="Result", interactive=False)
+                v2v_button = gr.Button("Generate", variant="primary")
+            v2v_output_video = gr.Video(label="Result", interactive=False)
             
-    a2v_botton.click(
+        gr.Examples(
+            examples=[
+                ["configs/inference/ref_images/Aragaki.png", "configs/inference/video/Aragaki_song.mp4"],
+                ["configs/inference/ref_images/solo.png", "configs/inference/video/Aragaki_song.mp4"],
+                ["configs/inference/ref_images/lyl.png", "configs/inference/head_pose_temp/pose_ref_video.mp4"],
+            ],
+            inputs=[v2v_ref_img, v2v_source_video, a2v_headpose_video],
+        )
+            
+    a2v_button.click(
         fn=audio2video,
         inputs=[a2v_input_audio, a2v_ref_img, a2v_headpose_video,
                 a2v_size_slider, a2v_step_slider, a2v_length, a2v_seed], 
         outputs=[a2v_output_video, a2v_ref_img]
     )
+    v2v_button.click(
+        fn=video2video,
+        inputs=[v2v_ref_img, v2v_source_video,
+                v2v_size_slider, v2v_step_slider, v2v_length, v2v_seed], 
+        outputs=[v2v_output_video, v2v_ref_img]
+    )
+    
+    with gr.Tab("TTS"):
+        with gr.Row():
+            with gr.Column():
+                with gr.Row():
+                    tts_input_audio = gr.Audio(type="filepath", label="Input audio for feature extraction")
+                    tts_text_input = gr.Textbox(lines=5, label="Input text", placeholder="Enter text to synthesize...")
+                
+                tts_button = gr.Button("Synthesize", variant="primary")
+            tts_output_audio = gr.Audio(label="Synthesized Audio", interactive=False)
+    
     tts_button.click(
-        fn=audio2video,  # 추후 TTS 관련 함수로 대체 필요
-        inputs=[tts_text_input, tts_ref_img, None,
-                tts_size_slider, tts_step_slider, tts_length, tts_seed], 
-        outputs=[tts_output_video, tts_ref_img]
+        fn=tts_function,
+        inputs=[tts_input_audio, tts_text_input], 
+        outputs=[tts_output_audio]
     )
     
-demo.launch()
+demo.launch(debug=True)