fail to make narrative

app.py

@@ -6,16 +6,13 @@ from baseline_utils import (detect_text_in_image,
                             analyze_writer_image,
                             generate_video,
                             break_diary_to_scenes,
-                            scenes_caption,
-                            summarizer_for_audio,
-                            narration_generate)
+                            scenes_caption)
 import os
 
 # Load secrets from Hugging Face Spaces environment
 openai_api_key = os.getenv("OPENAI_API_KEY")
 google_service_account_info = json.loads(os.getenv("GOOGLE_SERVICE_ACCOUNT"))
 gemini_api_key = os.getenv("GEMINI_API_KEY")
-eleven_api_key = os.getenv("ELEVEN_API_KEY")
 
 # Initialize OpenAI
 openai.api_key = openai_api_key
@@ -26,7 +23,7 @@ def get_google_credentials():
     return service_account.Credentials.from_service_account_info(google_service_account_info)
 
 
-def process_images(diary_image, writer_image, audio_option):
+def process_images(diary_image, writer_image):
     # Save the file-like objects as image files
     diary_image_path = "temp_upload_images/temp_diary_image.png"
     writer_image_path = "temp_upload_images/temp_writer_image.png"
@@ -45,12 +42,8 @@ def process_images(diary_image, writer_image, audio_option):
     scene_list = [scene.strip() for scene in scenes.split("Scene")[1:]]
     scene_list = [scene.split(": ", 1)[1] for scene in scene_list]
 
-    # Generate the narration audio which is less than 10 second
-    # This will create a mp3 file for narration
-    narration_summarize = summarizer_for_audio(detected_text)
-    narration_generate(narration_summarize, eleven_api_key)
     # Generate the video based on the summaries
-    video_path = generate_video(scene_list, writer_summary, audio_option, fps=24)
+    video_path = generate_video(scene_list, writer_summary, fps=24)
 
     caption = scenes_caption(scene_list, openai_api_key)
 
@@ -58,9 +51,9 @@ def process_images(diary_image, writer_image, audio_option):
 
 
 # Define the Gradio interface
-def gradio_interface(diary_image, writer_image, audio_option):
+def gradio_interface(diary_image, writer_image):
     # Process the images and generate the video
-    video_paths, prompts = process_images(diary_image, writer_image, audio_option)
+    video_paths, prompts = process_images(diary_image, writer_image)
 
     # Return the paths and corresponding prompts
     return video_paths, prompts
@@ -75,12 +68,6 @@ with gr.Blocks() as interface:
         with gr.Column():
             diary_image_input = gr.Image(label="Upload your handwritten diary image", type="pil")
             writer_image_input = gr.Image(label="Upload a photo of the writer", type="pil")
-            # Add a radio button for selecting audio options
-            audio_option = gr.Radio(
-                ["Narration", "Meow"],
-                label="Choose Audio Option",
-                value="Narration"  # Default selection
-            )
             submit_button = gr.Button("Generate Video")
 
         # Right column for generated video and caption
@@ -91,7 +78,7 @@ with gr.Blocks() as interface:
     # Bind the submit button click to trigger the video generation and display
     submit_button.click(
         fn=gradio_interface,
-        inputs=[diary_image_input, writer_image_input, audio_option],
+        inputs=[diary_image_input, writer_image_input],
         outputs=[video_output, caption_output]
     )
 

baseline_utils.py

@@ -8,9 +8,6 @@ from diffusers.utils import export_to_video
 import os
 import spaces
 from moviepy.editor import VideoFileClip, AudioFileClip, concatenate_videoclips
-from elevenlabs import generate, play
-import requests
-from transformers import pipeline
 
 # Utilize the Google Cloud Vision API to recognize text in the
 # input input_images (diary input_images), https://cloud.google.com/vision.
@@ -123,7 +120,7 @@ def scenes_caption(scenes, api_key):
 
 
 @spaces.GPU
-def generate_video(scene_list, writer_description, audio_option, fps=24):  # Lower fps
+def generate_video(scene_list, writer_description, fps=24):  # Lower fps
 
     pipe = CogVideoXPipeline.from_pretrained(
         "THUDM/CogVideoX-5b",
@@ -164,14 +161,11 @@ def generate_video(scene_list, writer_description, audio_option, fps=24):  # Low
 
     # Concatenate the generated videos into a single video
     concatenated_video_path = "videos/combined_video.mp4"
-    if audio_option == "Narration":
-        concatenate_videos(video_paths, concatenated_video_path, audio_path="narration.mp3")
-    else:
-        concatenate_videos(video_paths, concatenated_video_path, audio_path="meow-meow-meow-tiktok.mp3")
+    concatenate_videos(video_paths, concatenated_video_path, audio_path="meow-meow-meow-tiktok.mp3")
     return concatenated_video_path
 
 
-def concatenate_videos(video_paths, output_path, audio_path):
+def concatenate_videos(video_paths, output_path, audio_path="meow-meow-meow-tiktok.mp3"):
     # Load each video file as a VideoFileClip
     clips = [VideoFileClip(video) for video in video_paths]
 
@@ -189,39 +183,3 @@ def concatenate_videos(video_paths, output_path, audio_path):
 
     # Write the concatenated video to a file
     final_clip.write_videofile(output_path, codec="libx264", audio_codec="aac")
-
-def narration_generate(input, api_key):
-    url = "https://api.elevenlabs.io/v1/text-to-speech/9BWtsMINqrJLrRacOk9x"
-    headers = {
-        "Accept": "audio/mpeg",
-        "Content-Type": "application/json",
-        "xi-api-key": api_key
-    }
-
-    data = {
-        "text": input,
-        "model_id": "eleven_monolingual_v1",
-        "voice_settings": {
-            "stability": 0.5,
-            "similarity_boost": 0.5
-        }
-    }
-
-    response = requests.post(url, json=data, headers=headers)
-    with open('narration.mp3', 'wb') as f:
-        for chunk in response.iter_content(chunk_size=1024):
-            if chunk:
-                f.write(chunk)
-
-def summarizer_for_audio(input_text):
-    summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
-
-    # Generate the summary
-    summary = summarizer(
-        input_text,
-        max_length=25,
-        min_length=20,
-        do_sample=False
-    )[0]["summary_text"]
-
-    return summary

requirements.txt

@@ -4,7 +4,8 @@ google-auth
 google-generativeai
 diffusers
 torch
+streamlit
 transformers
 accelerate
-SentencePiece
-moviepy
+moviepy
+SentencePiece