app.py

import streamlit as st
import openai
from openai import OpenAI
import os
import base64
import cv2
from moviepy.editor import VideoFileClip

# documentation
# 1. Cookbook:  https://cookbook.openai.com/examples/gpt4o/introduction_to_gpt4o
# 2. Configure your Project and Orgs to limit/allow Models:  https://platform.openai.com/settings/organization/general
# 3. Watch your Billing!  https://platform.openai.com/settings/organization/billing/overview


# Set API key and organization ID from environment variables
openai.api_key = os.getenv('OPENAI_API_KEY')
openai.organization = os.getenv('OPENAI_ORG_ID')
client = OpenAI(api_key= os.getenv('OPENAI_API_KEY'), organization=os.getenv('OPENAI_ORG_ID'))

# Define the model to be used
#MODEL = "gpt-4o"
MODEL = "gpt-4o-2024-05-13"

def process_text():
    text_input = st.text_input("Enter your text:")
    if text_input:
        completion = client.chat.completions.create(
            model=MODEL,
            messages=[
                {"role": "system", "content": "You are a helpful assistant. Help me with my math homework!"},
                {"role": "user", "content": f"Hello! Could you solve {text_input}?"}
            ]
        )
        st.write("Assistant: " + completion.choices[0].message.content)

def process_image(image_input):
    if image_input:
        base64_image = base64.b64encode(image_input.read()).decode("utf-8")
        response = client.chat.completions.create(
            model=MODEL,
            prompt=f"You are a helpful assistant that responds in Markdown. Help me with my math homework! What's the area of the triangle? [image: data:image/png;base64,{base64_image}]",
            max_tokens=100,
            temperature=0.5,
        )
        st.markdown(response.choices[0].text.strip())

def process_audio(audio_input):
    if audio_input:
        transcription = openai.Audio.transcriptions.create(
            model="whisper-1",
            file=audio_input,
        )
        response = openai.Completion.create(
            model=MODEL,
            prompt=f"You are generating a transcript summary. Create a summary of the provided transcription. Respond in Markdown. The audio transcription is: {transcription['text']}",
            max_tokens=100,
            temperature=0.5,
        )
        st.markdown(response.choices[0].text.strip())

def process_video(video_input):
    if video_input:
        base64Frames, audio_path = process_video_frames(video_input)
        transcription = openai.Audio.transcriptions.create(
            model="whisper-1",
            file=open(audio_path, "rb"),
        )
        frames_text = " ".join([f"[image: data:image/jpg;base64,{frame}]" for frame in base64Frames])
        response = openai.Completion.create(
            model=MODEL,
            prompt=f"You are generating a video summary. Create a summary of the provided video and its transcript. Respond in Markdown. These are the frames from the video. {frames_text} The audio transcription is: {transcription['text']}",
            max_tokens=500,
            temperature=0.5,
        )
        st.markdown(response.choices[0].text.strip())

def process_video_frames(video_path, seconds_per_frame=2):
    base64Frames = []
    base_video_path, _ = os.path.splitext(video_path.name)
    video = cv2.VideoCapture(video_path.name)
    total_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
    fps = video.get(cv2.CAP_PROP_FPS)
    frames_to_skip = int(fps * seconds_per_frame)
    curr_frame = 0
    while curr_frame < total_frames - 1:
        video.set(cv2.CAP_PROP_POS_FRAMES, curr_frame)
        success, frame = video.read()
        if not success:
            break
        _, buffer = cv2.imencode(".jpg", frame)
        base64Frames.append(base64.b64encode(buffer).decode("utf-8"))
        curr_frame += frames_to_skip
    video.release()
    audio_path = f"{base_video_path}.mp3"
    clip = VideoFileClip(video_path.name)
    clip.audio.write_audiofile(audio_path, bitrate="32k")
    clip.audio.close()
    clip.close()
    return base64Frames, audio_path

def main():
    st.title("Omni Demo")
    option = st.selectbox("Select an option", ("Text", "Image", "Audio", "Video"))
    if option == "Text":
        process_text()
    elif option == "Image":
        image_input = st.file_uploader("Upload an image", type=["jpg", "jpeg", "png"])
        process_image(image_input)
    elif option == "Audio":
        audio_input = st.file_uploader("Upload an audio file", type=["mp3", "wav"])
        process_audio(audio_input)
    elif option == "Video":
        video_input = st.file_uploader("Upload a video file", type=["mp4"])
        process_video(video_input)

if __name__ == "__main__":
    main()