grok_analyze.py

import os
import base64

from xai_sdk import Client
from xai_sdk.chat import user, image
from dotenv import load_dotenv
import cv2
import json
from create_captions import create_caps
load_dotenv()

def encode_image_to_base64_f(frame):
    """
    Encode an OpenCV frame (NumPy array) to a base64 string.
    
    Args:
        frame (np.ndarray): OpenCV frame in BGR format.
        
    Returns:
        str: Base64-encoded string of the image with data URL prefix, or None if encoding fails.
    """
    try:
        # Convert BGR frame to JPEG-encoded bytes
        _, buffer = cv2.imencode('.jpg', frame)
        # Encode to base64 and convert to string
        encoded_string = base64.b64encode(buffer).decode('utf-8')
        # Prefix with data URL format for JPEG images
        return f"data:image/jpeg;base64,{encoded_string}"
    except Exception as e:
        print(f"Error encoding frame to base64: {str(e)}")
        return None

def analyze_image_with_grok_f( frame):
    """
    Analyze an image using Grok via xAI API.
    
    Args:
        api_key (str): xAI API key.
        image_path (str): Path to the local image file.
        
    Returns:
        str: Grok's analysis of the image or error message.
    """
    # Initialize xAI client
    api_key = os.getenv("X_AI")
    client = Client(
        api_key=api_key,
    )
    
    # Encode image to base64
    base64_image = encode_image_to_base64_f(frame)
    if not base64_image:
        return "Error: Could not encode image. Check if the file exists and is valid."
    
    # Create chat session
    chat = client.chat.create(model="grok-4-0709")  # Use vision-capable model
    
    # Append user message with base64-encoded image
    chat.append(
        user(
            "Write a short and concise description of the image",
            image(base64_image)  # Pass base64-encoded string
        )
    )
    
    try:
        response = chat.sample()
        return response.content
    except Exception as e:
        return f"Error during API call: {str(e)}"

def get_story_with_grok(metadata,current_context,dialog):
        # Initialize xAI client
    api_key = os.getenv("X_AI")
    client = Client(
        api_key=api_key,
    )
    # Create chat session
    chat = client.chat.create(model="grok-4-0709")  # Use vision-capable model
    print(current_context)
    if current_context!="":
    # Append user message with base64-encoded image
        chat.append(
            user(
                # "Based on the following context, the phrase: "+current_context+ " and the given dialog, generate an engaging random story relevant to the dialog with no more than 4 scenes.The description should be short engaging, relevant to the dialog, with less adjectives, with no more than three short sentences, in present tense and first person. The story should be given as JSON, with the scenes in the following format:scenes:[path:file_path,start:,end:,description:].The path is extracted from the metadata file. Do not include a title, a main description, or anything else besides the scenes in the story as JSON format. The metadata: " + str(metadata) + " And the following dialog: "+ dialog,
                  "Based on the following metadata,the phrase: "+current_context+ ", and the given dialog, generate an engaging story with no more than 4 scenes. The story should be given as JSON, with the scenes in the following format:scenes:[path:file_path,start:,end:,description:].The path is extracted from the metadata file. Do not include a title, a main description, or anything else besides the scenes in the story as JSON format. The metadata: " + str(metadata) + " And the following dialog: "+ dialog,

       ))
    else:
        print("here")
        chat.append(
            user(
                # "Based on the following context and the given dialog, generate an engaging random story with no more than 4 scenes extracted randomly from the given metadata. The description should be short engaging, relevant to the given dialog,with less adjectives. with no more than three short sentences that take into account the transcripted dialog, in present tense and first person. The story should be given as JSON, with the scenes in the following format:scenes:[path:file_path,start:,end:,description:].The path is extracted from the metadata file. Do not include a title, a main description, or anything else besides the scenes in the story as JSON format. The metadata: " + str(metadata) + " And the following dialog: "+ dialog,
           "You are a vlogger, making videos about your life and your adventures. Based on the following metadata and the given dialog, generate an engaging story with no more than 4 scenes related to the metadata. Use less adjectives with short sentences. The story should be given as JSON, with the scenes in the following format:scenes:[path:file_path,start:,end:,description:].The path is extracted from the metadata file. Do not include a title, a main description, or anything else besides the scenes in the story as JSON format. The metadata: " + str(metadata) + " And the following dialog: "+ dialog,
            # "Based on the following metadata and the given dialog, generate an engaging story with all scenes in the metadata. The story should be given as JSON, with the scenes in the following format:scenes:[path:file_path,start:,end:,description:].The path is extracted from the metadata file. Do not include a title, a main description, or anything else besides the scenes in the story as JSON format. The metadata: " + str(metadata) + " And the following dialog: "+ dialog,
            )
        )
        
    try:
        response = chat.sample()
        data= response.content
        video_data = json.loads(data)
        print(video_data)
        return video_data['scenes']
    except Exception as e:
        return f"Error during API call: {str(e)}"



def create_caps_with_grok(steps):
    api_key = os.getenv("X_AI")
    client = Client(
        api_key=api_key,
    )
    # chat = client.chat.create(model="grok-4-0709") 
    for step in steps:
        chat = client.chat.create(model="grok-4-0709") 
        chat.append(
            user(
                "You are a vlogger, making videos about your life. Based on the following description generate a short caption, with no more than three short sentences, less adjectives, in present tense, first person that is fit for a story. Do not include a title or anython else besides the final caption. the description: "+step["description"],
        ))
        try:
            response = chat.sample()
            data= response.content
            step["description"]=data
        except Exception as e:
            return f"Error during API call: {str(e)}"
    return steps



def main():

    context_path="/Users/georgia.bucea/products/ShortsAI/all_files_metadata.json"
    with open(context_path, 'r') as file:

        data = json.load(file)
        print("here we are")
        steps=get_story_with_grok(data,"")
        print(steps)
        steps=create_caps_with_grok(steps)
        print(steps)
        
    

if __name__ == "__main__":
    main()