import os import base64 from xai_sdk import Client from xai_sdk.chat import user, image from dotenv import load_dotenv import cv2 import json from create_captions import create_caps load_dotenv() def encode_image_to_base64_f(frame): """ Encode an OpenCV frame (NumPy array) to a base64 string. Args: frame (np.ndarray): OpenCV frame in BGR format. Returns: str: Base64-encoded string of the image with data URL prefix, or None if encoding fails. """ try: # Convert BGR frame to JPEG-encoded bytes _, buffer = cv2.imencode('.jpg', frame) # Encode to base64 and convert to string encoded_string = base64.b64encode(buffer).decode('utf-8') # Prefix with data URL format for JPEG images return f"data:image/jpeg;base64,{encoded_string}" except Exception as e: print(f"Error encoding frame to base64: {str(e)}") return None def analyze_image_with_grok_f( frame): """ Analyze an image using Grok via xAI API. Args: api_key (str): xAI API key. image_path (str): Path to the local image file. Returns: str: Grok's analysis of the image or error message. """ # Initialize xAI client api_key = os.getenv("X_AI") client = Client( api_key=api_key, ) # Encode image to base64 base64_image = encode_image_to_base64_f(frame) if not base64_image: return "Error: Could not encode image. Check if the file exists and is valid." # Create chat session chat = client.chat.create(model="grok-4-0709") # Use vision-capable model # Append user message with base64-encoded image chat.append( user( "Write a short and concise description of the image", image(base64_image) # Pass base64-encoded string ) ) try: response = chat.sample() return response.content except Exception as e: return f"Error during API call: {str(e)}" def get_story_with_grok(metadata,current_context,dialog): # Initialize xAI client api_key = os.getenv("X_AI") client = Client( api_key=api_key, ) # Create chat session chat = client.chat.create(model="grok-4-0709") # Use vision-capable model print(current_context) if current_context!="": # Append user message with base64-encoded image chat.append( user( # "Based on the following context, the phrase: "+current_context+ " and the given dialog, generate an engaging random story relevant to the dialog with no more than 4 scenes.The description should be short engaging, relevant to the dialog, with less adjectives, with no more than three short sentences, in present tense and first person. The story should be given as JSON, with the scenes in the following format:scenes:[path:file_path,start:,end:,description:].The path is extracted from the metadata file. Do not include a title, a main description, or anything else besides the scenes in the story as JSON format. The metadata: " + str(metadata) + " And the following dialog: "+ dialog, "Based on the following metadata,the phrase: "+current_context+ ", and the given dialog, generate an engaging story with no more than 4 scenes. The story should be given as JSON, with the scenes in the following format:scenes:[path:file_path,start:,end:,description:].The path is extracted from the metadata file. Do not include a title, a main description, or anything else besides the scenes in the story as JSON format. The metadata: " + str(metadata) + " And the following dialog: "+ dialog, )) else: print("here") chat.append( user( # "Based on the following context and the given dialog, generate an engaging random story with no more than 4 scenes extracted randomly from the given metadata. The description should be short engaging, relevant to the given dialog,with less adjectives. with no more than three short sentences that take into account the transcripted dialog, in present tense and first person. The story should be given as JSON, with the scenes in the following format:scenes:[path:file_path,start:,end:,description:].The path is extracted from the metadata file. Do not include a title, a main description, or anything else besides the scenes in the story as JSON format. The metadata: " + str(metadata) + " And the following dialog: "+ dialog, "You are a vlogger, making videos about your life and your adventures. Based on the following metadata and the given dialog, generate an engaging story with no more than 4 scenes related to the metadata. Use less adjectives with short sentences. The story should be given as JSON, with the scenes in the following format:scenes:[path:file_path,start:,end:,description:].The path is extracted from the metadata file. Do not include a title, a main description, or anything else besides the scenes in the story as JSON format. The metadata: " + str(metadata) + " And the following dialog: "+ dialog, # "Based on the following metadata and the given dialog, generate an engaging story with all scenes in the metadata. The story should be given as JSON, with the scenes in the following format:scenes:[path:file_path,start:,end:,description:].The path is extracted from the metadata file. Do not include a title, a main description, or anything else besides the scenes in the story as JSON format. The metadata: " + str(metadata) + " And the following dialog: "+ dialog, ) ) try: response = chat.sample() data= response.content video_data = json.loads(data) print(video_data) return video_data['scenes'] except Exception as e: return f"Error during API call: {str(e)}" def create_caps_with_grok(steps): api_key = os.getenv("X_AI") client = Client( api_key=api_key, ) # chat = client.chat.create(model="grok-4-0709") for step in steps: chat = client.chat.create(model="grok-4-0709") chat.append( user( "You are a vlogger, making videos about your life. Based on the following description generate a short caption, with no more than three short sentences, less adjectives, in present tense, first person that is fit for a story. Do not include a title or anython else besides the final caption. the description: "+step["description"], )) try: response = chat.sample() data= response.content step["description"]=data except Exception as e: return f"Error during API call: {str(e)}" return steps def main(): context_path="/Users/georgia.bucea/products/ShortsAI/all_files_metadata.json" with open(context_path, 'r') as file: data = json.load(file) print("here we are") steps=get_story_with_grok(data,"") print(steps) steps=create_caps_with_grok(steps) print(steps) if __name__ == "__main__": main()