gamora commited on
Commit
dde1cc7
·
1 Parent(s): 4f1458d
Files changed (8) hide show
  1. .gitignore +27 -0
  2. app.py +0 -0
  3. extract_metadata.py +134 -0
  4. find_steps.py +124 -0
  5. grok_analyze.py +135 -0
  6. requirements.txt +98 -0
  7. video_editing.py +138 -0
  8. video_editing_ffmpeg.py +90 -0
.gitignore ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ .env
2
+ /data/
3
+ shortsai-469312-ab95bf92b7fb.json
4
+
5
+ /env
6
+
7
+ /__pycache__
8
+
9
+ /whisperenv
10
+
11
+ temp_audio.wav
12
+
13
+ test_audio.wav
14
+
15
+ /16_aug
16
+ /21_aug
17
+
18
+ /data_flood
19
+ resources_info.json
20
+
21
+ /metadata
22
+
23
+ .gradio/
24
+
25
+ .DS_Store
26
+
27
+ /edited_videos/
app.py ADDED
File without changes
extract_metadata.py ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from scenedetect import VideoManager, SceneManager
2
+ from scenedetect.detectors import ContentDetector
3
+ import cv2
4
+ import os
5
+ import numpy as np
6
+ import ffmpeg
7
+ from moviepy.editor import VideoFileClip
8
+ import time
9
+ from utils import read_files
10
+ from grok_analyze import analyze_image_with_grok_f
11
+ import json
12
+ import random
13
+ import string
14
+ def extract_scenes_scenedetect(video_path, threshold=30.0, min_scene_duration=5.0):
15
+
16
+ vid_obj={"type":"video", "path":video_path}
17
+
18
+ vid_obj["scenes"]=[ ]
19
+
20
+ # Initialize video and scene manager
21
+ video = VideoManager([video_path])
22
+ scene_manager = SceneManager()
23
+ scene_manager.add_detector(ContentDetector(threshold=threshold))
24
+
25
+ # Detect scenes
26
+ video.start()
27
+ scene_manager.detect_scenes(video)
28
+ scene_list = scene_manager.get_scene_list()
29
+
30
+ # Get video FPS for time calculations
31
+ cap = cv2.VideoCapture(video_path)
32
+ fps = cap.get(cv2.CAP_PROP_FPS)
33
+ min_frames = int(min_scene_duration * fps) # Convert 25 seconds to frames
34
+
35
+ # Filter scenes to ensure at least 25 seconds (min_frames) between them
36
+ filtered_scenes = []
37
+ last_frame = -min_frames # Allow the first scene to start at frame 0
38
+ for scene in scene_list:
39
+ print(scene)
40
+ start_frame = scene[0].get_frames()
41
+ if start_frame >= last_frame + min_frames:
42
+ filtered_scenes.append(scene)
43
+ last_frame = start_frame
44
+
45
+ print(f"Total scenes detected: {len(scene_list)}")
46
+ print(f"Filtered scenes (at least {min_scene_duration} seconds apart): {len(filtered_scenes)}")
47
+
48
+ if not filtered_scenes:
49
+ print("No scenes detected; extracting the first frame of the video.")
50
+ cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
51
+ ret, frame = cap.read()
52
+ if ret:
53
+ # Calculate timestamp for frame 0
54
+ timestamp = 0 / fps
55
+ minutes = int(timestamp // 60)
56
+ seconds = int(timestamp % 60)
57
+ milliseconds = int((timestamp % 1) * 1000)
58
+ print(f"First Frame - Timestamp: {minutes:02d}:{seconds:02d}.{milliseconds:03d}")
59
+
60
+ # Analyze frame and store scene info
61
+ description = analyze_image_with_grok_f(frame)
62
+ scene_info = {
63
+ "timestamp": f"{minutes:02d}:{seconds:02d}.{milliseconds:03d}",
64
+ "description": description
65
+ }
66
+ vid_obj["scenes"].append(scene_info)
67
+ else:
68
+ print("Failed to read the first frame of the video.")
69
+ cap.release()
70
+ video.release()
71
+ return vid_obj
72
+ else:
73
+ # Extract frames from filtered scenes
74
+ for i, scene in enumerate(filtered_scenes):
75
+ # Calculate and format timestamp
76
+ start_frame = scene[0].get_frames()
77
+ timestamp = start_frame / fps
78
+ minutes = int(timestamp // 60)
79
+ seconds = int(timestamp % 60)
80
+ milliseconds = int((timestamp % 1) * 1000)
81
+ print(f"Scene {i:04d} - Timestamp: {minutes:02d}:{seconds:02d}.{milliseconds:03d}")
82
+
83
+ # Seek to the start of the scene
84
+ cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame)
85
+ ret, frame = cap.read()
86
+ description=analyze_image_with_grok_f(frame)
87
+ scene={"timestamp":f"{minutes:02d}:{seconds:02d}.{milliseconds:03d}", "description":description}
88
+ vid_obj["scenes"].append(scene)
89
+
90
+ # if ret:
91
+ # output_path = os.path.join(output_folder, f"scene_{i:04d}.jpg")
92
+ # cv2.imwrite(output_path, frame)
93
+ print(vid_obj)
94
+ cap.release()
95
+ video.release()
96
+
97
+ return vid_obj
98
+
99
+
100
+ def get_metadata(all_files):
101
+
102
+ all_files_info=[]
103
+ for file_path in all_files:
104
+ print(file_path)
105
+ if file_path.split(".")[-1] in ["MOV","mp4"]:
106
+ vid_info=extract_scenes_scenedetect(file_path)
107
+ all_files_info.append(vid_info)
108
+
109
+ random_string = ''.join(random.choices(string.ascii_lowercase + string.digits, k=8))
110
+ file_name = f"all_files_info_{random_string}.json"
111
+
112
+ with open(file_name, "w") as file:
113
+ json.dump(all_files_info, file, indent=4)
114
+
115
+ return all_files_info,file_name
116
+
117
+
118
+
119
+
120
+ if __name__ == "__main__":
121
+ all_files_info=[]
122
+ dir_path="/Users/georgia.bucea/products/ShortsAI/21_aug"
123
+ all_files=read_files(dir_path)
124
+
125
+ for file in all_files:
126
+ file_path=dir_path+"/"+file
127
+ if file_path.split(".")[-1] in ["MOV","mp4"]:
128
+ vid_info=extract_scenes_scenedetect(file_path)
129
+ all_files_info.append(vid_info)
130
+
131
+ with open("all_files_info.json", "w") as file:
132
+ json.dump(all_files_info, file, indent=4)
133
+
134
+
find_steps.py ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from sentence_transformers import SentenceTransformer, util
2
+
3
+ import json
4
+ from datetime import datetime
5
+ # Function to read and parse the JSON file
6
+ def find_next_step(data, current_context):
7
+ try:
8
+ # Read the JSON file
9
+ # with open(file_path, 'r') as file:
10
+ # data = json.load(file)
11
+
12
+ # Initialize counters
13
+ total_videos = len(data)
14
+ total_scenes = 0
15
+ max_score=0.0
16
+ step={
17
+ "path" :"",
18
+ "start":"",
19
+ "end": "",
20
+ }
21
+ # Iterate through each video entry
22
+ print(f"Processing {total_videos} videos...\n")
23
+ for index, video in enumerate(data, 1):
24
+ video_type = video.get('type', 'Unknown')
25
+ video_path = video.get('path', 'No path provided')
26
+ scenes = video.get('scenes', [])
27
+ scene_count = len(scenes)
28
+ total_scenes += scene_count
29
+
30
+
31
+ # Iterate through each scene in the video
32
+ for scene_index, scene in enumerate(scenes, 1):
33
+ timestamp = scene.get('timestamp', 'No timestamp')
34
+ description = scene.get('description', 'No description')
35
+
36
+ time_obj = datetime.strptime(timestamp, "%M:%S.%f")
37
+ start = time_obj.minute * 60 + time_obj.second + time_obj.microsecond / 1_000_000
38
+ score=compare_phrases(description,current_context)
39
+
40
+ if score > max_score:
41
+ if scene_index+1<len(scenes):
42
+ next_scene=scenes[scene_index+1]
43
+ next_timestamp=next_scene.get('timestamp', 'No timestamp')
44
+ time_obj = datetime.strptime(next_timestamp, "%M:%S.%f")
45
+
46
+ stop = time_obj.minute * 60 + time_obj.second + time_obj.microsecond / 1_000_000
47
+ step={
48
+ "path" :video_path,
49
+ "start":start,
50
+ "end": stop,
51
+ "description":description
52
+ }
53
+ else:
54
+ step={
55
+ "path" :video_path,
56
+ "start":start,
57
+ "end": "",
58
+ "description":description
59
+ }
60
+
61
+ max_score=score
62
+
63
+ if "Error" in description:
64
+ print("[Note: This scene contains an error]")
65
+
66
+ # Print overall summary
67
+ print(f"Summary:")
68
+ print(f" Total Videos: {total_videos}")
69
+ print(f" Total Scenes: {total_scenes}")
70
+
71
+ except FileNotFoundError:
72
+ print(f"Error: File '{file_path}' not found.")
73
+ except json.JSONDecodeError:
74
+ print("Error: Invalid JSON format.")
75
+ except Exception as e:
76
+ print(f"Error: An unexpected error occurred: {str(e)}")
77
+
78
+ return step
79
+
80
+
81
+
82
+ def compare_phrases(phrase1,phrase2="Passing around a dj mixing a song"):
83
+ model = SentenceTransformer('all-MiniLM-L6-v2',local_files_only=True)
84
+ # Get embeddings
85
+ embedding1 = model.encode(phrase1, convert_to_tensor=True)
86
+ embedding2 = model.encode(phrase2, convert_to_tensor=True)
87
+
88
+ # Compute cosine similarity
89
+ similarity_score = util.cos_sim(embedding1, embedding2)[0][0]
90
+
91
+
92
+ # print(f"Similarity score: {similarity_score:.4f}")
93
+ # print(phrase1)
94
+ return similarity_score
95
+
96
+
97
+
98
+ def find_all_steps(data, context):
99
+
100
+ context = context.replace("\n", "").lower()
101
+ all_actions=context.split(".")
102
+ steps=[]
103
+ for action in all_actions:
104
+ step=find_next_step(data,action)
105
+ steps.append(step)
106
+ return steps
107
+
108
+
109
+
110
+
111
+
112
+ if __name__ == "__main__":
113
+ file_path = '/Users/georgia.bucea/products/ShortsAI/all_files_metadata.json' # Update this with the actual file path
114
+
115
+ context = """I am walking around San Francisco and passing around the big white building. I am walking around a music band and DJ mixing a song. I am showing a demo on the phone, I am explaining the demo on the mobile phone"""
116
+ # Read the JSON file
117
+ with open(file_path, 'r') as file:
118
+ data = json.load(file)
119
+
120
+ steps=find_all_steps(data,context)
121
+ print(steps)
122
+
123
+
124
+
grok_analyze.py ADDED
@@ -0,0 +1,135 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import base64
3
+ from xai_sdk import Client
4
+ from xai_sdk.chat import user, image
5
+ from dotenv import load_dotenv
6
+ import cv2
7
+ import json
8
+ load_dotenv()
9
+
10
+ def encode_image_to_base64_f(frame):
11
+ """
12
+ Encode an OpenCV frame (NumPy array) to a base64 string.
13
+
14
+ Args:
15
+ frame (np.ndarray): OpenCV frame in BGR format.
16
+
17
+ Returns:
18
+ str: Base64-encoded string of the image with data URL prefix, or None if encoding fails.
19
+ """
20
+ try:
21
+ # Convert BGR frame to JPEG-encoded bytes
22
+ _, buffer = cv2.imencode('.jpg', frame)
23
+ # Encode to base64 and convert to string
24
+ encoded_string = base64.b64encode(buffer).decode('utf-8')
25
+ # Prefix with data URL format for JPEG images
26
+ return f"data:image/jpeg;base64,{encoded_string}"
27
+ except Exception as e:
28
+ print(f"Error encoding frame to base64: {str(e)}")
29
+ return None
30
+
31
+ def analyze_image_with_grok_f( frame):
32
+ """
33
+ Analyze an image using Grok via xAI API.
34
+
35
+ Args:
36
+ api_key (str): xAI API key.
37
+ image_path (str): Path to the local image file.
38
+
39
+ Returns:
40
+ str: Grok's analysis of the image or error message.
41
+ """
42
+ # Initialize xAI client
43
+ api_key = os.getenv("X_AI")
44
+ client = Client(
45
+ api_key=api_key,
46
+ )
47
+
48
+ # Encode image to base64
49
+ base64_image = encode_image_to_base64_f(frame)
50
+ if not base64_image:
51
+ return "Error: Could not encode image. Check if the file exists and is valid."
52
+
53
+ # Create chat session
54
+ chat = client.chat.create(model="grok-4-0709") # Use vision-capable model
55
+
56
+ # Append user message with base64-encoded image
57
+ chat.append(
58
+ user(
59
+ "Write a short and concise description of the image",
60
+ image(base64_image) # Pass base64-encoded string
61
+ )
62
+ )
63
+
64
+ try:
65
+ response = chat.sample()
66
+ return response.content
67
+ except Exception as e:
68
+ return f"Error during API call: {str(e)}"
69
+
70
+ def get_story_with_grok(context,current_context):
71
+ # Initialize xAI client
72
+ api_key = os.getenv("X_AI")
73
+ client = Client(
74
+ api_key=api_key,
75
+ )
76
+
77
+ # Create chat session
78
+ chat = client.chat.create(model="grok-4-0709") # Use vision-capable model
79
+ print(current_context)
80
+ if current_context!="":
81
+ # Append user message with base64-encoded image
82
+ chat.append(
83
+ user(
84
+ "Based on the following context, and the phrase: "+current_context+ " generate a random story with no more than 4 scenes. The story should be given as JSON, with the scenes in the following format: path:file_path,start:,end:mdescription. Do not include a title, a main description, or anything else besides the scenes in the story as JSON format. " + str(context),
85
+ )
86
+ )
87
+ else:
88
+ chat.append(
89
+ user(
90
+ "Based on the following context, generate a random story with no more than 4 scenes. The story should be given as JSON, with the scenes in the following format: path:file_path,start:,end:mdescription. Do not include a title, a main description, or anything else besides the scenes in the story as JSON format. " + str(context),
91
+ )
92
+ )
93
+
94
+ try:
95
+ response = chat.sample()
96
+ data= response.content
97
+ video_data = json.loads(data)
98
+ return video_data
99
+ except Exception as e:
100
+ return f"Error during API call: {str(e)}"
101
+
102
+
103
+ def main():
104
+ # Your xAI API key
105
+ # Path to your image
106
+ # image_path = "/Users/georgia.bucea/products/ShortsAI/frames_scenesdetect/scene_0000.jpg"
107
+
108
+ # # Verify image path exists
109
+ # if not os.path.exists(image_path):
110
+ # print(f"Error: Image file not found at {image_path}")
111
+ # return
112
+
113
+ # print("Analyzing image...")
114
+ # analysis = analyze_image_with_grok_f( image_path)
115
+ # print("Grok's Analysis:")
116
+ # print(analysis)
117
+
118
+ context_path="/Users/georgia.bucea/products/ShortsAI/all_files_metadata.json"
119
+ with open(context_path, 'r') as file:
120
+
121
+ data = json.load(file)
122
+ print("here we are")
123
+ print(get_story_with_grok(data))
124
+
125
+
126
+
127
+ if __name__ == "__main__":
128
+ main()
129
+
130
+
131
+
132
+
133
+
134
+
135
+
requirements.txt ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ aiofiles==24.1.0
2
+ aiohappyeyeballs==2.6.1
3
+ aiohttp==3.12.15
4
+ aiosignal==1.4.0
5
+ annotated-types==0.7.0
6
+ anyio==4.10.0
7
+ attrs==25.3.0
8
+ Brotli==1.1.0
9
+ certifi==2025.8.3
10
+ charset-normalizer==3.4.3
11
+ click==8.2.1
12
+ decorator==4.4.2
13
+ fastapi==0.116.1
14
+ ffmpeg==1.4
15
+ ffmpeg-python==0.2.0
16
+ ffmpy==0.6.1
17
+ filelock==3.19.1
18
+ frozenlist==1.7.0
19
+ fsspec==2025.9.0
20
+ future==1.0.0
21
+ gradio==5.44.1
22
+ gradio_client==1.12.1
23
+ groovy==0.1.2
24
+ grpcio==1.74.0
25
+ h11==0.16.0
26
+ hf-xet==1.1.9
27
+ httpcore==1.0.9
28
+ httpx==0.28.1
29
+ huggingface-hub==0.34.4
30
+ idna==3.10
31
+ imageio==2.37.0
32
+ imageio-ffmpeg==0.6.0
33
+ importlib_metadata==8.7.0
34
+ Jinja2==3.1.6
35
+ joblib==1.5.2
36
+ markdown-it-py==4.0.0
37
+ MarkupSafe==3.0.2
38
+ mdurl==0.1.2
39
+ moviepy==1.0.3
40
+ mpmath==1.3.0
41
+ multidict==6.6.4
42
+ networkx==3.5
43
+ numpy==2.2.6
44
+ opencv-python==4.12.0.88
45
+ opentelemetry-api==1.36.0
46
+ opentelemetry-sdk==1.36.0
47
+ opentelemetry-semantic-conventions==0.57b0
48
+ orjson==3.11.3
49
+ packaging==25.0
50
+ pandas==2.3.2
51
+ pillow==11.3.0
52
+ pillow_heif==1.1.0
53
+ platformdirs==4.4.0
54
+ proglog==0.1.12
55
+ propcache==0.3.2
56
+ protobuf==6.32.0
57
+ pydantic==2.11.7
58
+ pydantic_core==2.33.2
59
+ pydub==0.25.1
60
+ Pygments==2.19.2
61
+ python-dateutil==2.9.0.post0
62
+ python-dotenv==1.1.1
63
+ python-multipart==0.0.20
64
+ pytz==2025.2
65
+ PyYAML==6.0.2
66
+ regex==2025.9.1
67
+ requests==2.32.5
68
+ rich==14.1.0
69
+ ruff==0.12.11
70
+ safehttpx==0.1.6
71
+ safetensors==0.6.2
72
+ scenedetect==0.6.7
73
+ scikit-learn==1.7.1
74
+ scipy==1.16.1
75
+ semantic-version==2.10.0
76
+ sentence-transformers==5.1.0
77
+ setuptools==80.9.0
78
+ shellingham==1.5.4
79
+ six==1.17.0
80
+ sniffio==1.3.1
81
+ starlette==0.47.3
82
+ sympy==1.14.0
83
+ threadpoolctl==3.6.0
84
+ tokenizers==0.22.0
85
+ tomlkit==0.13.3
86
+ torch==2.8.0
87
+ tqdm==4.67.1
88
+ transformers==4.56.0
89
+ typer==0.17.3
90
+ typing-inspection==0.4.1
91
+ typing_extensions==4.15.0
92
+ tzdata==2025.2
93
+ urllib3==2.5.0
94
+ uvicorn==0.35.0
95
+ websockets==15.0.1
96
+ xai-sdk==1.1.0
97
+ yarl==1.20.1
98
+ zipp==3.23.0
video_editing.py ADDED
@@ -0,0 +1,138 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from moviepy.editor import VideoFileClip, concatenate_videoclips,ColorClip,CompositeVideoClip
2
+ from find_steps import find_all_steps
3
+ import json
4
+ import random
5
+ import string
6
+ step={
7
+ "path" :"/Users/georgia.bucea/products/ShortsAI/21_aug/IMG_8299.MOV",
8
+ "start":0.0,
9
+ "end": '',
10
+ }
11
+
12
+ def crop_video(step, target_resolution=(1080, 1920), target_fps=30):
13
+ """
14
+ Crop a video clip to the specified start and end times, and resize to target resolution
15
+ while maintaining the original aspect ratio with padding if necessary.
16
+
17
+ Args:
18
+ step (dict): Dictionary with 'path', 'start', and 'end' keys.
19
+ target_resolution (tuple): Desired output resolution (width, height).
20
+ target_fps (int): Desired frame rate for the output video.
21
+
22
+ Returns:
23
+ VideoClip: Cropped and resized video clip with preserved audio.
24
+ """
25
+ video = VideoFileClip(step["path"])
26
+
27
+ # Crop the video based on start and end times
28
+ if step.get("end"):
29
+ video = video.subclip(step["start"], step["end"])
30
+ else:
31
+ video = video.subclip(step["start"])
32
+
33
+ # Set consistent frame rate
34
+ video = video.set_fps(target_fps)
35
+
36
+ # Calculate target aspect ratio
37
+ target_width, target_height = target_resolution
38
+ target_aspect = target_width / target_height
39
+ video_aspect = video.w / video.h
40
+
41
+ # Resize or pad to match target resolution while preserving aspect ratio
42
+ if abs(video_aspect - target_aspect) > 0.01: # Allow small tolerance
43
+ print("I'm here in abs")
44
+ if video_aspect > target_aspect:
45
+ # Video is wider than target: scale to match height, add black bars on sides
46
+ print("I'm here in video aspect >")
47
+ new_width = int(target_height * video_aspect)
48
+ video = video.resize(height=target_height)
49
+ # Create a background clip with black padding
50
+ background = ColorClip(size=(target_width, target_height), color=(0, 0, 0))
51
+ video = video.set_position(("center", "center")).on_color(size=(target_width, target_height), color=(0, 0, 0))
52
+ else:
53
+ print("I'm here in video aspect")
54
+ # Video is taller than target: scale to match width, add black bars on top/bottom
55
+ new_height = int(target_width / video_aspect)
56
+ video = video.resize(width=target_width)
57
+ # Create a background clip with black padding
58
+ background = ColorClip(size=(target_width, target_height), color=(0, 0, 0))
59
+ video = video.set_position(("center", "center")).on_color(size=(target_width, target_height), color=(0, 0, 0))
60
+ else:
61
+ print("I'm here in video aspect")
62
+ # Aspect ratio matches, resize directly
63
+ video = video.resize(target_resolution)
64
+
65
+ # Ensure audio is preserved
66
+ if video.audio is None:
67
+ print(f"Warning: No audio in {step['path']}")
68
+
69
+ return video
70
+
71
+ def get_final_video(subclips, aspect_ratio="9:16"):
72
+ """
73
+ Concatenate video clips and save the final video with consistent resolution and audio.
74
+
75
+ Args:
76
+ subclips (list): List of VideoClip objects.
77
+ aspect_ratio (str): Desired aspect ratio (e.g., '9:16').
78
+
79
+ Returns:
80
+ str: Path to the saved video file.
81
+ """
82
+ # Parse aspect ratio
83
+ width_ratio, height_ratio = map(int, aspect_ratio.split(":"))
84
+ target_aspect = width_ratio / height_ratio
85
+
86
+ # Set target resolution (e.g., 1080x1920 for 9:16)
87
+ target_height = 1920 # Standard for vertical videos
88
+ target_width = int(target_height * target_aspect)
89
+ target_resolution = (target_width, target_height)
90
+
91
+ # Concatenate clips
92
+ final_clip = concatenate_videoclips(subclips, method="compose")
93
+
94
+ # Generate unique filename
95
+ random_string = ''.join(random.choices(string.ascii_lowercase + string.digits, k=8))
96
+ file_name = f"edited_videos/video_edited{random_string}.mp4" # Changed to .mp4 for compatibility
97
+
98
+ # Write the final video file
99
+ final_clip.write_videofile(
100
+ file_name,
101
+ codec='libx264',
102
+ audio_codec='aac',
103
+ fps=30, # Consistent frame rate
104
+ preset='medium',
105
+ ffmpeg_params=['-pix_fmt', 'yuv420p', '-aspect', aspect_ratio], # Ensure compatibility with most players
106
+ verbose=False,
107
+ temp_audiofile=f"temp_audio_{random_string}.m4a", # Explicit temp audio file
108
+ )
109
+
110
+ # Close clips to free memory
111
+ final_clip.close()
112
+ # for clip in sub:
113
+ # clip.close()
114
+
115
+ return file_name
116
+
117
+
118
+
119
+
120
+
121
+ if __name__ == "__main__":
122
+ file_path = '/Users/georgia.bucea/products/ShortsAI/all_files_metadata.json' # Update this with the actual file path
123
+
124
+ context = """I am putting stuff in a car, with pipes and a guy in a red shirt. I am walking past a band with a DJ. I am explaining something on the phone"""
125
+
126
+ with open(file_path, 'r') as file:
127
+ data = json.load(file)
128
+
129
+ steps=find_all_steps(data,context)
130
+ all_videos=[]
131
+ for step in steps:
132
+ v=crop_video(step)
133
+ all_videos.append(v)
134
+
135
+ # v=crop_video(step)
136
+
137
+ get_final_video(all_videos)
138
+
video_editing_ffmpeg.py ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import ffmpeg
2
+ import random
3
+ import string
4
+ # Define target resolution
5
+ target_width = 1080
6
+ target_height = 1920
7
+
8
+
9
+
10
+ def concatenate_videos(files_timestamps):
11
+ processed_streams = []
12
+ for file_info in files_timestamps:
13
+ file_path = file_info["path"]
14
+ start_time = file_info["start"]
15
+ end_time = file_info["end"]
16
+
17
+ try:
18
+ if end_time == "":
19
+ # Create input stream from start_time to end of video
20
+ stream = ffmpeg.input(file_path, ss=start_time)
21
+ else:
22
+ # Create input stream with trimming
23
+ stream = ffmpeg.input(file_path, ss=start_time, t=end_time - start_time)
24
+ video = stream.video
25
+ audio = stream.audio
26
+
27
+ # Get video metadata to calculate aspect ratio
28
+ probe = ffmpeg.probe(file_path)
29
+ video_stream = next(s for s in probe['streams'] if s['codec_type'] == 'video')
30
+ video_width = int(video_stream['width'])
31
+ video_height = int(video_stream['height'])
32
+ video_aspect = video_width / video_height
33
+ target_aspect = target_width / target_height
34
+
35
+ print(f"Processing {file_path}: resolution={video_width}x{video_height}, aspect={video_aspect:.2f}, trimmed from {start_time}s to {end_time}s")
36
+
37
+ # Scale the video to fit within target resolution while preserving aspect ratio
38
+ if abs(video_aspect - target_aspect) > 0.01: # Allow small tolerance
39
+ if video_aspect > target_aspect:
40
+ # Video is wider: scale to target height, ensure width <= target_width
41
+ video = video.filter('scale', f'min({target_width},iw*({target_height}/ih))', target_height, force_original_aspect_ratio='decrease')
42
+ # Pad to target resolution, align to left for right-side padding
43
+ video = video.filter('pad', target_width, target_height, 0, '(oh-ih)/2', color='black')
44
+ else:
45
+ # Video is taller: scale to target width, ensure height <= target_height
46
+ video = video.filter('scale', target_width, f'min({target_height},ih*({target_width}/iw))', force_original_aspect_ratio='decrease')
47
+ # Pad to target resolution, center vertically
48
+ video = video.filter('pad', target_width, target_height, '(ow-iw)/2', '(oh-ih)/2', color='black')
49
+ else:
50
+ # Aspect ratio matches: scale directly to target resolution
51
+ video = video.filter('scale', target_width, target_height, force_original_aspect_ratio='decrease')
52
+
53
+ # Ensure consistent frame rate
54
+ video = video.filter('fps', fps=30)
55
+
56
+ print(f"After scaling {file_path}: target resolution={target_width}x{target_height}")
57
+
58
+ processed_streams.append(video)
59
+ processed_streams.append(audio)
60
+
61
+ except ffmpeg.Error as e:
62
+ print(f"Error processing {file_path}: {e.stderr.decode()}")
63
+ continue
64
+ except StopIteration:
65
+ print(f"Error: No video stream found in {file_path}")
66
+ continue
67
+ except Exception as e:
68
+ print(f"Unexpected error processing {file_path}: {str(e)}")
69
+ continue
70
+
71
+ # Check if we have valid streams to concatenate
72
+ if not processed_streams:
73
+ print("Error: No valid streams to concatenate")
74
+ exit(1)
75
+
76
+ # Concatenate all video and audio streams
77
+ joined = ffmpeg.concat(*processed_streams, v=1, a=1).node
78
+
79
+ # Output the final video
80
+ try:
81
+ random_string = ''.join(random.choices(string.ascii_lowercase + string.digits, k=8))
82
+ file_name = f"edited_videos/video_edited{random_string}.mp4" # Changed to .mp4 for compatibility
83
+
84
+ ffmpeg.output(joined[0], joined[1], file_name, **{'c:v': 'libx264', 'c:a': 'aac'}, preset='fast').run(overwrite_output=True)
85
+ print("Video concatenation successful")
86
+ return file_name
87
+ except ffmpeg.Error as e:
88
+ print(f"Error during concatenation: {e.stderr.decode()}")
89
+
90
+