Tohru127 commited on
Commit
a6f791b
·
verified ·
1 Parent(s): d2ee5d9

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +287 -254
app.py CHANGED
@@ -1,9 +1,6 @@
1
  """
2
  360° Video Frame Extraction + 3D Reconstruction for Outdoor Scenes
3
-
4
- Two modes:
5
- 1. Quick Frame Extraction - Just get the frames (30-60s)
6
- 2. Full 3D Reconstruction - Extract frames + create 3D model (5-10 min)
7
  """
8
 
9
  import gradio as gr
@@ -19,34 +16,15 @@ from transformers import DPTForDepthEstimation, DPTImageProcessor
19
  import open3d as o3d
20
  import plotly.graph_objects as go
21
  import warnings
 
22
  warnings.filterwarnings('ignore')
23
 
24
- # ============================================================================
25
- # RESPONSIBLE USE GUIDELINES
26
- # ============================================================================
27
-
28
- RESPONSIBLE_AI_NOTICE = """
29
- ## ⚠️ Responsible Use Guidelines
30
-
31
- ### Privacy & Consent
32
- - **Do not upload videos containing identifiable people without consent**
33
- - **Do not use for surveillance or tracking**
34
-
35
- ### Ethical Use
36
- - For **educational, research, and creative purposes only**
37
-
38
- ### Data Usage
39
- - Videos processed locally, not stored on servers
40
- - You retain all rights to your content
41
-
42
- **By using this tool, you agree to these guidelines.**
43
- """
44
-
45
  # ============================================================================
46
  # MODEL LOADING
47
  # ============================================================================
48
 
49
  print("Loading depth estimation model...")
 
50
  try:
51
  dpt_processor = DPTImageProcessor.from_pretrained("Intel/dpt-large")
52
  dpt_model = DPTForDepthEstimation.from_pretrained("Intel/dpt-large")
@@ -54,6 +32,7 @@ try:
54
  dpt_model = dpt_model.cuda()
55
  print("✓ Using GPU")
56
  dpt_model.eval()
 
57
  print("✓ Model loaded!")
58
  except Exception as e:
59
  print(f"⚠️ Model loading failed: {e}")
@@ -66,39 +45,51 @@ except Exception as e:
66
 
67
  def extract_frames_from_360_video(video_path, frame_step=30, max_frames=150):
68
  """Extract frames from 360° video"""
69
- cap = cv2.VideoCapture(video_path)
70
-
71
- if not cap.isOpened():
72
- return [], None, 0, 0, "Error: Could not open video file"
73
-
74
- fps = cap.get(cv2.CAP_PROP_FPS)
75
- total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
76
-
77
- frames_dir = tempfile.mkdtemp()
78
- extracted_frames = []
79
- frame_count = 0
80
- saved_count = 0
81
-
82
- while cap.isOpened() and saved_count < max_frames:
83
- ret, frame = cap.read()
84
 
85
- if not ret:
86
- break
87
-
88
- if frame_count % frame_step == 0:
89
- frame_filename = os.path.join(frames_dir, f"frame_{saved_count:04d}.jpg")
90
- cv2.imwrite(frame_filename, frame, [cv2.IMWRITE_JPEG_QUALITY, 95])
91
- extracted_frames.append(frame_filename)
92
- saved_count += 1
 
 
 
 
 
 
 
 
 
 
 
93
 
94
- frame_count += 1
95
-
96
- cap.release()
97
-
98
- if len(extracted_frames) == 0:
99
- return [], None, fps, total_frames, "Error: No frames extracted"
100
-
101
- return extracted_frames, frames_dir, fps, total_frames, "Success"
 
 
 
 
 
 
 
 
 
 
 
 
 
102
 
103
  # ============================================================================
104
  # 3D RECONSTRUCTION
@@ -106,45 +97,45 @@ def extract_frames_from_360_video(video_path, frame_step=30, max_frames=150):
106
 
107
  def estimate_depth(image, processor, model):
108
  """Estimate depth for a single image"""
109
- inputs = processor(images=image, return_tensors="pt")
110
-
111
- if torch.cuda.is_available():
112
- inputs = {k: v.cuda() for k, v in inputs.items()}
113
-
114
- with torch.no_grad():
115
- outputs = model(**inputs)
116
- predicted_depth = outputs.predicted_depth
117
-
118
- prediction = torch.nn.functional.interpolate(
119
- predicted_depth.unsqueeze(1),
120
- size=image.shape[:2],
121
- mode="bicubic",
122
- align_corners=False,
123
- )
124
-
125
- depth = prediction.squeeze().cpu().numpy()
126
- depth = (depth - depth.min()) / (depth.max() - depth.min())
127
-
128
- return depth
 
 
 
 
129
 
130
  def depth_to_point_cloud(image, depth):
131
  """Convert depth map to 3D point cloud"""
132
  h, w = depth.shape
133
 
134
- # Create mesh grid
135
  x = np.linspace(0, w-1, w)
136
  y = np.linspace(0, h-1, h)
137
  xv, yv = np.meshgrid(x, y)
138
 
139
- # Flatten
140
  x_flat = xv.flatten()
141
  y_flat = yv.flatten()
142
  z_flat = depth.flatten()
143
 
144
- # Stack to 3D points
145
  points = np.stack([x_flat, y_flat, z_flat], axis=-1)
146
 
147
- # Get colors from image
148
  if len(image.shape) == 3:
149
  colors = image.reshape(-1, 3) / 255.0
150
  else:
@@ -152,100 +143,90 @@ def depth_to_point_cloud(image, depth):
152
 
153
  return points, colors
154
 
155
- def create_3d_model(frames, max_frames_for_3d=5, progress=gr.Progress()):
156
  """Create 3D model from extracted frames"""
157
- if dpt_model is None or dpt_processor is None:
158
- return None, None, "❌ Depth model not loaded"
159
-
160
- progress(0, desc="Starting 3D reconstruction...")
161
 
162
- all_points = []
163
- all_colors = []
164
-
165
- # Process subset of frames
166
- frames_to_process = frames[:max_frames_for_3d]
167
-
168
- for idx, frame_path in enumerate(frames_to_process):
169
- progress((idx + 1) / len(frames_to_process), desc=f"Processing frame {idx+1}/{len(frames_to_process)}...")
170
-
171
- # Load image
172
- img = cv2.imread(frame_path)
173
- img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
174
 
175
- # Resize for speed
176
- img_small = cv2.resize(img_rgb, (512, 256))
177
 
178
- # Estimate depth
179
- depth = estimate_depth(img_small, dpt_processor, dpt_model)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
180
 
181
- # Convert to point cloud
182
- points, colors = depth_to_point_cloud(img_small, depth)
 
 
 
 
183
 
184
- # Offset points by frame index to separate views
185
- points[:, 0] += idx * 600
186
 
187
- all_points.append(points)
188
- all_colors.append(colors)
189
-
190
- progress(0.9, desc="Combining point clouds...")
191
-
192
- # Combine all point clouds
193
- final_points = np.vstack(all_points)
194
- final_colors = np.vstack(all_colors)
195
-
196
- # Downsample if too large
197
- if len(final_points) > 100000:
198
- indices = np.random.choice(len(final_points), 100000, replace=False)
199
- final_points = final_points[indices]
200
- final_colors = final_colors[indices]
201
-
202
- progress(0.95, desc="Creating visualization...")
203
-
204
- # Create Plotly visualization
205
- fig = go.Figure(data=[go.Scatter3d(
206
- x=final_points[:, 0],
207
- y=final_points[:, 1],
208
- z=final_points[:, 2],
209
- mode='markers',
210
- marker=dict(
211
- size=1,
212
- color=final_colors,
213
- opacity=0.8
214
- )
215
- )])
216
-
217
- fig.update_layout(
218
- title="3D Reconstruction from 360° Video",
219
- scene=dict(
220
- xaxis_title="X",
221
- yaxis_title="Y",
222
- zaxis_title="Depth",
223
- aspectmode='data'
224
- ),
225
- width=800,
226
- height=600
227
- )
228
-
229
- # Create PLY file
230
- progress(0.98, desc="Saving 3D model...")
231
- ply_path = os.path.join(tempfile.gettempdir(), f"3d_model_{datetime.now().strftime('%Y%m%d_%H%M%S')}.ply")
232
-
233
- pcd = o3d.geometry.PointCloud()
234
- pcd.points = o3d.utility.Vector3dVector(final_points)
235
- pcd.colors = o3d.utility.Vector3dVector(final_colors)
236
- o3d.io.write_point_cloud(ply_path, pcd)
237
-
238
- progress(1.0, desc="Done!")
239
-
240
- return fig, ply_path, f"✅ 3D model created with {len(final_points):,} points"
241
 
242
  # ============================================================================
243
  # PACKAGE CREATION
244
  # ============================================================================
245
 
246
- def create_readme(video_info):
247
- """Create README for frame package"""
248
- return f"""360° OUTDOOR PHOTOGRAMMETRY PACKAGE
 
 
 
249
  Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
250
 
251
  VIDEO INFO:
@@ -256,60 +237,78 @@ VIDEO INFO:
256
  METASHAPE WORKFLOW:
257
  1. Import Photos
258
  2. Set Camera Type to "Spherical"
259
- 3. Align Photos (High accuracy)
260
  4. Build Dense Cloud
261
  5. Build Mesh
262
  6. Build Texture
263
 
264
- SOFTWARE: Agisoft Metashape ($179) recommended
265
  Good luck! 🌍📸
266
  """
267
-
268
- def create_download_package(frames_dir, video_info):
269
- """Create ZIP with frames"""
270
- zip_path = os.path.join(tempfile.gettempdir(), f"360_frames_{datetime.now().strftime('%Y%m%d_%H%M%S')}.zip")
271
-
272
- with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
273
- readme_content = create_readme(video_info)
274
- readme_path = os.path.join(tempfile.gettempdir(), "README.txt")
275
- with open(readme_path, 'w') as f:
276
- f.write(readme_content)
277
- zipf.write(readme_path, "README.txt")
278
-
279
- for frame_file in os.listdir(frames_dir):
280
- if frame_file.endswith('.jpg'):
281
- frame_path = os.path.join(frames_dir, frame_file)
282
- zipf.write(frame_path, f"frames/{frame_file}")
283
-
284
- return zip_path
285
 
286
  # ============================================================================
287
- # MAIN PROCESSING
288
  # ============================================================================
289
 
290
- def process_video_frames_only(video_file, consent, frame_interval_seconds, max_frames):
291
  """Quick frame extraction only"""
292
  try:
293
- if not consent:
294
- return None, "❌ Please agree to the guidelines", None
295
 
296
  if video_file is None:
297
- return None, "⚠️ Please upload a video", None
 
 
 
 
 
 
 
 
 
 
 
 
298
 
 
299
  cap = cv2.VideoCapture(video_file)
 
 
 
300
  fps = cap.get(cv2.CAP_PROP_FPS)
301
  total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
 
302
  cap.release()
303
 
304
  if fps == 0:
305
- return None, "❌ Could not read video", None
 
 
306
 
307
  frame_step = max(1, int(fps * frame_interval_seconds))
 
308
 
309
- status = f"⚙️ Extracting frames...\n"
310
- status += f" Video: {total_frames} frames at {fps:.2f} FPS\n"
311
- status += f" • Interval: every {frame_step} frames (~{frame_interval_seconds}s)\n\n"
312
 
 
313
  extracted_frames, frames_dir, video_fps, _, extract_status = extract_frames_from_360_video(
314
  video_file, frame_step=frame_step, max_frames=max_frames
315
  )
@@ -317,10 +316,16 @@ def process_video_frames_only(video_file, consent, frame_interval_seconds, max_f
317
  if extract_status != "Success":
318
  return None, status + f"❌ {extract_status}", None
319
 
 
 
 
320
  first_frame = cv2.imread(extracted_frames[0])
321
  first_frame_rgb = cv2.cvtColor(first_frame, cv2.COLOR_BGR2RGB)
322
  preview_img = Image.fromarray(first_frame_rgb)
323
 
 
 
 
324
  video_info = {
325
  'fps': video_fps,
326
  'total_frames': total_frames,
@@ -330,61 +335,86 @@ def process_video_frames_only(video_file, consent, frame_interval_seconds, max_f
330
 
331
  zip_path = create_download_package(frames_dir, video_info)
332
 
333
- result = f"""✅ FRAMES EXTRACTED!
 
 
 
 
 
334
 
335
  📊 Summary:
336
  • Extracted: {len(extracted_frames)} frames
337
  • Interval: ~{frame_interval_seconds}s
 
 
 
 
338
 
339
- 📦 Download ZIP with frames below
340
- 🎯 Import to Metashape for 3D reconstruction
341
  """
342
 
343
- return preview_img, result, zip_path
344
 
345
  except Exception as e:
346
- return None, f"❌ ERROR: {str(e)}", None
 
347
 
348
- def process_video_with_3d(video_file, consent, frame_interval_seconds, max_frames, max_frames_3d, progress=gr.Progress()):
349
  """Extract frames AND create 3D model"""
350
  try:
351
- if not consent:
352
- return None, "❌ Please agree to the guidelines", None, None, None
353
 
354
  if video_file is None:
355
- return None, "⚠️ Please upload a video", None, None, None
356
 
357
- progress(0, desc="Analyzing video...")
 
358
 
 
 
 
 
 
 
 
 
 
 
 
359
  cap = cv2.VideoCapture(video_file)
360
  fps = cap.get(cv2.CAP_PROP_FPS)
361
  total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
362
  cap.release()
363
 
364
- if fps == 0:
365
- return None, "❌ Could not read video", None, None, None
366
-
367
  frame_step = max(1, int(fps * frame_interval_seconds))
368
 
369
- progress(0.1, desc="Extracting frames...")
370
 
371
  extracted_frames, frames_dir, video_fps, _, extract_status = extract_frames_from_360_video(
372
  video_file, frame_step=frame_step, max_frames=max_frames
373
  )
374
 
375
  if extract_status != "Success":
376
- return None, f"❌ {extract_status}", None, None, None
 
 
377
 
 
378
  first_frame = cv2.imread(extracted_frames[0])
379
  first_frame_rgb = cv2.cvtColor(first_frame, cv2.COLOR_BGR2RGB)
380
  preview_img = Image.fromarray(first_frame_rgb)
381
 
382
- progress(0.3, desc="Creating 3D model...")
 
 
383
 
384
- # Create 3D model
385
- fig, ply_path, model_status = create_3d_model(extracted_frames, max_frames_3d, progress)
 
 
 
 
386
 
387
- # Create frame package
388
  video_info = {
389
  'fps': video_fps,
390
  'total_frames': total_frames,
@@ -396,74 +426,79 @@ def process_video_with_3d(video_file, consent, frame_interval_seconds, max_frame
396
 
397
  result = f"""✅ COMPLETE!
398
 
399
- 📊 Frames:
400
- Extracted: {len(extracted_frames)} frames
401
- Used for 3D: {min(max_frames_3d, len(extracted_frames))} frames
402
-
403
- 🎨 3D Model:
404
- {model_status}
405
 
406
  📦 Downloads:
407
  • ZIP: Frames for Metashape
408
  • PLY: 3D point cloud
409
 
410
- Note: Basic 3D preview shown. For professional quality, use frames in Metashape!
411
  """
412
 
413
- return preview_img, result, zip_path, fig, ply_path
414
 
415
  except Exception as e:
416
- import traceback
417
- return None, f"❌ ERROR: {str(e)}\n{traceback.format_exc()}", None, None, None
418
 
419
  # ============================================================================
420
  # INTERFACE
421
  # ============================================================================
422
 
423
- with gr.Blocks(title="360° Outdoor Photogrammetry + 3D", theme=gr.themes.Soft()) as demo:
424
 
425
  gr.Markdown("# 🌍 360° Video: Frame Extraction + 3D Reconstruction")
426
- gr.Markdown("**Two modes:** Quick frames only (30s) OR Full 3D reconstruction (5-10min)")
 
427
 
428
  with gr.Tabs():
429
- with gr.Tab("🚀 Quick - Frames Only"):
430
- gr.Markdown(RESPONSIBLE_AI_NOTICE)
 
 
 
 
 
431
 
432
  with gr.Row():
433
  with gr.Column():
434
- consent1 = gr.Checkbox(label=" I agree to guidelines", value=False)
435
- video1 = gr.Video(label="Upload 360° Video")
436
- interval1 = gr.Slider(0.5, 5.0, 1.5, step=0.5, label="Frame Interval (seconds)")
437
- max_frames1 = gr.Slider(20, 300, 100, step=10, label="Max Frames")
438
- btn1 = gr.Button("🎬 Extract Frames (Fast!)", variant="primary")
439
 
440
  with gr.Column():
441
- status1 = gr.Textbox(label="Status", lines=10)
442
- preview1 = gr.Image(label="Preview")
443
 
444
  download1 = gr.File(label="📦 Download Frames (ZIP)")
445
 
446
  btn1.click(
447
  fn=process_video_frames_only,
448
- inputs=[video1, consent1, interval1, max_frames1],
449
  outputs=[preview1, status1, download1]
450
  )
451
 
452
- with gr.Tab("🎨 Full - Frames + 3D Model"):
453
- gr.Markdown(RESPONSIBLE_AI_NOTICE)
454
- gr.Markdown("**⚠️ This takes 5-10 minutes! Uses GPU for depth estimation.**")
 
 
 
 
455
 
456
  with gr.Row():
457
  with gr.Column():
458
- consent2 = gr.Checkbox(label=" I agree to guidelines", value=False)
459
- video2 = gr.Video(label="Upload 360° Video")
460
  interval2 = gr.Slider(0.5, 5.0, 2.0, step=0.5, label="Frame Interval (seconds)")
461
- max_frames2 = gr.Slider(20, 150, 50, step=10, label="Max Frames to Extract")
462
- max_3d = gr.Slider(2, 10, 5, step=1, label="Frames for 3D (fewer = faster)")
463
  btn2 = gr.Button("🎨 Extract + Create 3D", variant="primary")
464
 
465
  with gr.Column():
466
- status2 = gr.Textbox(label="Status", lines=10)
467
  preview2 = gr.Image(label="Preview")
468
 
469
  with gr.Row():
@@ -475,25 +510,23 @@ with gr.Blocks(title="360° Outdoor Photogrammetry + 3D", theme=gr.themes.Soft()
475
 
476
  btn2.click(
477
  fn=process_video_with_3d,
478
- inputs=[video2, consent2, interval2, max_frames2, max_3d],
479
  outputs=[preview2, status2, download2, viz, ply_download]
480
  )
481
 
482
  gr.Markdown("""
483
  ---
484
- ### 🎯 Which Mode to Use?
485
-
486
- **Quick Mode (Frames Only):**
487
- - Super fast (30-60 seconds)
488
- - 📦 Get frames for Metashape
489
- - 🎓 Professional photogrammetry workflow
490
- - ✅ Best for serious 3D models
491
-
492
- **Full Mode (Frames + 3D):**
493
- - 🎨 See 3D preview immediately
494
- - ⏱️ Takes 5-10 minutes
495
- - 🖥️ Requires GPU
496
- - 📊 Basic quality (Metashape is better!)
497
 
498
  Made for outdoor photogrammetry! 🏔️
499
  """)
 
1
  """
2
  360° Video Frame Extraction + 3D Reconstruction for Outdoor Scenes
3
+ Robust version with better error handling
 
 
 
4
  """
5
 
6
  import gradio as gr
 
16
  import open3d as o3d
17
  import plotly.graph_objects as go
18
  import warnings
19
+ import traceback
20
  warnings.filterwarnings('ignore')
21
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
22
  # ============================================================================
23
  # MODEL LOADING
24
  # ============================================================================
25
 
26
  print("Loading depth estimation model...")
27
+ MODEL_LOADED = False
28
  try:
29
  dpt_processor = DPTImageProcessor.from_pretrained("Intel/dpt-large")
30
  dpt_model = DPTForDepthEstimation.from_pretrained("Intel/dpt-large")
 
32
  dpt_model = dpt_model.cuda()
33
  print("✓ Using GPU")
34
  dpt_model.eval()
35
+ MODEL_LOADED = True
36
  print("✓ Model loaded!")
37
  except Exception as e:
38
  print(f"⚠️ Model loading failed: {e}")
 
45
 
46
  def extract_frames_from_360_video(video_path, frame_step=30, max_frames=150):
47
  """Extract frames from 360° video"""
48
+ try:
49
+ if not os.path.exists(video_path):
50
+ return [], None, 0, 0, f"Error: Video file not found at {video_path}"
 
 
 
 
 
 
 
 
 
 
 
 
51
 
52
+ cap = cv2.VideoCapture(video_path)
53
+
54
+ if not cap.isOpened():
55
+ return [], None, 0, 0, "Error: Could not open video file. Check format (MP4 recommended)"
56
+
57
+ fps = cap.get(cv2.CAP_PROP_FPS)
58
+ total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
59
+
60
+ if fps == 0 or total_frames == 0:
61
+ cap.release()
62
+ return [], None, 0, 0, "Error: Invalid video file"
63
+
64
+ frames_dir = tempfile.mkdtemp()
65
+ extracted_frames = []
66
+ frame_count = 0
67
+ saved_count = 0
68
+
69
+ while cap.isOpened() and saved_count < max_frames:
70
+ ret, frame = cap.read()
71
 
72
+ if not ret:
73
+ break
74
+
75
+ if frame_count % frame_step == 0:
76
+ frame_filename = os.path.join(frames_dir, f"frame_{saved_count:04d}.jpg")
77
+ success = cv2.imwrite(frame_filename, frame, [cv2.IMWRITE_JPEG_QUALITY, 95])
78
+ if success:
79
+ extracted_frames.append(frame_filename)
80
+ saved_count += 1
81
+
82
+ frame_count += 1
83
+
84
+ cap.release()
85
+
86
+ if len(extracted_frames) == 0:
87
+ return [], None, fps, total_frames, "Error: No frames could be extracted"
88
+
89
+ return extracted_frames, frames_dir, fps, total_frames, "Success"
90
+
91
+ except Exception as e:
92
+ return [], None, 0, 0, f"Error during extraction: {str(e)}"
93
 
94
  # ============================================================================
95
  # 3D RECONSTRUCTION
 
97
 
98
  def estimate_depth(image, processor, model):
99
  """Estimate depth for a single image"""
100
+ try:
101
+ inputs = processor(images=image, return_tensors="pt")
102
+
103
+ if torch.cuda.is_available():
104
+ inputs = {k: v.cuda() for k, v in inputs.items()}
105
+
106
+ with torch.no_grad():
107
+ outputs = model(**inputs)
108
+ predicted_depth = outputs.predicted_depth
109
+
110
+ prediction = torch.nn.functional.interpolate(
111
+ predicted_depth.unsqueeze(1),
112
+ size=image.shape[:2],
113
+ mode="bicubic",
114
+ align_corners=False,
115
+ )
116
+
117
+ depth = prediction.squeeze().cpu().numpy()
118
+ depth = (depth - depth.min()) / (depth.max() - depth.min())
119
+
120
+ return depth
121
+ except Exception as e:
122
+ print(f"Depth estimation error: {e}")
123
+ return None
124
 
125
  def depth_to_point_cloud(image, depth):
126
  """Convert depth map to 3D point cloud"""
127
  h, w = depth.shape
128
 
 
129
  x = np.linspace(0, w-1, w)
130
  y = np.linspace(0, h-1, h)
131
  xv, yv = np.meshgrid(x, y)
132
 
 
133
  x_flat = xv.flatten()
134
  y_flat = yv.flatten()
135
  z_flat = depth.flatten()
136
 
 
137
  points = np.stack([x_flat, y_flat, z_flat], axis=-1)
138
 
 
139
  if len(image.shape) == 3:
140
  colors = image.reshape(-1, 3) / 255.0
141
  else:
 
143
 
144
  return points, colors
145
 
146
+ def create_3d_model(frames, max_frames_for_3d=5):
147
  """Create 3D model from extracted frames"""
148
+ if not MODEL_LOADED or dpt_model is None or dpt_processor is None:
149
+ return None, None, "❌ Depth model not loaded. Use Quick Mode instead."
 
 
150
 
151
+ try:
152
+ all_points = []
153
+ all_colors = []
 
 
 
 
 
 
 
 
 
154
 
155
+ frames_to_process = frames[:max_frames_for_3d]
 
156
 
157
+ for idx, frame_path in enumerate(frames_to_process):
158
+ print(f"Processing frame {idx+1}/{len(frames_to_process)}...")
159
+
160
+ if not os.path.exists(frame_path):
161
+ continue
162
+
163
+ img = cv2.imread(frame_path)
164
+ if img is None:
165
+ continue
166
+
167
+ img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
168
+ img_small = cv2.resize(img_rgb, (512, 256))
169
+
170
+ depth = estimate_depth(img_small, dpt_processor, dpt_model)
171
+ if depth is None:
172
+ continue
173
+
174
+ points, colors = depth_to_point_cloud(img_small, depth)
175
+ points[:, 0] += idx * 600
176
+
177
+ all_points.append(points)
178
+ all_colors.append(colors)
179
+
180
+ if len(all_points) == 0:
181
+ return None, None, "❌ No frames could be processed"
182
+
183
+ final_points = np.vstack(all_points)
184
+ final_colors = np.vstack(all_colors)
185
+
186
+ # Downsample
187
+ if len(final_points) > 100000:
188
+ indices = np.random.choice(len(final_points), 100000, replace=False)
189
+ final_points = final_points[indices]
190
+ final_colors = final_colors[indices]
191
+
192
+ # Create visualization
193
+ fig = go.Figure(data=[go.Scatter3d(
194
+ x=final_points[:, 0],
195
+ y=final_points[:, 1],
196
+ z=final_points[:, 2],
197
+ mode='markers',
198
+ marker=dict(size=1, color=final_colors, opacity=0.8)
199
+ )])
200
+
201
+ fig.update_layout(
202
+ title="3D Reconstruction",
203
+ scene=dict(xaxis_title="X", yaxis_title="Y", zaxis_title="Depth"),
204
+ width=800,
205
+ height=600
206
+ )
207
 
208
+ # Save PLY
209
+ ply_path = os.path.join(tempfile.gettempdir(), f"3d_model_{datetime.now().strftime('%Y%m%d_%H%M%S')}.ply")
210
+ pcd = o3d.geometry.PointCloud()
211
+ pcd.points = o3d.utility.Vector3dVector(final_points)
212
+ pcd.colors = o3d.utility.Vector3dVector(final_colors)
213
+ o3d.io.write_point_cloud(ply_path, pcd)
214
 
215
+ return fig, ply_path, f"✅ Created {len(final_points):,} points"
 
216
 
217
+ except Exception as e:
218
+ return None, None, f"❌ 3D creation error: {str(e)}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
219
 
220
  # ============================================================================
221
  # PACKAGE CREATION
222
  # ============================================================================
223
 
224
+ def create_download_package(frames_dir, video_info):
225
+ """Create ZIP with frames"""
226
+ try:
227
+ zip_path = os.path.join(tempfile.gettempdir(), f"360_frames_{datetime.now().strftime('%Y%m%d_%H%M%S')}.zip")
228
+
229
+ readme_content = f"""360° OUTDOOR PHOTOGRAMMETRY PACKAGE
230
  Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
231
 
232
  VIDEO INFO:
 
237
  METASHAPE WORKFLOW:
238
  1. Import Photos
239
  2. Set Camera Type to "Spherical"
240
+ 3. Align Photos (High accuracy, Sequential)
241
  4. Build Dense Cloud
242
  5. Build Mesh
243
  6. Build Texture
244
 
245
+ SOFTWARE: Agisoft Metashape ($179)
246
  Good luck! 🌍📸
247
  """
248
+
249
+ with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
250
+ readme_path = os.path.join(tempfile.gettempdir(), "README.txt")
251
+ with open(readme_path, 'w') as f:
252
+ f.write(readme_content)
253
+ zipf.write(readme_path, "README.txt")
254
+
255
+ for frame_file in os.listdir(frames_dir):
256
+ if frame_file.endswith('.jpg'):
257
+ frame_path = os.path.join(frames_dir, frame_file)
258
+ if os.path.exists(frame_path):
259
+ zipf.write(frame_path, f"frames/{frame_file}")
260
+
261
+ return zip_path
262
+ except Exception as e:
263
+ print(f"ZIP creation error: {e}")
264
+ return None
 
265
 
266
  # ============================================================================
267
+ # MAIN PROCESSING FUNCTIONS
268
  # ============================================================================
269
 
270
+ def process_video_frames_only(video_file, frame_interval_seconds, max_frames):
271
  """Quick frame extraction only"""
272
  try:
273
+ print(f"Starting frame extraction. Video: {video_file}")
 
274
 
275
  if video_file is None:
276
+ return None, "⚠️ Please upload a video file", None
277
+
278
+ # Check file exists and size
279
+ if not os.path.exists(video_file):
280
+ return None, f"❌ Video file not found: {video_file}", None
281
+
282
+ file_size = os.path.getsize(video_file) / (1024*1024) # MB
283
+ print(f"Video file size: {file_size:.2f} MB")
284
+
285
+ if file_size > 1000:
286
+ return None, f"❌ Video too large ({file_size:.0f}MB). Max 1GB. Please compress the video.", None
287
+
288
+ status = f"📹 Processing video ({file_size:.1f}MB)...\n\n"
289
 
290
+ # Get video info
291
  cap = cv2.VideoCapture(video_file)
292
+ if not cap.isOpened():
293
+ return None, status + "❌ Could not open video. Try MP4 format.", None
294
+
295
  fps = cap.get(cv2.CAP_PROP_FPS)
296
  total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
297
+ duration = total_frames / fps if fps > 0 else 0
298
  cap.release()
299
 
300
  if fps == 0:
301
+ return None, status + "❌ Invalid video file", None
302
+
303
+ status += f"✓ Video: {duration:.1f}s, {fps:.1f} FPS, {total_frames} frames\n\n"
304
 
305
  frame_step = max(1, int(fps * frame_interval_seconds))
306
+ estimated_frames = min(max_frames, total_frames // frame_step)
307
 
308
+ status += f"⚙️ Extracting ~{estimated_frames} frames...\n"
309
+ status += f" (every {frame_step} frames = ~{frame_interval_seconds}s interval)\n\n"
 
310
 
311
+ # Extract frames
312
  extracted_frames, frames_dir, video_fps, _, extract_status = extract_frames_from_360_video(
313
  video_file, frame_step=frame_step, max_frames=max_frames
314
  )
 
316
  if extract_status != "Success":
317
  return None, status + f"❌ {extract_status}", None
318
 
319
+ status += f"✓ Extracted {len(extracted_frames)} frames\n\n"
320
+
321
+ # Create preview
322
  first_frame = cv2.imread(extracted_frames[0])
323
  first_frame_rgb = cv2.cvtColor(first_frame, cv2.COLOR_BGR2RGB)
324
  preview_img = Image.fromarray(first_frame_rgb)
325
 
326
+ # Create ZIP
327
+ status += "📦 Creating download package...\n"
328
+
329
  video_info = {
330
  'fps': video_fps,
331
  'total_frames': total_frames,
 
335
 
336
  zip_path = create_download_package(frames_dir, video_info)
337
 
338
+ if zip_path is None:
339
+ return preview_img, status + "❌ Could not create ZIP", None
340
+
341
+ zip_size = os.path.getsize(zip_path) / (1024*1024)
342
+
343
+ result = f"""✅ SUCCESS!
344
 
345
  📊 Summary:
346
  • Extracted: {len(extracted_frames)} frames
347
  • Interval: ~{frame_interval_seconds}s
348
+ • ZIP size: {zip_size:.1f}MB
349
+
350
+ 📦 Download ZIP below
351
+ 🎯 Import to Metashape for 3D model
352
 
353
+ Next: Use Agisoft Metashape ($179) to create professional 3D model
 
354
  """
355
 
356
+ return preview_img, status + result, zip_path
357
 
358
  except Exception as e:
359
+ error_trace = traceback.format_exc()
360
+ return None, f"❌ ERROR:\n{str(e)}\n\n{error_trace}", None
361
 
362
+ def process_video_with_3d(video_file, frame_interval_seconds, max_frames, max_frames_3d):
363
  """Extract frames AND create 3D model"""
364
  try:
365
+ print(f"Starting full 3D processing. Video: {video_file}")
 
366
 
367
  if video_file is None:
368
+ return None, "⚠️ Please upload a video file", None, None, None
369
 
370
+ if not MODEL_LOADED:
371
+ return None, "❌ 3D model not loaded. Use Quick Mode instead.", None, None, None
372
 
373
+ if not os.path.exists(video_file):
374
+ return None, f"❌ Video file not found: {video_file}", None, None, None
375
+
376
+ file_size = os.path.getsize(video_file) / (1024*1024)
377
+
378
+ if file_size > 500:
379
+ return None, f"❌ Video too large for 3D mode ({file_size:.0f}MB). Max 500MB. Use Quick Mode or compress video.", None, None, None
380
+
381
+ status = f"📹 Full 3D Processing ({file_size:.1f}MB)...\n\n"
382
+
383
+ # Extract frames first
384
  cap = cv2.VideoCapture(video_file)
385
  fps = cap.get(cv2.CAP_PROP_FPS)
386
  total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
387
  cap.release()
388
 
 
 
 
389
  frame_step = max(1, int(fps * frame_interval_seconds))
390
 
391
+ status += f"⚙️ Step 1/3: Extracting frames...\n"
392
 
393
  extracted_frames, frames_dir, video_fps, _, extract_status = extract_frames_from_360_video(
394
  video_file, frame_step=frame_step, max_frames=max_frames
395
  )
396
 
397
  if extract_status != "Success":
398
+ return None, status + f"❌ {extract_status}", None, None, None
399
+
400
+ status += f"✓ Extracted {len(extracted_frames)} frames\n\n"
401
 
402
+ # Preview
403
  first_frame = cv2.imread(extracted_frames[0])
404
  first_frame_rgb = cv2.cvtColor(first_frame, cv2.COLOR_BGR2RGB)
405
  preview_img = Image.fromarray(first_frame_rgb)
406
 
407
+ # Create 3D
408
+ status += f"⚙️ Step 2/3: Creating 3D model (using {min(max_frames_3d, len(extracted_frames))} frames)...\n"
409
+ status += "This may take 5-10 minutes...\n\n"
410
 
411
+ fig, ply_path, model_status = create_3d_model(extracted_frames, max_frames_3d)
412
+
413
+ status += f"{model_status}\n\n"
414
+
415
+ # Create ZIP
416
+ status += f"⚙️ Step 3/3: Creating frame package...\n"
417
 
 
418
  video_info = {
419
  'fps': video_fps,
420
  'total_frames': total_frames,
 
426
 
427
  result = f"""✅ COMPLETE!
428
 
429
+ 📊 Results:
430
+ Frames: {len(extracted_frames)}
431
+ 3D points: {model_status}
 
 
 
432
 
433
  📦 Downloads:
434
  • ZIP: Frames for Metashape
435
  • PLY: 3D point cloud
436
 
437
+ Note: This is a basic preview. Use Metashape for professional quality!
438
  """
439
 
440
+ return preview_img, status + result, zip_path, fig, ply_path
441
 
442
  except Exception as e:
443
+ error_trace = traceback.format_exc()
444
+ return None, f"❌ ERROR:\n{str(e)}\n\n{error_trace}", None, None, None
445
 
446
  # ============================================================================
447
  # INTERFACE
448
  # ============================================================================
449
 
450
+ with gr.Blocks(title="360° Outdoor Photogrammetry + 3D") as demo:
451
 
452
  gr.Markdown("# 🌍 360° Video: Frame Extraction + 3D Reconstruction")
453
+ gr.Markdown("**Two modes:** Quick frames (30s) OR Full 3D (5-10min)")
454
+ gr.Markdown("⚠️ **Max file size:** Quick Mode: 1GB | Full 3D: 500MB | **8-minute videos OK!**")
455
 
456
  with gr.Tabs():
457
+ with gr.Tab("🚀 Quick - Frames Only (RECOMMENDED)"):
458
+ gr.Markdown("""
459
+ ### Fast & Free!
460
+ - Extract frames in 30-60 seconds
461
+ - Works on FREE tier
462
+ - Best for professional Metashape workflow
463
+ """)
464
 
465
  with gr.Row():
466
  with gr.Column():
467
+ video1 = gr.Video(label="Upload 360° Video (MP4 recommended, max 1GB - 8 min videos OK!)")
468
+ interval1 = gr.Slider(0.5, 5.0, 2.0, step=0.5, label="Frame Interval (seconds) - 2s good for 8min videos")
469
+ max_frames1 = gr.Slider(20, 500, 150, step=10, label="Max Frames - 150-200 good for 8min")
470
+ btn1 = gr.Button("🎬 Extract Frames", variant="primary", size="lg")
 
471
 
472
  with gr.Column():
473
+ status1 = gr.Textbox(label="Status", lines=15)
474
+ preview1 = gr.Image(label="Preview (First Frame)")
475
 
476
  download1 = gr.File(label="📦 Download Frames (ZIP)")
477
 
478
  btn1.click(
479
  fn=process_video_frames_only,
480
+ inputs=[video1, interval1, max_frames1],
481
  outputs=[preview1, status1, download1]
482
  )
483
 
484
+ with gr.Tab("🎨 Full - Frames + 3D (SLOW, NEEDS GPU)"):
485
+ gr.Markdown("""
486
+ ### Creates 3D Preview
487
+ - Takes 5-10 minutes
488
+ - Requires GPU upgrade ($0.60/hour)
489
+ - Basic quality (Metashape is better!)
490
+ """)
491
 
492
  with gr.Row():
493
  with gr.Column():
494
+ video2 = gr.Video(label="Upload 360° Video (MP4, max 500MB - compress long videos)")
 
495
  interval2 = gr.Slider(0.5, 5.0, 2.0, step=0.5, label="Frame Interval (seconds)")
496
+ max_frames2 = gr.Slider(20, 100, 30, step=10, label="Max Frames")
497
+ max_3d = gr.Slider(2, 8, 4, step=1, label="Frames for 3D (fewer = faster)")
498
  btn2 = gr.Button("🎨 Extract + Create 3D", variant="primary")
499
 
500
  with gr.Column():
501
+ status2 = gr.Textbox(label="Status", lines=15)
502
  preview2 = gr.Image(label="Preview")
503
 
504
  with gr.Row():
 
510
 
511
  btn2.click(
512
  fn=process_video_with_3d,
513
+ inputs=[video2, interval2, max_frames2, max_3d],
514
  outputs=[preview2, status2, download2, viz, ply_download]
515
  )
516
 
517
  gr.Markdown("""
518
  ---
519
+ ### 💡 Tips for 8-Minute Videos:
520
+ - **Quick Mode** - Handles up to 1GB (8 min at 5K: ~400-600MB)
521
+ - **Frame interval: 2-3 seconds** - Gets 160-240 frames from 8 min
522
+ - **Use MP4 format** - Best compatibility
523
+ - **If over 1GB** - Compress with HandBrake (target 5-8 Mbps)
524
+ - **For best 3D quality** - Use Metashape with extracted frames
525
+
526
+ ### 📐 Expected Frames from 8-Min Video:
527
+ - 1s interval: ~480 frames (very dense, slow processing)
528
+ - 2s interval: ~240 frames (recommended for outdoor)
529
+ - 3s interval: ~160 frames (good for large landscapes)
 
 
530
 
531
  Made for outdoor photogrammetry! 🏔️
532
  """)