Hug0endob commited on
Commit
9c7140f
·
verified ·
1 Parent(s): 31baa74

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +258 -162
app.py CHANGED
@@ -45,7 +45,7 @@ _temp_preview_files_to_delete = []
45
 
46
  def _cleanup_all_temp_preview_files():
47
  """Removes all temporary files created for previews upon application exit."""
48
- for f_path in list(_temp_preview_files_to_delete): # Iterate over a copy
49
  if os.path.exists(f_path):
50
  try:
51
  os.remove(f_path)
@@ -158,17 +158,54 @@ def convert_to_jpeg_bytes(img_bytes: bytes, base_h: int = 480) -> bytes:
158
  def b64_bytes(b: bytes, mime: str = "image/jpeg") -> str:
159
  return f"data:{mime};base64," + base64.b64encode(b).decode("utf-8")
160
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
161
  def extract_best_frames_bytes(media_path: str, sample_count: int = 5, timeout_extract: int = 15, progress=None) -> List[bytes]:
 
 
 
162
  frames: List[bytes] = []
163
  if not FFMPEG_BIN or not os.path.exists(media_path):
164
  return frames
165
  if progress is not None:
166
- progress(0.05, desc="Preparing frame extraction...")
167
- timestamps = [0.5, 1.0, 2.0, 3.0, 4.0][:sample_count]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
168
  for i, t in enumerate(timestamps):
169
- fd, tmp = tempfile.mkstemp(suffix=f"_{i}.jpg")
170
  os.close(fd)
171
- _temp_preview_files_to_delete.append(tmp) # Track for cleanup
 
172
  cmd = [
173
  FFMPEG_BIN,
174
  "-nostdin",
@@ -180,12 +217,12 @@ def extract_best_frames_bytes(media_path: str, sample_count: int = 5, timeout_ex
180
  "-frames:v",
181
  "1",
182
  "-q:v",
183
- "2",
184
  tmp,
185
  ]
186
  try:
187
  if progress is not None:
188
- progress(0.1 + (i / max(1, sample_count)) * 0.2, desc=f"Extracting frame {i+1}/{sample_count}...")
189
  subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=timeout_extract)
190
  if os.path.exists(tmp) and os.path.getsize(tmp) > 0:
191
  with open(tmp, "rb") as f:
@@ -193,16 +230,70 @@ def extract_best_frames_bytes(media_path: str, sample_count: int = 5, timeout_ex
193
  except Exception:
194
  pass
195
  finally:
196
- # frame is read into memory, temp file can be removed early if not already done by atexit
197
- try:
198
- if tmp in _temp_preview_files_to_delete:
199
- _temp_preview_files_to_delete.remove(tmp)
200
- os.remove(tmp)
201
  except Exception: pass
202
  if progress is not None:
203
- progress(0.45, desc=f"Extracted {len(frames)} frames")
204
  return frames
205
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
206
  def chat_complete(client, model: str, messages, timeout: int = 120, progress=None) -> str:
207
  try:
208
  if progress is not None:
@@ -302,7 +393,12 @@ def analyze_image_structured(client, img_bytes: bytes, prompt: str, progress=Non
302
  except Exception as e:
303
  return f"Error analyzing image: {e}"
304
 
305
- def analyze_video_cohesive(client, video_path: str, prompt: str, progress=None) -> str:
 
 
 
 
 
306
  try:
307
  if progress is not None:
308
  progress(0.3, desc="Uploading video for full analysis...")
@@ -315,18 +411,28 @@ def analyze_video_cohesive(client, video_path: str, prompt: str, progress=None)
315
  {"role": "system", "content": SYSTEM_INSTRUCTION},
316
  {"role": "user", "content": extra_msg + "\n\n" + prompt},
317
  ]
318
- return chat_complete(client, VIDEO_MODEL, messages, progress=progress)
 
 
 
319
  except Exception as e:
320
  if progress is not None:
321
  progress(0.35, desc="Upload failed, extracting frames as fallback...")
322
- frames = extract_best_frames_bytes(video_path, sample_count=6, progress=progress)
323
- if not frames:
324
- return f"Error: could not upload video and no frames could be extracted. ({e})"
 
 
 
 
 
 
 
325
  image_entries = []
326
- for i, fb in enumerate(frames, start=1):
327
  try:
328
  if progress is not None:
329
- progress(0.4 + (i / len(frames)) * 0.2, desc=f"Preparing frame {i}/{len(frames)}...")
330
  j = convert_to_jpeg_bytes(fb, base_h=720)
331
  image_entries.append(
332
  {
@@ -342,25 +448,10 @@ def analyze_video_cohesive(client, video_path: str, prompt: str, progress=None)
342
  {"role": "system", "content": SYSTEM_INSTRUCTION},
343
  {"role": "user", "content": content},
344
  ]
345
- return chat_complete(client, PIXTRAL_MODEL, messages, progress=progress)
 
346
 
347
  # --- FFmpeg Helpers for Preview ---
348
- def _ffprobe_streams(path: str) -> Optional[dict]:
349
- """Probes video codecs via ffprobe; returns dict with streams info or None on failure."""
350
- if not FFMPEG_BIN:
351
- return None
352
- ffprobe = FFMPEG_BIN.replace("ffmpeg", "ffprobe") if "ffmpeg" in FFMPEG_BIN else "ffprobe"
353
- if not shutil.which(ffprobe):
354
- ffprobe = "ffprobe" # Try system PATH
355
- cmd = [
356
- ffprobe, "-v", "error", "-print_format", "json", "-show_streams", "-show_format", path
357
- ]
358
- try:
359
- out = subprocess.check_output(cmd, stderr=subprocess.DEVNULL)
360
- return json.loads(out)
361
- except Exception:
362
- return None
363
-
364
  def _convert_video_for_preview_if_needed(path: str) -> str:
365
  """
366
  Returns a path that the Gradio video component can play.
@@ -371,7 +462,6 @@ def _convert_video_for_preview_if_needed(path: str) -> str:
371
  if not FFMPEG_BIN or not os.path.exists(path):
372
  return path # Cannot convert, return original
373
 
374
- # Quick check for MP4 and common codecs
375
  if path.lower().endswith((".mp4", ".m4v", ".mov")):
376
  info = _ffprobe_streams(path)
377
  if info:
@@ -379,7 +469,6 @@ def _convert_video_for_preview_if_needed(path: str) -> str:
379
  if video_streams and any(s.get("codec_name") in ("h264", "h265", "avc1") for s in video_streams):
380
  return path # Already playable
381
 
382
- # Need conversion → write to a new temp MP4
383
  out_path = _temp_file(b"", suffix=".mp4") # Create an empty temp file and add to cleanup list
384
  cmd = [
385
  FFMPEG_BIN, "-y", "-i", path,
@@ -396,7 +485,7 @@ def _convert_video_for_preview_if_needed(path: str) -> str:
396
  _temp_preview_files_to_delete.remove(out_path)
397
  try: os.remove(out_path)
398
  except Exception: pass
399
- return path # Gradio will show its own warning if not playable
400
 
401
  # --- Preview Generation Logic ---
402
  def _get_playable_preview_path_from_raw(src_url: str, raw_bytes: bytes) -> str:
@@ -407,22 +496,17 @@ def _get_playable_preview_path_from_raw(src_url: str, raw_bytes: bytes) -> str:
407
  is_img, is_vid = determine_media_type(src_url)
408
 
409
  if is_vid:
410
- # Save raw video bytes to a temp file for potential conversion
411
  temp_raw_video_path = _temp_file(raw_bytes, suffix=ext_from_src(src_url) or ".mp4")
412
-
413
- # Convert it for browser playback if necessary; this might return a new temp path or the original
414
  playable_path = _convert_video_for_preview_if_needed(temp_raw_video_path)
415
 
416
- # If a new path was created by conversion, the original temp_raw_video_path is no longer needed
417
- # and should be explicitly removed if it's no longer tracked or if it's tracked separately
418
- if playable_path != temp_raw_video_path and os.path.exists(temp_raw_video_path):
419
- if temp_raw_video_path in _temp_preview_files_to_delete:
420
- _temp_preview_files_to_delete.remove(temp_raw_video_path)
421
  try: os.remove(temp_raw_video_path)
422
  except Exception: pass
423
  return playable_path
424
- else: # Assume image or unknown treated as image for preview
425
- # Convert image bytes to JPEG and save as temp file
426
  return _temp_file(convert_to_jpeg_bytes(raw_bytes, base_h=1024), suffix=".jpg")
427
 
428
  def _fetch_with_retries_bytes(src: str, timeout: int = 15, max_retries: int = 3):
@@ -432,25 +516,24 @@ def _fetch_with_retries_bytes(src: str, timeout: int = 15, max_retries: int = 3)
432
  attempt += 1
433
  try:
434
  if is_remote(src):
435
- r = requests.get(src, timeout=timeout, stream=True)
436
- if r.status_code == 200:
437
  return r.content
438
- if r.status_code == 429: # Rate limit
439
- ra = r.headers.get("Retry-After")
440
- try: delay = float(ra) if ra is not None else delay
441
- except Exception: pass
442
- r.raise_for_status()
443
  else:
444
  with open(src, "rb") as fh:
445
  return fh.read()
446
- except requests.exceptions.RequestException:
447
- if attempt >= max_retries: raise
 
 
448
  time.sleep(delay)
449
  delay *= 2
450
  except FileNotFoundError:
451
  raise
452
- except Exception:
453
- if attempt >= max_retries: raise
 
 
454
  time.sleep(delay)
455
  delay *= 2
456
 
@@ -467,8 +550,8 @@ def _save_local_playable_preview(src: str) -> Optional[str]:
467
  is_img, is_vid = determine_media_type(src)
468
  if is_vid:
469
  return _convert_video_for_preview_if_needed(src)
470
- return src # Local image, return as is
471
- return None # Local path does not exist
472
 
473
  # Remote source
474
  try:
@@ -492,6 +575,8 @@ def create_demo():
492
  with gr.Column(scale=1):
493
  preview_image = gr.Image(label="Preview Image", type="filepath", elem_classes="preview_media", visible=False)
494
  preview_video = gr.Video(label="Preview Video", elem_classes="preview_media", visible=False, format="mp4")
 
 
495
  preview_status = gr.Textbox(label="Preview status", interactive=False, lines=1, value="", visible=True)
496
  with gr.Column(scale=2):
497
  url_input = gr.Textbox(label="Image / Video URL", placeholder="https://...", lines=1)
@@ -505,77 +590,97 @@ def create_demo():
505
  progress_md = gr.Markdown("Idle")
506
  output_md = gr.Markdown("")
507
 
508
- # State to track overall processing status (idle, busy, done, error)
509
  status_state = gr.State("idle")
510
- # State to hold the current path of the file being used for preview (whether from URL input or worker)
511
- preview_path_state = gr.State("")
 
 
512
 
513
- # Function to handle URL input change and update preview
514
- def load_preview(url: str):
515
  """
516
- Loads a preview for the given URL and updates the preview components.
517
- Returns (image_update, video_update, status_message, new_preview_path_for_state).
 
518
  """
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
519
  if not url:
520
- return gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value=""), ""
 
521
 
522
  try:
523
- local_playable_path = _save_local_playable_preview(url)
524
  if not local_playable_path:
525
- return gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value="Preview load failed: could not fetch resource or make playable."), ""
 
 
526
 
527
- # Determine if it's an image or video for display
528
  is_img_preview = False
529
  try:
530
  Image.open(local_playable_path).verify()
531
  is_img_preview = True
532
  except Exception:
533
- pass # Not an image, treat as video
534
 
535
  if is_img_preview:
536
- return gr.update(value=local_playable_path, visible=True), gr.update(value=None, visible=False), gr.update(value="Image preview loaded."), local_playable_path
537
- else: # Assume video (Gradio will render if playable)
538
- return gr.update(value=None, visible=False), gr.update(value=local_playable_path, visible=True), gr.update(value="Video preview loaded."), local_playable_path
 
 
 
 
539
 
540
  except Exception as e:
541
- return gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value=f"Preview load failed: {e}"), ""
 
 
542
 
543
- # Bind load_preview to the URL input change event
544
  url_input.change(
545
- fn=load_preview,
546
  inputs=[url_input],
547
- outputs=[preview_image, preview_video, preview_status, preview_path_state]
548
  )
549
 
550
- # Function to clear all inputs and outputs
551
- def clear_all(current_preview_path: str):
552
- """Clears all inputs/outputs and cleans up the currently displayed preview file."""
553
- if current_preview_path and os.path.exists(current_preview_path) and current_preview_path in _temp_preview_files_to_delete:
554
- try:
555
- os.remove(current_preview_path)
556
- _temp_preview_files_to_delete.remove(current_preview_path)
557
- except Exception as e:
558
- print(f"Error cleaning up on clear: {e}")
559
-
560
- return "", None, None, "idle", "Idle", "", "" # url_input, preview_image, preview_video, status_state, progress_md, output_md, preview_path_state
561
-
562
- clear_btn.click(
563
- fn=clear_all,
564
- inputs=[preview_path_state], # Pass current preview path for cleanup
565
- outputs=[url_input, preview_image, preview_video, status_state, progress_md, output_md, preview_path_state]
566
- )
567
-
568
- # Main worker function for analysis
569
  def worker(url: str, prompt: str, key: str, progress=gr.Progress()):
570
  """
571
  Performs the media analysis.
572
- Returns (status, markdown_output, new_preview_path_for_state).
573
  """
574
- temp_media_file_for_analysis = None # Temporary file for analysis (video-only for voxtral)
575
- generated_preview_path = "" # Path for the Gradio preview components
 
 
 
576
  try:
577
  if not url:
578
- return "error", "**Error:** No URL provided.", ""
579
 
580
  progress(0.01, desc="Starting media processing")
581
  progress(0.02, desc="Checking URL / content‑type")
@@ -590,16 +695,13 @@ def create_demo():
590
  progress(0.05, desc="Downloading video for analysis")
591
  raw_bytes = fetch_bytes(url, timeout=120, progress=progress)
592
  if not raw_bytes:
593
- return "error", "Failed to download video bytes.", ""
594
 
595
- # Create a temporary file for analysis (Mistral API needs a path for video upload)
596
  temp_media_file_for_analysis = _temp_file(raw_bytes, suffix=ext_from_src(url) or ".mp4")
597
-
598
- progress(0.15, desc="Preparing video preview")
599
- generated_preview_path = _get_playable_preview_path_from_raw(url, raw_bytes)
600
 
601
  progress(0.25, desc="Running full‑video analysis")
602
- result = analyze_video_cohesive(client, temp_media_file_for_analysis, prompt, progress=progress)
603
 
604
  # --- Image Processing Path ---
605
  elif is_img:
@@ -607,17 +709,17 @@ def create_demo():
607
  raw_bytes = fetch_bytes(url, progress=progress)
608
 
609
  progress(0.15, desc="Preparing image preview")
610
- generated_preview_path = _get_playable_preview_path_from_raw(url, raw_bytes)
611
 
612
  progress(0.20, desc="Running image analysis")
613
- result = analyze_image_structured(client, raw_bytes, prompt, progress=progress)
 
614
 
615
  # --- Unknown Media Type (Fallback) ---
616
  else:
617
  progress(0.07, desc="Downloading unknown media for type determination")
618
  raw_bytes = fetch_bytes(url, timeout=120, progress=progress)
619
 
620
- # Try to open as image first
621
  is_definitely_img = False
622
  try:
623
  Image.open(BytesIO(raw_bytes)).verify()
@@ -627,35 +729,34 @@ def create_demo():
627
 
628
  if is_definitely_img:
629
  progress(0.15, desc="Preparing image preview (fallback)")
630
- generated_preview_path = _get_playable_preview_path_from_raw(url, raw_bytes)
631
  progress(0.20, desc="Running image analysis (fallback)")
632
- result = analyze_image_structured(client, raw_bytes, prompt, progress=progress)
633
  else: # Treat as video fallback
634
  progress(0.15, desc="Preparing video preview (fallback)")
635
  temp_media_file_for_analysis = _temp_file(raw_bytes, suffix=ext_from_src(url) or ".mp4")
636
- generated_preview_path = _get_playable_preview_path_from_raw(url, raw_bytes)
637
  progress(0.25, desc="Running video analysis (fallback)")
638
- result = analyze_video_cohesive(client, temp_media_file_for_analysis, prompt, progress=progress)
639
 
640
- status = "done" if not (isinstance(result, str) and result.lower().startswith("error")) else "error"
641
 
642
- return status, result if isinstance(result, str) else str(result), generated_preview_path
643
 
644
  except Exception as exc:
645
- return "error", f"Unexpected worker error: {exc}", ""
646
  finally:
647
- # Clean up the file used for analysis, if it was a temporary file
648
  if temp_media_file_for_analysis and os.path.exists(temp_media_file_for_analysis):
649
  if temp_media_file_for_analysis in _temp_preview_files_to_delete:
650
- _temp_preview_files_to_delete.remove(temp_media_file_for_analysis) # Remove from list if also added there
651
  try: os.remove(temp_media_file_for_analysis)
652
  except Exception as e: print(f"Error cleaning up analysis temp file {temp_media_file_for_analysis}: {e}")
653
 
654
- # Bind worker function to submit button click
655
  submit_btn.click(
656
  fn=worker,
657
  inputs=[url_input, custom_prompt, api_key],
658
- outputs=[status_state, output_md, preview_path_state], # Worker updates preview_path_state
659
  show_progress="full",
660
  show_progress_on=progress_md,
661
  )
@@ -668,47 +769,42 @@ def create_demo():
668
  return {"idle": "Idle", "busy": "Processing…", "done": "Completed", "error": "Error — see output"}.get(s, s)
669
  status_state.change(fn=status_to_progress_text, inputs=[status_state], outputs=[progress_md])
670
 
671
- # Function to react to changes in preview_path_state and update the UI
672
- def apply_preview_change(new_path: str, old_path: str):
673
- """
674
- Handles updating the preview_image/preview_video components and cleaning up old files.
675
- `old_path` is implicitly passed by Gradio for State components.
676
- """
677
- # Clean up the OLD preview file if it was a temporary file managed by us
678
- if old_path and os.path.exists(old_path) and old_path in _temp_preview_files_to_delete:
679
  try:
680
- os.remove(old_path)
681
- _temp_preview_files_to_delete.remove(old_path) # Remove from tracking list
 
 
 
 
 
 
 
 
 
682
  except Exception as e:
683
- print(f"Error cleaning up old preview file {old_path}: {e}")
684
 
685
- # If new_path is empty, clear both components and status
686
- if not new_path:
687
- return gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value="")
688
 
689
- # Determine if new_path is an image or video and update components
690
- try:
691
- is_img_preview = False
692
- try:
693
- Image.open(new_path).verify()
694
- is_img_preview = True
695
- except Exception:
696
- pass # Not an image, treat as video
697
-
698
- if is_img_preview:
699
- return gr.update(value=new_path, visible=True), gr.update(value=None, visible=False), gr.update(value="Preview updated.")
700
- else: # Assume video (Gradio will render if playable)
701
- return gr.update(value=None, visible=False), gr.update(value=new_path, visible=True), gr.update(value="Preview updated.")
702
- except Exception as e:
703
- print(f"Error applying new preview from path {new_path}: {e}")
704
- return gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value=f"Preview failed for path: {e}")
705
-
706
- # Register the change event for preview_path_state
707
- # Gradio will automatically pass the new value as the first argument and the old value as the second.
708
- preview_path_state.change(
709
- fn=apply_preview_change,
710
- inputs=[preview_path_state], # `preview_path_state` will be `new_path`. `old_path` is passed implicitly.
711
- outputs=[preview_image, preview_video, preview_status]
712
  )
713
 
714
  return demo
 
45
 
46
  def _cleanup_all_temp_preview_files():
47
  """Removes all temporary files created for previews upon application exit."""
48
+ for f_path in list(_temp_preview_files_to_delete): # Iterate over a copy to allow modification
49
  if os.path.exists(f_path):
50
  try:
51
  os.remove(f_path)
 
158
  def b64_bytes(b: bytes, mime: str = "image/jpeg") -> str:
159
  return f"data:{mime};base64," + base64.b64encode(b).decode("utf-8")
160
 
161
+ def _ffprobe_streams(path: str) -> Optional[dict]:
162
+ """Probes video codecs via ffprobe; returns dict with streams info or None on failure."""
163
+ if not FFMPEG_BIN:
164
+ return None
165
+ ffprobe = FFMPEG_BIN.replace("ffmpeg", "ffprobe") if "ffmpeg" in FFMPEG_BIN else "ffprobe"
166
+ if not shutil.which(ffprobe):
167
+ ffprobe = "ffprobe" # Try system PATH
168
+ cmd = [
169
+ ffprobe, "-v", "error", "-print_format", "json", "-show_streams", "-show_format", path
170
+ ]
171
+ try:
172
+ out = subprocess.check_output(cmd, stderr=subprocess.DEVNULL)
173
+ return json.loads(out)
174
+ except Exception:
175
+ return None
176
+
177
  def extract_best_frames_bytes(media_path: str, sample_count: int = 5, timeout_extract: int = 15, progress=None) -> List[bytes]:
178
+ """
179
+ Extracts frames as bytes for model input. These temp files are immediately deleted.
180
+ """
181
  frames: List[bytes] = []
182
  if not FFMPEG_BIN or not os.path.exists(media_path):
183
  return frames
184
  if progress is not None:
185
+ progress(0.05, desc="Preparing frame extraction for model...")
186
+
187
+ duration = 0.0
188
+ info = _ffprobe_streams(media_path)
189
+ if info and "format" in info and "duration" in info["format"]:
190
+ try:
191
+ duration = float(info["format"]["duration"])
192
+ except ValueError:
193
+ pass
194
+
195
+ timestamps: List[float] = []
196
+ if duration > 0 and sample_count > 0:
197
+ # Sample evenly across the video
198
+ step = duration / (sample_count + 1)
199
+ timestamps = [step * (i + 1) for i in range(sample_count)]
200
+ else:
201
+ # Fallback to fixed timestamps
202
+ timestamps = [0.5, 1.0, 2.0, 3.0, 4.0][:sample_count]
203
+
204
  for i, t in enumerate(timestamps):
205
+ fd, tmp = tempfile.mkstemp(suffix=f"_{i}_model.jpg")
206
  os.close(fd)
207
+ # This temp file is for immediate read and deletion, not persistent tracking
208
+
209
  cmd = [
210
  FFMPEG_BIN,
211
  "-nostdin",
 
217
  "-frames:v",
218
  "1",
219
  "-q:v",
220
+ "2", # High quality JPEG
221
  tmp,
222
  ]
223
  try:
224
  if progress is not None:
225
+ progress(0.1 + (i / max(1, sample_count)) * 0.2, desc=f"Extracting frame {i+1}/{sample_count} for model...")
226
  subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=timeout_extract)
227
  if os.path.exists(tmp) and os.path.getsize(tmp) > 0:
228
  with open(tmp, "rb") as f:
 
230
  except Exception:
231
  pass
232
  finally:
233
+ try: os.remove(tmp)
 
 
 
 
234
  except Exception: pass
235
  if progress is not None:
236
+ progress(0.45, desc=f"Extracted {len(frames)} frames for model")
237
  return frames
238
 
239
+ def extract_and_save_frames_for_gallery(media_path: str, sample_count: int = 5, timeout_extract: int = 15, base_h: int = 128, progress=None) -> List[str]:
240
+ """
241
+ Extracts frames from a video, converts them to small JPEGs, saves them as temp files
242
+ (tracked for cleanup), and returns a list of paths to these temporary files for gallery display.
243
+ """
244
+ frame_paths: List[str] = []
245
+ if not FFMPEG_BIN or not os.path.exists(media_path):
246
+ return frame_paths
247
+
248
+ duration = 0.0
249
+ info = _ffprobe_streams(media_path)
250
+ if info and "format" in info and "duration" in info["format"]:
251
+ try:
252
+ duration = float(info["format"]["duration"])
253
+ except ValueError:
254
+ pass
255
+
256
+ timestamps: List[float] = []
257
+ if duration > 0 and sample_count > 0:
258
+ step = duration / (sample_count + 1)
259
+ timestamps = [step * (i + 1) for i in range(sample_count)]
260
+ else:
261
+ timestamps = [0.5, 1.0, 2.0, 3.0, 4.0][:sample_count] # Fallback to fixed times
262
+
263
+ for i, t in enumerate(timestamps):
264
+ if progress is not None:
265
+ progress(0.1 + (i / max(1, sample_count)) * 0.2, desc=f"Extracting frame {i+1}/{sample_count} for gallery...")
266
+
267
+ # Extract to a temp PNG first for best quality, then process with PIL
268
+ fd_raw, tmp_png_path = tempfile.mkstemp(suffix=".png")
269
+ os.close(fd_raw)
270
+
271
+ # Command to extract frame to PNG
272
+ cmd_extract = [
273
+ FFMPEG_BIN, "-nostdin", "-y", "-ss", str(t), "-i", media_path,
274
+ "-frames:v", "1", "-pix_fmt", "rgb24", tmp_png_path,
275
+ ]
276
+
277
+ try:
278
+ subprocess.run(cmd_extract, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=timeout_extract)
279
+
280
+ if os.path.exists(tmp_png_path) and os.path.getsize(tmp_png_path) > 0:
281
+ # Convert extracted PNG to a smaller JPEG and save as new temp file
282
+ jpeg_bytes = convert_to_jpeg_bytes(open(tmp_png_path, "rb").read(), base_h=base_h)
283
+ temp_jpeg_path = _temp_file(jpeg_bytes, suffix=f"_gallery_{i}.jpg") # _temp_file tracks this for cleanup
284
+ frame_paths.append(temp_jpeg_path)
285
+
286
+ except Exception as e:
287
+ print(f"Error processing frame {i+1} for gallery: {e}")
288
+ finally:
289
+ if os.path.exists(tmp_png_path):
290
+ try: os.remove(tmp_png_path)
291
+ except Exception: pass
292
+
293
+ if progress is not None:
294
+ progress(0.45, desc=f"Extracted {len(frame_paths)} frames for gallery")
295
+ return frame_paths
296
+
297
  def chat_complete(client, model: str, messages, timeout: int = 120, progress=None) -> str:
298
  try:
299
  if progress is not None:
 
393
  except Exception as e:
394
  return f"Error analyzing image: {e}"
395
 
396
+ def analyze_video_cohesive(client, video_path: str, prompt: str, progress=None) -> Tuple[str, List[str]]:
397
+ """
398
+ Analyzes video, either by uploading or by extracting frames.
399
+ Returns analysis result (str) and a list of paths to gallery frames (List[str]).
400
+ """
401
+ gallery_frame_paths: List[str] = []
402
  try:
403
  if progress is not None:
404
  progress(0.3, desc="Uploading video for full analysis...")
 
411
  {"role": "system", "content": SYSTEM_INSTRUCTION},
412
  {"role": "user", "content": extra_msg + "\n\n" + prompt},
413
  ]
414
+ result = chat_complete(client, VIDEO_MODEL, messages, progress=progress)
415
+ # If successful upload, still extract frames for gallery display
416
+ gallery_frame_paths = extract_and_save_frames_for_gallery(video_path, sample_count=6, base_h=128, progress=progress)
417
+ return result, gallery_frame_paths
418
  except Exception as e:
419
  if progress is not None:
420
  progress(0.35, desc="Upload failed, extracting frames as fallback...")
421
+
422
+ # Extract frames for model input (bytes)
423
+ frames_for_model_bytes = extract_best_frames_bytes(video_path, sample_count=6, progress=progress)
424
+
425
+ # Extract and save frames for gallery display (paths)
426
+ gallery_frame_paths = extract_and_save_frames_for_gallery(video_path, sample_count=6, base_h=128, progress=progress)
427
+
428
+ if not frames_for_model_bytes:
429
+ return f"Error: could not upload video and no frames could be extracted. ({e})", []
430
+
431
  image_entries = []
432
+ for i, fb in enumerate(frames_for_model_bytes, start=1):
433
  try:
434
  if progress is not None:
435
+ progress(0.4 + (i / len(frames_for_model_bytes)) * 0.2, desc=f"Preparing frame {i}/{len(frames_for_model_bytes)} for model...")
436
  j = convert_to_jpeg_bytes(fb, base_h=720)
437
  image_entries.append(
438
  {
 
448
  {"role": "system", "content": SYSTEM_INSTRUCTION},
449
  {"role": "user", "content": content},
450
  ]
451
+ result = chat_complete(client, PIXTRAL_MODEL, messages, progress=progress)
452
+ return result, gallery_frame_paths
453
 
454
  # --- FFmpeg Helpers for Preview ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
455
  def _convert_video_for_preview_if_needed(path: str) -> str:
456
  """
457
  Returns a path that the Gradio video component can play.
 
462
  if not FFMPEG_BIN or not os.path.exists(path):
463
  return path # Cannot convert, return original
464
 
 
465
  if path.lower().endswith((".mp4", ".m4v", ".mov")):
466
  info = _ffprobe_streams(path)
467
  if info:
 
469
  if video_streams and any(s.get("codec_name") in ("h264", "h265", "avc1") for s in video_streams):
470
  return path # Already playable
471
 
 
472
  out_path = _temp_file(b"", suffix=".mp4") # Create an empty temp file and add to cleanup list
473
  cmd = [
474
  FFMPEG_BIN, "-y", "-i", path,
 
485
  _temp_preview_files_to_delete.remove(out_path)
486
  try: os.remove(out_path)
487
  except Exception: pass
488
+ return path
489
 
490
  # --- Preview Generation Logic ---
491
  def _get_playable_preview_path_from_raw(src_url: str, raw_bytes: bytes) -> str:
 
496
  is_img, is_vid = determine_media_type(src_url)
497
 
498
  if is_vid:
 
499
  temp_raw_video_path = _temp_file(raw_bytes, suffix=ext_from_src(src_url) or ".mp4")
 
 
500
  playable_path = _convert_video_for_preview_if_needed(temp_raw_video_path)
501
 
502
+ # If conversion created a *new* temp path, and the original raw video path
503
+ # is no longer needed (and different), remove the raw path's tracking.
504
+ if playable_path != temp_raw_video_path and temp_raw_video_path in _temp_preview_files_to_delete:
505
+ _temp_preview_files_to_delete.remove(temp_raw_video_path)
 
506
  try: os.remove(temp_raw_video_path)
507
  except Exception: pass
508
  return playable_path
509
+ else:
 
510
  return _temp_file(convert_to_jpeg_bytes(raw_bytes, base_h=1024), suffix=".jpg")
511
 
512
  def _fetch_with_retries_bytes(src: str, timeout: int = 15, max_retries: int = 3):
 
516
  attempt += 1
517
  try:
518
  if is_remote(src):
519
+ with requests.get(src, timeout=timeout, stream=True) as r:
520
+ r.raise_for_status()
521
  return r.content
 
 
 
 
 
522
  else:
523
  with open(src, "rb") as fh:
524
  return fh.read()
525
+ except requests.exceptions.RequestException as e:
526
+ if attempt >= max_retries:
527
+ raise RuntimeError(f"Failed to fetch {src} after {max_retries} attempts: {e}")
528
+ print(f"Retrying fetch for {src} ({attempt}/{max_retries}). Delaying {delay:.1f}s...")
529
  time.sleep(delay)
530
  delay *= 2
531
  except FileNotFoundError:
532
  raise
533
+ except Exception as e:
534
+ if attempt >= max_retries:
535
+ raise RuntimeError(f"Failed to fetch {src} after {max_retries} attempts due to unexpected error: {e}")
536
+ print(f"Retrying fetch for {src} ({attempt}/{max_retries}). Delaying {delay:.1f}s...")
537
  time.sleep(delay)
538
  delay *= 2
539
 
 
550
  is_img, is_vid = determine_media_type(src)
551
  if is_vid:
552
  return _convert_video_for_preview_if_needed(src)
553
+ return src # Local image, return as is (assuming Gradio can display it)
554
+ return None
555
 
556
  # Remote source
557
  try:
 
575
  with gr.Column(scale=1):
576
  preview_image = gr.Image(label="Preview Image", type="filepath", elem_classes="preview_media", visible=False)
577
  preview_video = gr.Video(label="Preview Video", elem_classes="preview_media", visible=False, format="mp4")
578
+ # New gallery for screenshots, visible=False by default
579
+ screenshot_gallery = gr.Gallery(label="Extracted Screenshots", columns=5, rows=1, height="auto", object_fit="contain", visible=False)
580
  preview_status = gr.Textbox(label="Preview status", interactive=False, lines=1, value="", visible=True)
581
  with gr.Column(scale=2):
582
  url_input = gr.Textbox(label="Image / Video URL", placeholder="https://...", lines=1)
 
590
  progress_md = gr.Markdown("Idle")
591
  output_md = gr.Markdown("")
592
 
593
+ # State to track overall processing status
594
  status_state = gr.State("idle")
595
+ # State to hold the current path of the main preview (image/video)
596
+ main_preview_path_state = gr.State("")
597
+ # State to hold the list of screenshot paths for the gallery
598
+ screenshot_paths_state = gr.State([])
599
 
600
+ def clear_all_files_and_ui():
 
601
  """
602
+ Cleans up all tracked temporary files and resets all relevant UI components.
603
+ This function is meant to be called at the start of any new processing
604
+ or when the user explicitly clicks "Clear".
605
  """
606
+ for f_path in list(_temp_preview_files_to_delete):
607
+ if os.path.exists(f_path):
608
+ try:
609
+ os.remove(f_path)
610
+ except Exception as e:
611
+ print(f"Error during proactive cleanup of {f_path}: {e}")
612
+ _temp_preview_files_to_delete.clear()
613
+
614
+ return "", \
615
+ gr.update(value=None, visible=False), \
616
+ gr.update(value=None, visible=False), \
617
+ gr.update(value=[], visible=False), \
618
+ "idle", "Idle", "", "", "", [], gr.update(value="", visible=True) # url_input, preview_image, preview_video, screenshot_gallery, status_state, progress_md, output_md, main_preview_path_state, screenshot_paths_state, preview_status
619
+
620
+ clear_btn.click(
621
+ fn=clear_all_files_and_ui,
622
+ inputs=[],
623
+ outputs=[url_input, preview_image, preview_video, screenshot_gallery, status_state, progress_md, output_md, main_preview_path_state, screenshot_paths_state, preview_status]
624
+ )
625
+
626
+ # Function to handle URL input change and update main preview
627
+ def load_main_preview_and_clear_old(url: str):
628
+ # First, clear all existing temporary files and reset UI components
629
+ # This ensures a clean slate before loading new content
630
+ _, img_update_clear, video_update_clear, gallery_update_clear, _, _, _, \
631
+ main_path_clear, screenshot_paths_clear, status_update_clear = clear_all_files_and_ui() # Call the cleanup function
632
+
633
  if not url:
634
+ return img_update_clear, video_update_clear, gallery_update_clear, \
635
+ gr.update(value="", visible=True), main_path_clear, screenshot_paths_clear
636
 
637
  try:
638
+ local_playable_path = _save_local_playable_preview(url) # This adds to _temp_preview_files_to_delete
639
  if not local_playable_path:
640
+ return img_update_clear, video_update_clear, gallery_update_clear, \
641
+ gr.update(value="Preview load failed: could not fetch resource or make playable.", visible=True), \
642
+ "", []
643
 
 
644
  is_img_preview = False
645
  try:
646
  Image.open(local_playable_path).verify()
647
  is_img_preview = True
648
  except Exception:
649
+ pass
650
 
651
  if is_img_preview:
652
+ return gr.update(value=local_playable_path, visible=True), gr.update(value=None, visible=False), \
653
+ gr.update(value=[], visible=False), gr.update(value="Image preview loaded.", visible=True), \
654
+ local_playable_path, []
655
+ else: # Assume video
656
+ return gr.update(value=None, visible=False), gr.update(value=local_playable_path, visible=True), \
657
+ gr.update(value=[], visible=False), gr.update(value="Video preview loaded.", visible=True), \
658
+ local_playable_path, []
659
 
660
  except Exception as e:
661
+ return gr.update(value=None, visible=False), gr.update(value=None, visible=False), \
662
+ gr.update(value=[], visible=False), gr.update(value=f"Preview load failed: {e}", visible=True), \
663
+ "", []
664
 
 
665
  url_input.change(
666
+ fn=load_main_preview_and_clear_old,
667
  inputs=[url_input],
668
+ outputs=[preview_image, preview_video, screenshot_gallery, preview_status, main_preview_path_state, screenshot_paths_state]
669
  )
670
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
671
  def worker(url: str, prompt: str, key: str, progress=gr.Progress()):
672
  """
673
  Performs the media analysis.
674
+ Returns (status, markdown_output, main_preview_path_for_state, screenshot_paths_for_state).
675
  """
676
+ temp_media_file_for_analysis = None
677
+ generated_main_preview_path = "" # This should reflect the preview that was loaded by load_main_preview_and_clear_old
678
+ generated_screenshot_paths: List[str] = [] # List of paths for gallery
679
+ result_text = ""
680
+
681
  try:
682
  if not url:
683
+ return "error", "**Error:** No URL provided.", "", []
684
 
685
  progress(0.01, desc="Starting media processing")
686
  progress(0.02, desc="Checking URL / content‑type")
 
695
  progress(0.05, desc="Downloading video for analysis")
696
  raw_bytes = fetch_bytes(url, timeout=120, progress=progress)
697
  if not raw_bytes:
698
+ return "error", "Failed to download video bytes.", "", []
699
 
 
700
  temp_media_file_for_analysis = _temp_file(raw_bytes, suffix=ext_from_src(url) or ".mp4")
701
+ generated_main_preview_path = _get_playable_preview_path_from_raw(url, raw_bytes) # This generates the main video preview path
 
 
702
 
703
  progress(0.25, desc="Running full‑video analysis")
704
+ result_text, generated_screenshot_paths = analyze_video_cohesive(client, temp_media_file_for_analysis, prompt, progress=progress)
705
 
706
  # --- Image Processing Path ---
707
  elif is_img:
 
709
  raw_bytes = fetch_bytes(url, progress=progress)
710
 
711
  progress(0.15, desc="Preparing image preview")
712
+ generated_main_preview_path = _get_playable_preview_path_from_raw(url, raw_bytes) # This generates the main image preview path
713
 
714
  progress(0.20, desc="Running image analysis")
715
+ result_text = analyze_image_structured(client, raw_bytes, prompt, progress=progress)
716
+ # No screenshots for images
717
 
718
  # --- Unknown Media Type (Fallback) ---
719
  else:
720
  progress(0.07, desc="Downloading unknown media for type determination")
721
  raw_bytes = fetch_bytes(url, timeout=120, progress=progress)
722
 
 
723
  is_definitely_img = False
724
  try:
725
  Image.open(BytesIO(raw_bytes)).verify()
 
729
 
730
  if is_definitely_img:
731
  progress(0.15, desc="Preparing image preview (fallback)")
732
+ generated_main_preview_path = _get_playable_preview_path_from_raw(url, raw_bytes)
733
  progress(0.20, desc="Running image analysis (fallback)")
734
+ result_text = analyze_image_structured(client, raw_bytes, prompt, progress=progress)
735
  else: # Treat as video fallback
736
  progress(0.15, desc="Preparing video preview (fallback)")
737
  temp_media_file_for_analysis = _temp_file(raw_bytes, suffix=ext_from_src(url) or ".mp4")
738
+ generated_main_preview_path = _get_playable_preview_path_from_raw(url, raw_bytes)
739
  progress(0.25, desc="Running video analysis (fallback)")
740
+ result_text, generated_screenshot_paths = analyze_video_cohesive(client, temp_media_file_for_analysis, prompt, progress=progress)
741
 
742
+ status = "done" if not (isinstance(result_text, str) and result_text.lower().startswith("error")) else "error"
743
 
744
+ return status, result_text, generated_main_preview_path, generated_screenshot_paths
745
 
746
  except Exception as exc:
747
+ return "error", f"Unexpected worker error: {exc}", "", []
748
  finally:
 
749
  if temp_media_file_for_analysis and os.path.exists(temp_media_file_for_analysis):
750
  if temp_media_file_for_analysis in _temp_preview_files_to_delete:
751
+ _temp_preview_files_to_delete.remove(temp_media_file_for_analysis)
752
  try: os.remove(temp_media_file_for_analysis)
753
  except Exception as e: print(f"Error cleaning up analysis temp file {temp_media_file_for_analysis}: {e}")
754
 
755
+ # Worker output changed to include screenshot_paths_state
756
  submit_btn.click(
757
  fn=worker,
758
  inputs=[url_input, custom_prompt, api_key],
759
+ outputs=[status_state, output_md, main_preview_path_state, screenshot_paths_state],
760
  show_progress="full",
761
  show_progress_on=progress_md,
762
  )
 
769
  return {"idle": "Idle", "busy": "Processing…", "done": "Completed", "error": "Error — see output"}.get(s, s)
770
  status_state.change(fn=status_to_progress_text, inputs=[status_state], outputs=[progress_md])
771
 
772
+ # This function updates the UI components based on the state values.
773
+ # It should *not* perform cleanup, as that's handled by clear_all_files_and_ui or load_main_preview_and_clear_old.
774
+ def _update_preview_components(current_main_preview_path: str, current_screenshot_paths: List[str]):
775
+ img_update = gr.update(value=None, visible=False)
776
+ video_update = gr.update(value=None, visible=False)
777
+
778
+ if current_main_preview_path:
 
779
  try:
780
+ is_img_preview = False
781
+ try:
782
+ Image.open(current_main_preview_path).verify()
783
+ is_img_preview = True
784
+ except Exception:
785
+ pass # Not an image, treat as video
786
+
787
+ if is_img_preview:
788
+ img_update = gr.update(value=current_main_preview_path, visible=True)
789
+ else:
790
+ video_update = gr.update(value=current_main_preview_path, visible=True)
791
  except Exception as e:
792
+ print(f"Error setting main preview from path {current_main_preview_path}: {e}")
793
 
794
+ # Gallery is visible only if there are paths
795
+ gallery_update = gr.update(value=current_screenshot_paths, visible=bool(current_screenshot_paths))
796
+ return img_update, video_update, gallery_update
797
 
798
+ # Register changes to the states to update the UI components
799
+ main_preview_path_state.change(
800
+ fn=_update_preview_components,
801
+ inputs=[main_preview_path_state, screenshot_paths_state],
802
+ outputs=[preview_image, preview_video, screenshot_gallery]
803
+ )
804
+ screenshot_paths_state.change(
805
+ fn=_update_preview_components,
806
+ inputs=[main_preview_path_state, screenshot_paths_state],
807
+ outputs=[preview_image, preview_video, screenshot_gallery]
 
 
 
 
 
 
 
 
 
 
 
 
 
808
  )
809
 
810
  return demo