Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
|
@@ -98,7 +98,11 @@ def _temp_file(data: bytes, suffix: str) -> str:
|
|
| 98 |
"""
|
| 99 |
Write *data* to a temporary file and return its absolute path.
|
| 100 |
The path is added to `_temp_preview_files_to_delete` for automatic cleanup.
|
|
|
|
| 101 |
"""
|
|
|
|
|
|
|
|
|
|
| 102 |
fd, path = tempfile.mkstemp(suffix=suffix)
|
| 103 |
os.close(fd)
|
| 104 |
with open(path, "wb") as f:
|
|
@@ -150,7 +154,12 @@ def fetch_bytes(src: str, stream_threshold: int = STREAM_THRESHOLD, timeout: int
|
|
| 150 |
return data
|
| 151 |
|
| 152 |
def convert_to_jpeg_bytes(img_bytes: bytes, base_h: int = 480) -> bytes:
|
| 153 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
try:
|
| 155 |
if getattr(img, "is_animated", False): # Handles animated GIFs by taking first frame
|
| 156 |
img.seek(0)
|
|
@@ -245,12 +254,19 @@ def extract_frames_for_model_and_gallery(media_path: str, sample_count: int = 5,
|
|
| 245 |
|
| 246 |
# For model: convert to high-res JPEG bytes (model expects this)
|
| 247 |
jpeg_model_bytes = convert_to_jpeg_bytes(raw_frame_bytes, base_h=720) # Keep higher res for model
|
| 248 |
-
|
|
|
|
| 249 |
|
| 250 |
# For gallery: convert to smaller JPEG bytes and save as new temp file
|
| 251 |
jpeg_gallery_bytes = convert_to_jpeg_bytes(raw_frame_bytes, base_h=gallery_base_h)
|
| 252 |
-
|
| 253 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 254 |
|
| 255 |
except Exception as e:
|
| 256 |
print(f"Error processing frame {i+1} for model/gallery: {e}")
|
|
@@ -273,7 +289,7 @@ def chat_complete(client, model: str, messages, timeout: int = 120, progress=Non
|
|
| 273 |
|
| 274 |
# Prefer using the Mistral client if available and functional
|
| 275 |
if hasattr(client, "chat") and hasattr(client.chat, "complete"):
|
| 276 |
-
res = client.chat.complete(model=model, messages=messages, stream=False,
|
| 277 |
else:
|
| 278 |
api_key = getattr(client, "api_key", "") or DEFAULT_KEY
|
| 279 |
if not api_key:
|
|
@@ -394,6 +410,8 @@ def analyze_image_structured(client, img_bytes: bytes, prompt: str, progress=Non
|
|
| 394 |
if progress is not None:
|
| 395 |
progress(0.3, desc="Preparing image for analysis...")
|
| 396 |
jpeg = convert_to_jpeg_bytes(img_bytes, base_h=1024)
|
|
|
|
|
|
|
| 397 |
data_url = b64_bytes(jpeg, mime="image/jpeg")
|
| 398 |
messages = [
|
| 399 |
{"role": "system", "content": SYSTEM_INSTRUCTION},
|
|
@@ -484,7 +502,11 @@ def _convert_video_for_preview_if_needed(path: str) -> str:
|
|
| 484 |
if video_streams and any(s.get("codec_name") in ("h264", "h265", "avc1") for s in video_streams):
|
| 485 |
return path # Already playable
|
| 486 |
|
| 487 |
-
out_path = _temp_file(b"", suffix=".mp4") # Create an empty temp file and add to cleanup list
|
|
|
|
|
|
|
|
|
|
|
|
|
| 488 |
cmd = [
|
| 489 |
FFMPEG_BIN, "-y", "-i", path,
|
| 490 |
"-c:v", "libx264", "-preset", "veryfast", "-crf", "28", # H.264 video codec
|
|
@@ -508,10 +530,15 @@ def _convert_video_for_preview_if_needed(path: str) -> str:
|
|
| 508 |
def _get_playable_preview_path_from_raw(src_url: str, raw_bytes: bytes, is_image_hint: bool, is_video_hint: bool) -> str:
|
| 509 |
"""
|
| 510 |
Generates a playable preview file from raw bytes, using pre-determined media type hints.
|
| 511 |
-
Creates and tracks a new temporary file.
|
| 512 |
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 513 |
if is_video_hint: # Use the passed hint
|
| 514 |
temp_raw_video_path = _temp_file(raw_bytes, suffix=ext_from_src(src_url) or ".mp4")
|
|
|
|
| 515 |
playable_path = _convert_video_for_preview_if_needed(temp_raw_video_path)
|
| 516 |
|
| 517 |
# If conversion created a *new* temp path, and the original raw video path
|
|
@@ -522,32 +549,52 @@ def _get_playable_preview_path_from_raw(src_url: str, raw_bytes: bytes, is_image
|
|
| 522 |
except Exception: pass
|
| 523 |
return playable_path
|
| 524 |
elif is_image_hint: # Use the passed hint
|
| 525 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 526 |
else:
|
| 527 |
# Fallback if hints are unclear, try image first, then video
|
| 528 |
try:
|
| 529 |
# Attempt to open as image
|
| 530 |
img_bytes_io = BytesIO(raw_bytes)
|
| 531 |
-
Image.open(img_bytes_io)
|
|
|
|
| 532 |
img_bytes_io.seek(0) # Reset stream position after verify
|
| 533 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 534 |
except UnidentifiedImageError:
|
| 535 |
# If not an image, assume it's a video for preview purposes
|
| 536 |
print(f"Warning: Unknown media type for {src_url}, falling back to video preview attempt.")
|
| 537 |
temp_raw_video_path = _temp_file(raw_bytes, suffix=ext_from_src(src_url) or ".mp4")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 538 |
playable_path = _convert_video_for_preview_if_needed(temp_raw_video_path)
|
| 539 |
-
return playable_path
|
| 540 |
|
| 541 |
-
|
|
|
|
| 542 |
attempt = 0
|
| 543 |
delay = 1.0
|
| 544 |
while True:
|
| 545 |
attempt += 1
|
| 546 |
try:
|
| 547 |
if is_remote(src):
|
| 548 |
-
|
| 549 |
-
|
| 550 |
-
|
|
|
|
|
|
|
|
|
|
| 551 |
else:
|
| 552 |
with open(src, "rb") as fh:
|
| 553 |
return fh.read()
|
|
@@ -570,6 +617,7 @@ def _save_local_playable_preview(src: str, is_image_hint: bool, is_video_hint: b
|
|
| 570 |
"""
|
| 571 |
Fetches remote content or reads local, then ensures it's in a playable format
|
| 572 |
for Gradio preview components, using media type hints.
|
|
|
|
| 573 |
"""
|
| 574 |
if not src:
|
| 575 |
return None
|
|
@@ -578,16 +626,21 @@ def _save_local_playable_preview(src: str, is_image_hint: bool, is_video_hint: b
|
|
| 578 |
if os.path.exists(src):
|
| 579 |
if is_video_hint:
|
| 580 |
return _convert_video_for_preview_if_needed(src)
|
| 581 |
-
# For local images, return the path directly.
|
| 582 |
-
# _get_playable_preview_path_from_raw's internal checks would handle it.
|
| 583 |
-
return src
|
| 584 |
return None
|
| 585 |
|
| 586 |
# Remote source
|
| 587 |
try:
|
| 588 |
raw_bytes = _fetch_with_retries_bytes(src, timeout=15, max_retries=3)
|
| 589 |
-
if not raw_bytes:
|
| 590 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 591 |
except Exception as e:
|
| 592 |
print(f"Error creating local playable preview from {src}: {e}")
|
| 593 |
return None
|
|
@@ -706,7 +759,7 @@ def create_demo():
|
|
| 706 |
elif ext in VIDEO_EXTS:
|
| 707 |
is_vid_preview = True
|
| 708 |
|
| 709 |
-
# Fallback to PIL check if extension is ambiguous or unknown
|
| 710 |
if not is_img_preview and not is_vid_preview and os.path.exists(local_playable_path):
|
| 711 |
try:
|
| 712 |
Image.open(local_playable_path).verify()
|
|
@@ -780,7 +833,8 @@ def create_demo():
|
|
| 780 |
# --- Video Processing Path ---
|
| 781 |
if is_vid_worker:
|
| 782 |
temp_media_file_for_analysis = _temp_file(raw_bytes, suffix=ext_from_src(url) or ".mp4")
|
| 783 |
-
|
|
|
|
| 784 |
generated_main_preview_path = _get_playable_preview_path_from_raw(url, raw_bytes, is_img_worker, is_vid_worker)
|
| 785 |
|
| 786 |
progress(0.25, desc="Running full‑video analysis")
|
|
@@ -788,7 +842,6 @@ def create_demo():
|
|
| 788 |
|
| 789 |
# --- Image Processing Path ---
|
| 790 |
elif is_img_worker:
|
| 791 |
-
# Pass determined types to _get_playable_preview_path_from_raw
|
| 792 |
generated_main_preview_path = _get_playable_preview_path_from_raw(url, raw_bytes, is_img_worker, is_vid_worker)
|
| 793 |
|
| 794 |
progress(0.20, desc="Running image analysis")
|
|
@@ -800,6 +853,8 @@ def create_demo():
|
|
| 800 |
# If after all checks, it's still unknown, treat as video by default for analysis.
|
| 801 |
print(f"Warning: Could not definitively determine media type for {url}. Attempting video analysis.")
|
| 802 |
temp_media_file_for_analysis = _temp_file(raw_bytes, suffix=ext_from_src(url) or ".mp4")
|
|
|
|
|
|
|
| 803 |
# Even though type is 'unknown', we'll hint as video for preview creation if it's not an image
|
| 804 |
generated_main_preview_path = _get_playable_preview_path_from_raw(url, raw_bytes, False, True)
|
| 805 |
progress(0.25, desc="Running video analysis (fallback for unknown type)")
|
|
|
|
| 98 |
"""
|
| 99 |
Write *data* to a temporary file and return its absolute path.
|
| 100 |
The path is added to `_temp_preview_files_to_delete` for automatic cleanup.
|
| 101 |
+
Returns empty string if data is empty.
|
| 102 |
"""
|
| 103 |
+
if not data:
|
| 104 |
+
return "" # Do not create a file for empty data
|
| 105 |
+
|
| 106 |
fd, path = tempfile.mkstemp(suffix=suffix)
|
| 107 |
os.close(fd)
|
| 108 |
with open(path, "wb") as f:
|
|
|
|
| 154 |
return data
|
| 155 |
|
| 156 |
def convert_to_jpeg_bytes(img_bytes: bytes, base_h: int = 480) -> bytes:
|
| 157 |
+
try:
|
| 158 |
+
img = Image.open(BytesIO(img_bytes))
|
| 159 |
+
except UnidentifiedImageError:
|
| 160 |
+
print(f"Warning: convert_to_jpeg_bytes received unidentifiable image data.")
|
| 161 |
+
return b"" # Return empty bytes if data is not an identifiable image
|
| 162 |
+
|
| 163 |
try:
|
| 164 |
if getattr(img, "is_animated", False): # Handles animated GIFs by taking first frame
|
| 165 |
img.seek(0)
|
|
|
|
| 254 |
|
| 255 |
# For model: convert to high-res JPEG bytes (model expects this)
|
| 256 |
jpeg_model_bytes = convert_to_jpeg_bytes(raw_frame_bytes, base_h=720) # Keep higher res for model
|
| 257 |
+
if jpeg_model_bytes: # Only append if conversion was successful
|
| 258 |
+
frames_for_model.append(jpeg_model_bytes)
|
| 259 |
|
| 260 |
# For gallery: convert to smaller JPEG bytes and save as new temp file
|
| 261 |
jpeg_gallery_bytes = convert_to_jpeg_bytes(raw_frame_bytes, base_h=gallery_base_h)
|
| 262 |
+
if jpeg_gallery_bytes: # Only create temp file if conversion was successful
|
| 263 |
+
temp_jpeg_path = _temp_file(jpeg_gallery_bytes, suffix=f"_gallery_{i}.jpg") # _temp_file tracks this for cleanup
|
| 264 |
+
if temp_jpeg_path: # Only add to gallery if temp file was successfully created
|
| 265 |
+
frame_paths_for_gallery.append(temp_jpeg_path)
|
| 266 |
+
else:
|
| 267 |
+
print(f"Warning: Failed to convert extracted frame {i+1} to JPEG for gallery.")
|
| 268 |
+
else:
|
| 269 |
+
print(f"Warning: Extracted frame {i+1} was empty or non-existent at {tmp_png_path}.")
|
| 270 |
|
| 271 |
except Exception as e:
|
| 272 |
print(f"Error processing frame {i+1} for model/gallery: {e}")
|
|
|
|
| 289 |
|
| 290 |
# Prefer using the Mistral client if available and functional
|
| 291 |
if hasattr(client, "chat") and hasattr(client.chat, "complete"):
|
| 292 |
+
res = client.chat.complete(model=model, messages=messages, stream=False, timeout_ms=timeout * 1000) # FIX: use timeout_ms
|
| 293 |
else:
|
| 294 |
api_key = getattr(client, "api_key", "") or DEFAULT_KEY
|
| 295 |
if not api_key:
|
|
|
|
| 410 |
if progress is not None:
|
| 411 |
progress(0.3, desc="Preparing image for analysis...")
|
| 412 |
jpeg = convert_to_jpeg_bytes(img_bytes, base_h=1024)
|
| 413 |
+
if not jpeg: # Handle case where convert_to_jpeg_bytes failed
|
| 414 |
+
return "Error: Could not convert image for analysis."
|
| 415 |
data_url = b64_bytes(jpeg, mime="image/jpeg")
|
| 416 |
messages = [
|
| 417 |
{"role": "system", "content": SYSTEM_INSTRUCTION},
|
|
|
|
| 502 |
if video_streams and any(s.get("codec_name") in ("h264", "h265", "avc1") for s in video_streams):
|
| 503 |
return path # Already playable
|
| 504 |
|
| 505 |
+
out_path = _temp_file(b"", suffix=".mp4") # Create an empty temp file and add to cleanup list (will return "" if data is b"")
|
| 506 |
+
if not out_path: # If _temp_file returned empty path
|
| 507 |
+
print(f"Error: Could not create temporary file for video conversion from {path}.")
|
| 508 |
+
return path # Fallback to original path
|
| 509 |
+
|
| 510 |
cmd = [
|
| 511 |
FFMPEG_BIN, "-y", "-i", path,
|
| 512 |
"-c:v", "libx264", "-preset", "veryfast", "-crf", "28", # H.264 video codec
|
|
|
|
| 530 |
def _get_playable_preview_path_from_raw(src_url: str, raw_bytes: bytes, is_image_hint: bool, is_video_hint: bool) -> str:
|
| 531 |
"""
|
| 532 |
Generates a playable preview file from raw bytes, using pre-determined media type hints.
|
| 533 |
+
Creates and tracks a new temporary file. Returns empty string if no preview can be made.
|
| 534 |
"""
|
| 535 |
+
if not raw_bytes:
|
| 536 |
+
print(f"Error: No raw bytes provided for preview generation of {src_url}.")
|
| 537 |
+
return ""
|
| 538 |
+
|
| 539 |
if is_video_hint: # Use the passed hint
|
| 540 |
temp_raw_video_path = _temp_file(raw_bytes, suffix=ext_from_src(src_url) or ".mp4")
|
| 541 |
+
if not temp_raw_video_path: return "" # Handle if _temp_file failed
|
| 542 |
playable_path = _convert_video_for_preview_if_needed(temp_raw_video_path)
|
| 543 |
|
| 544 |
# If conversion created a *new* temp path, and the original raw video path
|
|
|
|
| 549 |
except Exception: pass
|
| 550 |
return playable_path
|
| 551 |
elif is_image_hint: # Use the passed hint
|
| 552 |
+
jpeg_bytes = convert_to_jpeg_bytes(raw_bytes, base_h=1024)
|
| 553 |
+
if not jpeg_bytes: # Handle if convert_to_jpeg_bytes failed
|
| 554 |
+
print(f"Warning: Could not convert image bytes for {src_url} to JPEG.")
|
| 555 |
+
return ""
|
| 556 |
+
return _temp_file(jpeg_bytes, suffix=".jpg")
|
| 557 |
else:
|
| 558 |
# Fallback if hints are unclear, try image first, then video
|
| 559 |
try:
|
| 560 |
# Attempt to open as image
|
| 561 |
img_bytes_io = BytesIO(raw_bytes)
|
| 562 |
+
img = Image.open(img_bytes_io)
|
| 563 |
+
img.verify() # Checks if it's a valid image without loading all pixels
|
| 564 |
img_bytes_io.seek(0) # Reset stream position after verify
|
| 565 |
+
jpeg_bytes = convert_to_jpeg_bytes(img_bytes_io.read(), base_h=1024)
|
| 566 |
+
if jpeg_bytes: # If conversion yielded bytes, create temp file
|
| 567 |
+
return _temp_file(jpeg_bytes, suffix=".jpg")
|
| 568 |
+
else: # If convert_to_jpeg_bytes failed even after verify
|
| 569 |
+
raise UnidentifiedImageError("Could not convert verified image to JPEG.") # Re-raise to fall to video path
|
| 570 |
except UnidentifiedImageError:
|
| 571 |
# If not an image, assume it's a video for preview purposes
|
| 572 |
print(f"Warning: Unknown media type for {src_url}, falling back to video preview attempt.")
|
| 573 |
temp_raw_video_path = _temp_file(raw_bytes, suffix=ext_from_src(src_url) or ".mp4")
|
| 574 |
+
if not temp_raw_video_path: return "" # If _temp_file returns empty due to empty raw_bytes
|
| 575 |
+
playable_path = _convert_video_for_preview_if_needed(temp_raw_video_path)
|
| 576 |
+
return playable_path
|
| 577 |
+
except Exception as e: # Catch other potential PIL/IO errors during image check
|
| 578 |
+
print(f"Warning: Generic error during image check for {src_url}: {e}. Falling back to video preview attempt.")
|
| 579 |
+
temp_raw_video_path = _temp_file(raw_bytes, suffix=ext_from_src(src_url) or ".mp4")
|
| 580 |
+
if not temp_raw_video_path: return ""
|
| 581 |
playable_path = _convert_video_for_preview_if_needed(temp_raw_video_path)
|
| 582 |
+
return playable_path
|
| 583 |
|
| 584 |
+
|
| 585 |
+
def _fetch_with_retries_bytes(src: str, timeout: int = 15, max_retries: int = 3) -> bytes:
|
| 586 |
attempt = 0
|
| 587 |
delay = 1.0
|
| 588 |
while True:
|
| 589 |
attempt += 1
|
| 590 |
try:
|
| 591 |
if is_remote(src):
|
| 592 |
+
# Using requests.get without stream and directly returning content for simplicity here,
|
| 593 |
+
# as stream logic is primarily handled in fetch_bytes for large files.
|
| 594 |
+
# For preview, we often need the full file quickly.
|
| 595 |
+
r = requests.get(src, timeout=timeout)
|
| 596 |
+
r.raise_for_status()
|
| 597 |
+
return r.content
|
| 598 |
else:
|
| 599 |
with open(src, "rb") as fh:
|
| 600 |
return fh.read()
|
|
|
|
| 617 |
"""
|
| 618 |
Fetches remote content or reads local, then ensures it's in a playable format
|
| 619 |
for Gradio preview components, using media type hints.
|
| 620 |
+
Returns None if no playable preview could be generated.
|
| 621 |
"""
|
| 622 |
if not src:
|
| 623 |
return None
|
|
|
|
| 626 |
if os.path.exists(src):
|
| 627 |
if is_video_hint:
|
| 628 |
return _convert_video_for_preview_if_needed(src)
|
| 629 |
+
return src # For local images, return the path directly.
|
|
|
|
|
|
|
| 630 |
return None
|
| 631 |
|
| 632 |
# Remote source
|
| 633 |
try:
|
| 634 |
raw_bytes = _fetch_with_retries_bytes(src, timeout=15, max_retries=3)
|
| 635 |
+
if not raw_bytes: # Handle case where fetch_bytes returns empty
|
| 636 |
+
print(f"Error: Failed to fetch any bytes for {src}.")
|
| 637 |
+
return None
|
| 638 |
+
|
| 639 |
+
playable_path = _get_playable_preview_path_from_raw(src, raw_bytes, is_image_hint, is_video_hint)
|
| 640 |
+
if not playable_path: # Handle case where _get_playable_preview_path_from_raw couldn't create a path
|
| 641 |
+
print(f"Error: No playable preview path generated for {src}.")
|
| 642 |
+
return None
|
| 643 |
+
return playable_path
|
| 644 |
except Exception as e:
|
| 645 |
print(f"Error creating local playable preview from {src}: {e}")
|
| 646 |
return None
|
|
|
|
| 759 |
elif ext in VIDEO_EXTS:
|
| 760 |
is_vid_preview = True
|
| 761 |
|
| 762 |
+
# Fallback to PIL check if extension is ambiguous or unknown, and if it's an actual file
|
| 763 |
if not is_img_preview and not is_vid_preview and os.path.exists(local_playable_path):
|
| 764 |
try:
|
| 765 |
Image.open(local_playable_path).verify()
|
|
|
|
| 833 |
# --- Video Processing Path ---
|
| 834 |
if is_vid_worker:
|
| 835 |
temp_media_file_for_analysis = _temp_file(raw_bytes, suffix=ext_from_src(url) or ".mp4")
|
| 836 |
+
if not temp_media_file_for_analysis: # Handle if _temp_file failed
|
| 837 |
+
return "error", "Failed to create temporary video file for analysis.", "", []
|
| 838 |
generated_main_preview_path = _get_playable_preview_path_from_raw(url, raw_bytes, is_img_worker, is_vid_worker)
|
| 839 |
|
| 840 |
progress(0.25, desc="Running full‑video analysis")
|
|
|
|
| 842 |
|
| 843 |
# --- Image Processing Path ---
|
| 844 |
elif is_img_worker:
|
|
|
|
| 845 |
generated_main_preview_path = _get_playable_preview_path_from_raw(url, raw_bytes, is_img_worker, is_vid_worker)
|
| 846 |
|
| 847 |
progress(0.20, desc="Running image analysis")
|
|
|
|
| 853 |
# If after all checks, it's still unknown, treat as video by default for analysis.
|
| 854 |
print(f"Warning: Could not definitively determine media type for {url}. Attempting video analysis.")
|
| 855 |
temp_media_file_for_analysis = _temp_file(raw_bytes, suffix=ext_from_src(url) or ".mp4")
|
| 856 |
+
if not temp_media_file_for_analysis: # Handle if _temp_file failed
|
| 857 |
+
return "error", "Failed to create temporary video file for analysis (unknown type fallback).", "", []
|
| 858 |
# Even though type is 'unknown', we'll hint as video for preview creation if it's not an image
|
| 859 |
generated_main_preview_path = _get_playable_preview_path_from_raw(url, raw_bytes, False, True)
|
| 860 |
progress(0.25, desc="Running video analysis (fallback for unknown type)")
|