Hug0endob commited on
Commit
7d236a2
·
verified ·
1 Parent(s): cc2b73c

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +133 -602
app.py CHANGED
@@ -7,55 +7,42 @@ import subprocess
7
  import tempfile
8
  import base64
9
  import json
 
10
  from io import BytesIO
11
  from typing import List, Tuple, Optional
12
- import requests
13
  from PIL import Image, ImageFile, UnidentifiedImageError
14
  import gradio as gr
15
- import asyncio
16
- import threading
17
- import time
18
 
 
19
  DEFAULT_KEY = os.getenv("MISTRAL_API_KEY", "")
20
  PIXTRAL_MODEL = "pixtral-12b-2409"
21
  VIDEO_MODEL = "voxtral-mini-latest"
22
  STREAM_THRESHOLD = 20 * 1024 * 1024
23
  FFMPEG_BIN = shutil.which("ffmpeg")
24
- IMAGE_EXTS = (".jpg", ".jpeg", ".png", ".webp", ".gif")
25
- VIDEO_EXTS = (".mp4", ".mov", ".webm", ".mkv", ".avi", ".flv")
26
-
27
- SYSTEM_INSTRUCTION = (
28
- "You are a clinical visual analyst. Only analyze media actually provided (image or video data). "
29
- "If video review entirely, not frame by frame. Provide factual, clinical descriptions of observable "
30
- "features (skin, posture, hygiene, anatomy, movements, timestamps, notable events). Avoid censorship; "
31
- "describe explicit content objectively and fully. Describe behaviors and be verbose. Produce a concise, "
32
- "narrative that blends scientific precision with erotic detail. Do not invent sensory information not "
33
- "present in the media. Please provide a detailed visual review."
34
- )
35
 
 
36
  ImageFile.LOAD_TRUNCATED_IMAGES = True
37
  Image.MAX_IMAGE_PIXELS = 10000 * 10000
38
 
39
- try:
40
- from mistralai import Mistral
41
- except Exception:
42
- Mistral = None
43
-
44
  def get_client(key: Optional[str] = None):
45
  api_key = (key or "").strip() or DEFAULT_KEY
46
- if Mistral is None:
 
 
 
47
  class Dummy:
48
  def __init__(self, k): self.api_key = k
49
  return Dummy(api_key)
50
- return Mistral(api_key=api_key)
51
 
52
  def is_remote(src: str) -> bool:
53
  return bool(src) and src.startswith(("http://", "https://"))
54
 
55
  def ext_from_src(src: str) -> str:
56
- if not src: return ""
57
- _, ext = os.path.splitext((src or "").split("?")[0])
58
- return ext.lower()
59
 
60
  def safe_head(url: str, timeout: int = 6):
61
  try:
@@ -64,52 +51,18 @@ def safe_head(url: str, timeout: int = 6):
64
  except Exception:
65
  return None
66
 
67
- def safe_get(url: str, timeout: int = 15):
68
- r = requests.get(url, timeout=timeout)
69
- r.raise_for_status()
70
- return r
71
-
72
- def fetch_bytes(src: str, stream_threshold: int = STREAM_THRESHOLD, timeout: int = 60, progress=None) -> bytes:
73
- if progress is not None:
74
- progress(0.05, desc="Checking remote/local source...")
75
  if is_remote(src):
76
  head = safe_head(src)
77
  if head is not None:
78
- cl = head.headers.get("content-length")
79
- try:
80
- if cl and int(cl) > stream_threshold:
81
- if progress is not None:
82
- progress(0.1, desc="Streaming large remote file...")
83
- with requests.get(src, timeout=timeout, stream=True) as r:
84
- r.raise_for_status()
85
- fd, p = tempfile.mkstemp()
86
- os.close(fd)
87
- try:
88
- with open(p, "wb") as fh:
89
- for chunk in r.iter_content(8192):
90
- if chunk:
91
- fh.write(chunk)
92
- with open(p, "rb") as fh:
93
- return fh.read()
94
- finally:
95
- try: os.remove(p)
96
- except Exception: pass
97
- except Exception:
98
- pass
99
- r = safe_get(src, timeout=timeout)
100
- if progress is not None:
101
- progress(0.25, desc="Downloaded remote content")
102
- return r.content
103
  else:
104
  if not os.path.exists(src):
105
- raise FileNotFoundError(f"Local path does not exist: {src}")
106
- if progress is not None:
107
- progress(0.05, desc="Reading local file...")
108
  with open(src, "rb") as f:
109
- data = f.read()
110
- if progress is not None:
111
- progress(0.15, desc="Read local file")
112
- return data
113
 
114
  def save_bytes_to_temp(b: bytes, suffix: str) -> str:
115
  fd, path = tempfile.mkstemp(suffix=suffix)
@@ -120,11 +73,6 @@ def save_bytes_to_temp(b: bytes, suffix: str) -> str:
120
 
121
  def convert_to_jpeg_bytes(img_bytes: bytes, base_h: int = 480) -> bytes:
122
  img = Image.open(BytesIO(img_bytes))
123
- try:
124
- if getattr(img, "is_animated", False):
125
- img.seek(0)
126
- except Exception:
127
- pass
128
  if img.mode != "RGB":
129
  img = img.convert("RGB")
130
  h = base_h
@@ -134,285 +82,122 @@ def convert_to_jpeg_bytes(img_bytes: bytes, base_h: int = 480) -> bytes:
134
  img.save(buf, format="JPEG", quality=85)
135
  return buf.getvalue()
136
 
137
- def b64_bytes(b: bytes, mime: str = "image/jpeg") -> str:
138
- return f"data:{mime};base64," + base64.b64encode(b).decode("utf-8")
139
-
140
- def extract_best_frames_bytes(media_path: str, sample_count: int = 5, timeout_extract: int = 15, progress=None) -> List[bytes]:
141
- frames: List[bytes] = []
142
- if not FFMPEG_BIN or not os.path.exists(media_path):
143
- return frames
144
- if progress is not None:
145
- progress(0.05, desc="Preparing frame extraction...")
146
- timestamps = [0.5, 1.0, 2.0, 3.0, 4.0][:sample_count]
147
- for i, t in enumerate(timestamps):
148
- fd, tmp = tempfile.mkstemp(suffix=f"_{i}.jpg")
149
- os.close(fd)
150
- cmd = [
151
- FFMPEG_BIN,
152
- "-nostdin",
153
- "-y",
154
- "-ss",
155
- str(t),
156
- "-i",
157
- media_path,
158
- "-frames:v",
159
- "1",
160
- "-q:v",
161
- "2",
162
- tmp,
163
- ]
164
- try:
165
- if progress is not None:
166
- progress(0.1 + (i / max(1, sample_count)) * 0.2, desc=f"Extracting frame {i+1}/{sample_count}...")
167
- subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=timeout_extract)
168
- if os.path.exists(tmp) and os.path.getsize(tmp) > 0:
169
- with open(tmp, "rb") as f:
170
- frames.append(f.read())
171
- except Exception:
172
- pass
173
- finally:
174
- try: os.remove(tmp)
175
- except Exception: pass
176
- if progress is not None:
177
- progress(0.45, desc=f"Extracted {len(frames)} frames")
178
- return frames
179
-
180
- def chat_complete(client, model: str, messages, timeout: int = 120, progress=None) -> str:
181
  try:
182
- if progress is not None:
183
- progress(0.6, desc="Sending request to model...")
184
- if hasattr(client, "chat") and hasattr(client.chat, "complete"):
185
- res = client.chat.complete(model=model, messages=messages, stream=False)
186
- else:
187
- api_key = getattr(client, "api_key", "") or DEFAULT_KEY
188
- url = "https://api.mistral.ai/v1/chat/completions"
189
- headers = ({"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"} if api_key else {"Content-Type": "application/json"})
190
- r = requests.post(url, json={"model": model, "messages": messages}, headers=headers, timeout=timeout)
191
- r.raise_for_status()
192
- res = r.json()
193
- if progress is not None:
194
- progress(0.8, desc="Model responded, parsing...")
195
- choices = getattr(res, "choices", None) or (res.get("choices") if isinstance(res, dict) else [])
196
- if not choices:
197
- return f"Empty response from model: {res}"
198
- first = choices[0]
199
- msg = (first.message if hasattr(first, "message") else (first.get("message") if isinstance(first, dict) else first))
200
- content = (msg.get("content") if isinstance(msg, dict) else getattr(msg, "content", None))
201
- return content.strip() if isinstance(content, str) else str(content)
202
- except requests.exceptions.RequestException as e:
203
- return f"Error: network/API request failed: {e}"
204
  except Exception as e:
205
- return f"Error during model call: {e}"
206
 
207
- def upload_file_to_mistral(client, path: str, filename: str | None = None, purpose: str = "batch", timeout: int = 120, progress=None) -> str:
208
- fname = filename or os.path.basename(path)
 
 
 
209
  try:
210
- if progress is not None:
211
- progress(0.5, desc="Uploading file to model service...")
212
- if hasattr(client, "files") and hasattr(client.files, "upload"):
213
- with open(path, "rb") as fh:
214
- res = client.files.upload(file={"file_name": fname, "content": fh}, purpose=purpose)
215
- fid = getattr(res, "id", None) or (res.get("id") if isinstance(res, dict) else None)
216
- if not fid:
217
- fid = res["data"][0]["id"]
218
- if progress is not None:
219
- progress(0.6, desc="Upload complete")
220
- return fid
221
  except Exception:
222
- pass
223
- api_key = getattr(client, "api_key", "") or DEFAULT_KEY
224
- url = "https://api.mistral.ai/v1/files"
225
- headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
 
 
 
226
  try:
227
- with open(path, "rb") as fh:
228
- files = {"file": (fname, fh)}
229
- data = {"purpose": purpose}
230
- r = requests.post(url, headers=headers, files=files, data=data, timeout=timeout)
231
- r.raise_for_status()
232
- jr = r.json()
233
- if progress is not None:
234
- progress(0.65, desc="Upload complete (REST)")
235
- return jr.get("id") or jr.get("data", [{}])[0].get("id")
236
- except requests.exceptions.RequestException as e:
237
- raise RuntimeError(f"File upload failed: {e}")
238
 
239
- def determine_media_type(src: str, progress=None) -> Tuple[bool, bool]:
240
- is_image = False
241
- is_video = False
242
- ext = ext_from_src(src)
243
- if ext in IMAGE_EXTS:
244
- is_image = True
245
- if ext in VIDEO_EXTS:
246
- is_video = True
247
- if is_remote(src):
248
- head = safe_head(src)
249
- if head:
250
- ctype = (head.headers.get("content-type") or "").lower()
251
- if ctype.startswith("image/"):
252
- is_image, is_video = True, False
253
- elif ctype.startswith("video/"):
254
- is_video, is_image = True, False
255
- if progress is not None:
256
- progress(0.02, desc="Determined media type")
257
- return is_image, is_video
258
 
259
- def analyze_image_structured(client, img_bytes: bytes, prompt: str, progress=None) -> str:
 
 
 
260
  try:
261
- if progress is not None:
262
- progress(0.3, desc="Preparing image for analysis...")
263
- jpeg = convert_to_jpeg_bytes(img_bytes, base_h=1024)
264
- data_url = b64_bytes(jpeg, mime="image/jpeg")
265
- messages = [
266
- {"role": "system", "content": SYSTEM_INSTRUCTION},
267
- {"role": "user", "content": [
268
- {"type": "text", "text": prompt},
269
- {"type": "image_url", "image_url": data_url},
270
- ]},
271
- ]
272
- return chat_complete(client, PIXTRAL_MODEL, messages, progress=progress)
273
- except UnidentifiedImageError:
274
- return "Error: provided file is not a valid image."
275
- except Exception as e:
276
- return f"Error analyzing image: {e}"
277
 
278
- def analyze_video_cohesive(client, video_path: str, prompt: str, progress=None) -> str:
279
  try:
280
- if progress is not None:
281
- progress(0.3, desc="Uploading video for full analysis...")
282
- file_id = upload_file_to_mistral(client, video_path, filename=os.path.basename(video_path), progress=progress)
283
- extra_msg = (
284
- f"Uploaded video file id: {file_id}\n\n"
285
- "Instruction: Analyze the entire video and produce a single cohesive narrative describing consistent observations."
286
- )
287
- messages = [
288
- {"role": "system", "content": SYSTEM_INSTRUCTION},
289
- {"role": "user", "content": extra_msg + "\n\n" + prompt},
290
- ]
291
- return chat_complete(client, VIDEO_MODEL, messages, progress=progress)
292
- except Exception as e:
293
- if progress is not None:
294
- progress(0.35, desc="Upload failed, extracting frames as fallback...")
295
- frames = extract_best_frames_bytes(video_path, sample_count=6, progress=progress)
296
- if not frames:
297
- return f"Error: could not upload video and no frames could be extracted. ({e})"
298
- image_entries = []
299
- for i, fb in enumerate(frames, start=1):
 
300
  try:
301
- if progress is not None:
302
- progress(0.4 + (i / len(frames)) * 0.2, desc=f"Preparing frame {i}/{len(frames)}...")
303
- j = convert_to_jpeg_bytes(fb, base_h=720)
304
- image_entries.append(
305
- {
306
- "type": "image_url",
307
- "image_url": b64_bytes(j, mime="image/jpeg"),
308
- "meta": {"frame_index": i},
309
- }
310
- )
311
  except Exception:
312
- continue
313
- content = [{"type": "text", "text": prompt + "\n\nPlease consolidate observations across these frames into a single cohesive narrative."}] + image_entries
314
- messages = [
315
- {"role": "system", "content": SYSTEM_INSTRUCTION},
316
- {"role": "user", "content": content},
317
- ]
318
- return chat_complete(client, PIXTRAL_MODEL, messages, progress=progress)
319
-
320
- def process_media(src: str, custom_prompt: str, api_key: str, progress=None) -> str:
321
- client = get_client(api_key)
322
- prompt = (custom_prompt or "").strip() or "Please provide a detailed visual review."
323
-
324
- if not src:
325
- return "Error: No URL or path provided."
326
-
327
- if progress is not None:
328
- progress(0.01, desc="Starting media processing")
329
-
330
- try:
331
- is_image, is_video = determine_media_type(src, progress=progress)
332
- except Exception as e:
333
- return f"Error determining media type: {e}"
334
-
335
- if is_image:
336
- try:
337
- if progress is not None:
338
- progress(0.05, desc="Fetching image bytes...")
339
- raw = fetch_bytes(src, progress=progress)
340
- except FileNotFoundError as e:
341
- return f"Error: {e}"
342
- except Exception as e:
343
- return f"Error fetching image: {e}"
344
 
345
- if progress is not None:
346
- progress(0.2, desc="Analyzing image")
347
- try:
348
- return analyze_image_structured(client, raw, prompt, progress=progress)
349
- except UnidentifiedImageError:
350
- return "Error: provided file is not a valid image."
351
- except Exception as e:
352
- return f"Error analyzing image: {e}"
353
-
354
- if is_video:
355
- try:
356
- if progress is not None:
357
- progress(0.05, desc="Fetching video bytes...")
358
- raw = fetch_bytes(src, timeout=120, progress=progress)
359
- except FileNotFoundError as e:
360
- return f"Error: {e}"
361
- except Exception as e:
362
- return f"Error fetching video: {e}"
363
-
364
- tmp_path = save_bytes_to_temp(raw, suffix=ext_from_src(src) or ".mp4")
365
-
366
- try:
367
- if progress is not None:
368
- progress(0.2, desc="Analyzing video")
369
- return analyze_video_cohesive(client, tmp_path, prompt, progress=progress)
370
- finally:
371
- try:
372
- os.remove(tmp_path)
373
- except Exception:
374
- pass
375
-
376
- try:
377
- if progress is not None:
378
- progress(0.05, desc="Treating input as image fallback...")
379
- raw = fetch_bytes(src, progress=progress)
380
- if progress is not None:
381
- progress(0.2, desc="Analyzing fallback image")
382
- return analyze_image_structured(client, raw, prompt, progress=progress)
383
  except Exception as e:
384
- return f"Unable to determine media type or fetch file: {e}"
385
-
386
- def _ensure_event_loop_for_thread():
387
- """
388
- Ensure the current thread has an asyncio event loop. Used when running blocking
389
- functions in a worker thread that need to run coroutines or use asyncio.get_event_loop().
390
- """
391
- try:
392
- asyncio.get_event_loop()
393
- except RuntimeError:
394
- loop = asyncio.new_event_loop()
395
- asyncio.set_event_loop(loop)
396
-
397
- def run_blocking_in_thread(fn, *args, **kwargs):
398
- """
399
- Run a blocking function in a thread but ensure the thread has an event loop.
400
- Returns concurrent.futures.Future; caller may call .result().
401
- """
402
- def target():
403
- _ensure_event_loop_for_thread()
404
- return fn(*args, **kwargs)
405
- import concurrent.futures
406
- executor = concurrent.futures.ThreadPoolExecutor(max_workers=8)
407
- return executor.submit(target)
408
-
409
- css = ".preview_media img, .preview_media video { max-width: 100%; height: auto; border-radius:6px; }"
410
-
411
- def _btn_label_for_status(status: str) -> str:
412
- return {"idle": "Submit", "busy": "Processing…", "done": "Submit", "error": "Retry"}.get(status, "Submit")
413
 
414
  def create_demo():
415
- with gr.Blocks(title="Flux Multimodal", css=css) as demo:
416
  with gr.Row():
417
  with gr.Column(scale=1):
418
  preview_image = gr.Image(label="Preview Image", type="filepath", elem_classes="preview_media", visible=False)
@@ -430,293 +215,39 @@ def create_demo():
430
  progress_md = gr.Markdown("Idle")
431
  output_md = gr.Markdown("")
432
  status_state = gr.State("idle")
433
- # hidden state to pass preview path from worker to frontend
434
  preview_path_state = gr.State("")
435
 
436
- # small helper: fetch URL into bytes with retries and respect Retry-After
437
- def _fetch_with_retries_bytes(src: str, timeout: int = 15, max_retries: int = 3):
438
- attempt = 0
439
- delay = 1.0
440
- while True:
441
- attempt += 1
442
- try:
443
- if is_remote(src):
444
- r = requests.get(src, timeout=timeout, stream=True)
445
- if r.status_code == 200:
446
- return r.content
447
- if r.status_code == 429:
448
- ra = r.headers.get("Retry-After")
449
- try:
450
- delay = float(ra) if ra is not None else delay
451
- except Exception:
452
- pass
453
- r.raise_for_status()
454
- else:
455
- with open(src, "rb") as fh:
456
- return fh.read()
457
- except requests.exceptions.RequestException:
458
- if attempt >= max_retries:
459
- raise
460
- time.sleep(delay)
461
- delay *= 2
462
- except FileNotFoundError:
463
- raise
464
- except Exception:
465
- if attempt >= max_retries:
466
- raise
467
- time.sleep(delay)
468
- delay *= 2
469
-
470
- # create a local temp file for a remote URL and return local path (or None)
471
- def _save_preview_local(src: str) -> Optional[str]:
472
- if not src:
473
- return None
474
- if not is_remote(src):
475
- return src if os.path.exists(src) else None
476
- try:
477
- b = _fetch_with_retries_bytes(src, timeout=15, max_retries=3)
478
- ext = ext_from_src(src) or ".bin"
479
- fd, tmp = tempfile.mkstemp(suffix=ext)
480
- os.close(fd)
481
- with open(tmp, "wb") as fh:
482
- fh.write(b)
483
- return tmp
484
- except Exception:
485
- return None
486
-
487
- def load_preview(url: str):
488
- # returns (preview_image_path, preview_video_path, status_msg)
489
- if not url:
490
- return gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value="")
491
- try:
492
- if is_remote(url):
493
- head = safe_head(url)
494
- if head:
495
- ctype = (head.headers.get("content-type") or "").lower()
496
- if ctype.startswith("video/") or any(url.lower().endswith(ext) for ext in VIDEO_EXTS):
497
- local = _save_preview_local(url)
498
- if local:
499
- return gr.update(value=None, visible=False), gr.update(value=local, visible=True), gr.update(value=f"Remote video detected (content-type={ctype}). Showing preview if browser-playable.")
500
- else:
501
- return gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value=f"Remote video detected but preview download failed (content-type={ctype}).")
502
- local = _save_preview_local(url)
503
- if not local:
504
- return gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value="Preview load failed: could not fetch resource.")
505
- try:
506
- img = Image.open(local)
507
- if getattr(img, "is_animated", False):
508
- img.seek(0)
509
- return gr.update(value=local, visible=True), gr.update(value=None, visible=False), gr.update(value="Image preview loaded.")
510
- except UnidentifiedImageError:
511
- if any(local.lower().endswith(ext) for ext in VIDEO_EXTS) or True:
512
- return gr.update(value=None, visible=False), gr.update(value=local, visible=True), gr.update(value="Non-image file — showing as video preview if playable.")
513
- return gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value="Preview load failed: file is not a valid image.")
514
- except Exception as e:
515
- return gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value=f"Preview load failed: {e}")
516
-
517
  url_input.change(fn=load_preview, inputs=[url_input], outputs=[preview_image, preview_video, preview_status])
518
 
519
  def clear_all():
520
  return "", None, None, "idle", "Idle", "", ""
521
-
522
  clear_btn.click(fn=clear_all, inputs=[], outputs=[url_input, preview_image, preview_video, status_state, progress_md, output_md, preview_path_state])
523
 
524
- def _convert_video_for_preview(path: str) -> str:
525
- if not FFMPEG_BIN or not os.path.exists(FFMPEG_BIN):
526
- return path
527
- out_fd, out_path = tempfile.mkstemp(suffix=".mp4")
528
- os.close(out_fd)
529
- cmd = [
530
- FFMPEG_BIN, "-nostdin", "-y", "-i", path,
531
- "-c:v", "libx264", "-preset", "veryfast", "-crf", "28",
532
- "-c:a", "aac", "-movflags", "+faststart", out_path
533
- ]
534
- try:
535
- subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60)
536
- return out_path
537
- except Exception:
538
- try: os.remove(out_path)
539
- except Exception: pass
540
- return path
541
-
542
- # --- Helper: probe codecs via ffprobe; returns dict with streams info or None on failure
543
- def _ffprobe_streams(path: str) -> Optional[dict]:
544
- if not FFMPEG_BIN:
545
- return None
546
- ffprobe = FFMPEG_BIN.replace("ffmpeg", "ffprobe") if "ffmpeg" in FFMPEG_BIN else "ffprobe"
547
- if not shutil.which(ffprobe):
548
- ffprobe = "ffprobe"
549
- cmd = [
550
- ffprobe, "-v", "error", "-print_format", "json", "-show_streams", "-show_format", path
551
- ]
552
- try:
553
- out = subprocess.check_output(cmd, stderr=subprocess.DEVNULL)
554
- return json.loads(out)
555
- except Exception:
556
- return None
557
-
558
- # --- Helper: is file already browser-playable (mp4 container with h264 video and aac audio OR at least playable video)
559
- def _is_browser_playable(path: str) -> bool:
560
- try:
561
- ext = (path or "").lower().split("?")[0]
562
- if any(ext.endswith(e) for e in [".mp4", ".m4v", ".mov"]):
563
- info = _ffprobe_streams(path)
564
- if not info:
565
- # fallback: trust .mp4 if probe failed
566
- return ext.endswith(".mp4")
567
- streams = info.get("streams", [])
568
- v_ok = any(
569
- s.get("codec_name") in ("h264", "h265", "avc1") and s.get("codec_type") == "video"
570
- for s in streams
571
- )
572
- # audio optional for preview
573
- return bool(v_ok)
574
- # other extensions: probe and accept if any video stream present
575
- info = _ffprobe_streams(path)
576
- if not info:
577
- return False
578
- streams = info.get("streams", [])
579
- return any(s.get("codec_type") == "video" for s in streams)
580
- except Exception:
581
- return False
582
-
583
- # --- Convert only if not browser-playable
584
- def _convert_video_for_preview_if_needed(path: str) -> str:
585
- try:
586
- if _is_browser_playable(path):
587
- return path
588
- except Exception:
589
- pass
590
- return _convert_video_for_preview(path)
591
-
592
- # Worker now returns (status_state, output_md, preview_path_state)
593
- def worker(url: str, prompt: str, key: str, progress=gr.Progress()):
594
- try:
595
- if not url:
596
- return ("error", "**Error:** No URL provided.", "")
597
- progress(0.01, desc="Starting processing...")
598
- progress(0.03, desc="Checking URL / content-type...")
599
- is_img, is_vid = determine_media_type(url, progress=progress)
600
- progress(0.06, desc=f"Determined media type: image={is_img}, video={is_vid}")
601
- client = get_client(key)
602
- preview_local = None
603
- if is_vid:
604
- progress(0.08, desc="Fetching video bytes (may take a while)...")
605
- raw = fetch_bytes(url, timeout=120, progress=progress)
606
- tmp = save_bytes_to_temp(raw, suffix=ext_from_src(url) or ".mp4")
607
- progress(0.18, desc="Saved video to temp; converting for preview if needed...")
608
- preview_tmp = _convert_video_for_preview(tmp)
609
- preview_local = preview_tmp if os.path.exists(preview_tmp) else tmp
610
- progress(0.25, desc="Starting video analysis...")
611
- res = analyze_video_cohesive(client, tmp, prompt or "", progress=progress)
612
- progress(0.98, desc="Finalizing result...")
613
- try:
614
- if preview_tmp != tmp and os.path.exists(preview_tmp):
615
- pass
616
- finally:
617
- try: os.remove(tmp)
618
- except Exception: pass
619
- status = "done" if not (isinstance(res, str) and res.lower().startswith("error")) else "error"
620
- return (status, res if isinstance(res, str) else str(res), preview_local or "")
621
- elif is_img:
622
- progress(0.08, desc="Fetching image bytes...")
623
- raw = fetch_bytes(url, progress=progress)
624
- try:
625
- preview_fd, preview_path = tempfile.mkstemp(suffix=".jpg")
626
- os.close(preview_fd)
627
- with open(preview_path, "wb") as fh:
628
- fh.write(convert_to_jpeg_bytes(raw, base_h=1024))
629
- preview_local = preview_path
630
- except Exception:
631
- preview_local = None
632
- progress(0.18, desc="Analyzing image...")
633
- try:
634
- res = analyze_image_structured(client, raw, prompt or "", progress=progress)
635
- except UnidentifiedImageError:
636
- return ("error", "Error: provided file is not a valid image.", preview_local or "")
637
- progress(0.98, desc="Finalizing result...")
638
- status = "done" if not (isinstance(res, str) and res.lower().startswith("error")) else "error"
639
- return (status, res if isinstance(res, str) else str(res), preview_local or "")
640
- else:
641
- progress(0.07, desc="Unknown media type — fetching bytes for heuristics...")
642
- raw = fetch_bytes(url, timeout=120, progress=progress)
643
- try:
644
- progress(0.15, desc="Attempting to interpret as image...")
645
- Image.open(BytesIO(raw))
646
- progress(0.2, desc="Image detected — analyzing...")
647
- res = analyze_image_structured(client, raw, prompt or "", progress=progress)
648
- status = "done" if not (isinstance(res, str) and res.lower().startswith("error")) else "error"
649
- try:
650
- preview_fd, preview_path = tempfile.mkstemp(suffix=".jpg")
651
- os.close(preview_fd)
652
- with open(preview_path, "wb") as fh:
653
- fh.write(convert_to_jpeg_bytes(raw, base_h=1024))
654
- preview_local = preview_path
655
- except Exception:
656
- preview_local = None
657
- return (status, res if isinstance(res, str) else str(res), preview_local or "")
658
- except Exception:
659
- fd, tmp = tempfile.mkstemp(suffix=ext_from_src(url) or ".mp4")
660
- os.close(fd)
661
- with open(tmp, "wb") as fh:
662
- fh.write(raw)
663
- try:
664
- progress(0.3, desc="Saved fallback video file; analyzing...")
665
- preview_tmp = _convert_video_for_preview(tmp)
666
- preview_local = preview_tmp if os.path.exists(preview_tmp) else tmp
667
- res = analyze_video_cohesive(client, tmp, prompt or "", progress=progress)
668
- status = "done" if not (isinstance(res, str) and res.lower().startswith("error")) else "error"
669
- return (status, res if isinstance(res, str) else str(res), preview_local or "")
670
- finally:
671
- try: os.remove(tmp)
672
- except Exception: pass
673
- except Exception as e:
674
- return ("error", f"Unexpected worker error: {e}", "")
675
-
676
- # immediate UI flip to "busy" so user sees work started
677
  submit_btn.click(fn=lambda: "busy", inputs=[], outputs=[status_state])
 
678
 
679
- # actual heavy work runs in the queue and shows progress (attach to progress_md)
680
- submit_btn.click(
681
- fn=worker,
682
- inputs=[url_input, custom_prompt, api_key],
683
- outputs=[status_state, output_md, preview_path_state],
684
- queue=True,
685
- show_progress="full",
686
- show_progress_on=progress_md,
687
- )
688
-
689
- # update submit button label from status
690
- def btn_label_from_state(s):
691
- return _btn_label_for_status(s)
692
-
693
  status_state.change(fn=btn_label_from_state, inputs=[status_state], outputs=[submit_btn])
694
 
695
- # map status to progress text
696
- def status_to_progress_text(s):
697
- return {"idle":"Idle","busy":"Processing…","done":"Completed","error":"Error — see output"}.get(s, s)
698
  status_state.change(fn=status_to_progress_text, inputs=[status_state], outputs=[progress_md])
699
 
700
- # when preview_path_state changes, update preview components appropriately
701
- def apply_preview(path: str):
702
- if not path:
703
- return gr.update(value=None, visible=False), gr.update(value=None, visible=False), ""
704
- try:
705
- if any(path.lower().endswith(ext) for ext in IMAGE_EXTS):
706
- return gr.update(value=path, visible=True), gr.update(value=None, visible=False), "Preview updated."
707
- if any(path.lower().endswith(ext) for ext in VIDEO_EXTS):
708
- return gr.update(value=None, visible=False), gr.update(value=path, visible=True), "Preview updated."
709
- try:
710
- Image.open(path)
711
- return gr.update(value=path, visible=True), gr.update(value=None, visible=False), "Preview updated."
712
- except Exception:
713
- return gr.update(value=None, visible=False), gr.update(value=path, visible=True), "Preview updated."
714
- except Exception:
715
- return gr.update(value=None, visible=False), gr.update(value=None, visible=False), ""
716
 
717
  preview_path_state.change(fn=apply_preview, inputs=[preview_path_state], outputs=[preview_image, preview_video, preview_status])
718
 
719
- # ensure global queue behavior
720
  demo.queue()
721
  return demo
722
 
 
7
  import tempfile
8
  import base64
9
  import json
10
+ import requests
11
  from io import BytesIO
12
  from typing import List, Tuple, Optional
 
13
  from PIL import Image, ImageFile, UnidentifiedImageError
14
  import gradio as gr
 
 
 
15
 
16
+ # Constants
17
  DEFAULT_KEY = os.getenv("MISTRAL_API_KEY", "")
18
  PIXTRAL_MODEL = "pixtral-12b-2409"
19
  VIDEO_MODEL = "voxtral-mini-latest"
20
  STREAM_THRESHOLD = 20 * 1024 * 1024
21
  FFMPEG_BIN = shutil.which("ffmpeg")
22
+ IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".webp", ".gif"}
23
+ VIDEO_EXTS = {".mp4", ".mov", ".webm", ".mkv", ".avi", ".flv"}
 
 
 
 
 
 
 
 
 
24
 
25
+ # Initialize ImageFile
26
  ImageFile.LOAD_TRUNCATED_IMAGES = True
27
  Image.MAX_IMAGE_PIXELS = 10000 * 10000
28
 
 
 
 
 
 
29
  def get_client(key: Optional[str] = None):
30
  api_key = (key or "").strip() or DEFAULT_KEY
31
+ try:
32
+ from mistralai import Mistral
33
+ return Mistral(api_key=api_key)
34
+ except ImportError:
35
  class Dummy:
36
  def __init__(self, k): self.api_key = k
37
  return Dummy(api_key)
 
38
 
39
  def is_remote(src: str) -> bool:
40
  return bool(src) and src.startswith(("http://", "https://"))
41
 
42
  def ext_from_src(src: str) -> str:
43
+ if not src:
44
+ return ""
45
+ return os.path.splitext(src.split("?")[0])[1].lower()
46
 
47
  def safe_head(url: str, timeout: int = 6):
48
  try:
 
51
  except Exception:
52
  return None
53
 
54
+ def fetch_bytes(src: str, timeout: int = 60) -> bytes:
 
 
 
 
 
 
 
55
  if is_remote(src):
56
  head = safe_head(src)
57
  if head is not None:
58
+ r = requests.get(src, timeout=timeout)
59
+ r.raise_for_status()
60
+ return r.content
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
61
  else:
62
  if not os.path.exists(src):
63
+ raise FileNotFoundError(f"Path does not exist: {src}")
 
 
64
  with open(src, "rb") as f:
65
+ return f.read()
 
 
 
66
 
67
  def save_bytes_to_temp(b: bytes, suffix: str) -> str:
68
  fd, path = tempfile.mkstemp(suffix=suffix)
 
73
 
74
  def convert_to_jpeg_bytes(img_bytes: bytes, base_h: int = 480) -> bytes:
75
  img = Image.open(BytesIO(img_bytes))
 
 
 
 
 
76
  if img.mode != "RGB":
77
  img = img.convert("RGB")
78
  h = base_h
 
82
  img.save(buf, format="JPEG", quality=85)
83
  return buf.getvalue()
84
 
85
+ def load_preview(url: str):
86
+ if not url:
87
+ return gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value="")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88
  try:
89
+ if is_remote(url):
90
+ head = safe_head(url)
91
+ ctype = (head.headers.get("content-type") or "").lower() if head else ""
92
+ if ctype.startswith("video/") or any(url.lower().endswith(ext) for ext in VIDEO_EXTS):
93
+ local = _save_preview_local(url)
94
+ if local:
95
+ return gr.update(value=None, visible=False), gr.update(value=local, visible=True), gr.update(value="Remote video detected.")
96
+ return gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value="Preview download failed.")
97
+
98
+ local = _save_preview_local(url)
99
+ if not local:
100
+ return gr.update(value=None, visible=False), gr.update(value=None, visible=False), gr.update(value="Preview load failed.")
101
+
102
+ img = Image.open(local)
103
+ if getattr(img, "is_animated", False):
104
+ img.seek(0)
105
+ return gr.update(value=local, visible=True), gr.update(value=None, visible=False), gr.update(value="Image preview loaded.")
106
+ except UnidentifiedImageError:
107
+ return gr.update(value=None, visible=False), gr.update(value=local, visible=True), gr.update(value="Non-image file showing as video if playable.")
 
 
 
108
  except Exception as e:
109
+ return gr.update(value=None, visible=False), gr.update(value=None,Here's the continuation of your Python script:
110
 
111
+ ```python
112
+ visible=False), gr.update(value=f"Preview load failed: {e}")
113
+
114
+ def _save_preview_local(url: str) -> Optional[str]:
115
+ if not url: return None
116
  try:
117
+ b = fetch_bytes(url)
118
+ ext = ext_from_src(url) or ".bin"
119
+ fd, tmp = tempfile.mkstemp(suffix=ext)
120
+ os.close(fd)
121
+ with open(tmp, "wb") as fh:
122
+ fh.write(b)
123
+ return tmp
 
 
 
 
124
  except Exception:
125
+ return None
126
+
127
+ def _convert_video_for_preview(path: str) -> str:
128
+ if not FFMPEG_BIN or not os.path.exists(FFMPEG_BIN): return path
129
+ out_fd, out_path = tempfile.mkstemp(suffix=".mp4")
130
+ os.close(out_fd)
131
+ cmd = [FFMPEG_BIN, "-nostdin", "-y", "-i", path, "-c:v", "libx264", "-preset", "veryfast", "-crf", "28", "-c:a", "aac", "-movflags", "+faststart", out_path]
132
  try:
133
+ subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=60)
134
+ return out_path
135
+ except Exception:
136
+ try: os.remove(out_path)
137
+ except Exception: pass
138
+ return path
 
 
 
 
 
139
 
140
+ def _is_browser_playable(path: str) -> bool:
141
+ try:
142
+ ext = (path or "").lower().split("?")[0]
143
+ if any(ext.endswith(e) for e in [".mp4", ".m4v", ".mov"]):
144
+ info = _ffprobe_streams(path)
145
+ if not info: return ext.endswith(".mp4")
146
+ streams = info.get("streams", [])
147
+ return any(s.get("codec_name") in ("h264", "h265", "avc1") and s.get("codec_type") == "video" for s in streams)
148
+ return False
149
+ except Exception:
150
+ return False
 
 
 
 
 
 
 
 
151
 
152
+ def _ffprobe_streams(path: str) -> Optional[dict]:
153
+ if not FFMPEG_BIN: return None
154
+ ffprobe = FFMPEG_BIN.replace("ffmpeg", "ffprobe") if "ffmpeg" in FFMPEG_BIN else "ffprobe"
155
+ cmd = [ffprobe, "-v", "error", "-print_format", "json", "-show_streams", "-show_format", path]
156
  try:
157
+ out = subprocess.check_output(cmd, stderr=subprocess.DEVNULL)
158
+ return json.loads(out)
159
+ except Exception:
160
+ return None
 
 
 
 
 
 
 
 
 
 
 
 
161
 
162
+ def worker(url: str, prompt: str, key: str, progress=gr.Progress()):
163
  try:
164
+ if not url: return ("error", "**Error:** No URL provided.", "")
165
+ progress(0.01, desc="Starting processing...")
166
+ is_img, is_vid = determine_media_type(url, progress=progress)
167
+ progress(0.06, desc=f"Media type detected: image={is_img}, video={is_vid}")
168
+ client = get_client(key)
169
+ preview_local = None
170
+
171
+ if is_vid:
172
+ progress(0.08, desc="Fetching video bytes...")
173
+ raw = fetch_bytes(url, timeout=120, progress=progress)
174
+ tmp = save_bytes_to_temp(raw, suffix=ext_from_src(url) or ".mp4")
175
+ preview_tmp = _convert_video_for_preview(tmp)
176
+ preview_local = preview_tmp if os.path.exists(preview_tmp) else tmp
177
+ res = analyze_video_cohesive(client, tmp, prompt or "", progress=progress)
178
+ elif is_img:
179
+ progress(0.08, desc="Fetching image bytes...")
180
+ raw = fetch_bytes(url, progress=progress)
181
+ preview_local = save_bytes_to_temp(convert_to_jpeg_bytes(raw), suffix=".jpg")
182
+ res = analyze_image_structured(client, raw, prompt or "", progress=progress)
183
+ else:
184
+ raw = fetch_bytes(url, timeout=120, progress=progress)
185
  try:
186
+ Image.open(BytesIO(raw))
187
+ res = analyze_image_structured(client, raw, prompt or "", progress=progress)
188
+ preview_local = save_bytes_to_temp(convert_to_jpeg_bytes(raw), suffix=".jpg")
 
 
 
 
 
 
 
189
  except Exception:
190
+ tmp = save_bytes_to_temp(raw, suffix=ext_from_src(url) or ".mp4")
191
+ preview_local = _convert_video_for_preview(tmp)
192
+ res = analyze_video_cohesive(client, tmp, prompt or "", progress=progress)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
193
 
194
+ status = "done" if not (isinstance(res, str) and res.lower().startswith("error")) else "error"
195
+ return (status, res if isinstance(res, str) else str(res), preview_local or "")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
196
  except Exception as e:
197
+ return ("error", f"Unexpected worker error: {e}", "")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
198
 
199
  def create_demo():
200
+ with gr.Blocks(title="Flux Multimodal") as demo:
201
  with gr.Row():
202
  with gr.Column(scale=1):
203
  preview_image = gr.Image(label="Preview Image", type="filepath", elem_classes="preview_media", visible=False)
 
215
  progress_md = gr.Markdown("Idle")
216
  output_md = gr.Markdown("")
217
  status_state = gr.State("idle")
 
218
  preview_path_state = gr.State("")
219
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
220
  url_input.change(fn=load_preview, inputs=[url_input], outputs=[preview_image, preview_video, preview_status])
221
 
222
  def clear_all():
223
  return "", None, None, "idle", "Idle", "", ""
 
224
  clear_btn.click(fn=clear_all, inputs=[], outputs=[url_input, preview_image, preview_video, status_state, progress_md, output_md, preview_path_state])
225
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
226
  submit_btn.click(fn=lambda: "busy", inputs=[], outputs=[status_state])
227
+ submit_btn.click(fn=worker, inputs=[url_input, custom_prompt, api_key], outputs=[status_state, output_md, preview_path_state], queue=True, show_progress="full", show_progress_on=progress_md)
228
 
229
+ def btn_label_from_state(s): return _btn_label_for_status(s)
 
 
 
 
 
 
 
 
 
 
 
 
 
230
  status_state.change(fn=btn_label_from_state, inputs=[status_state], outputs=[submit_btn])
231
 
232
+ def status_to_progress_text(s): return {"idle": "Idle", "busy": "Processing…", "done": "Completed", "error": "Error — see output"}.get(s, s)
 
 
233
  status_state.change(fn=status_to_progress_text, inputs=[status_state], outputs=[progress_md])
234
 
235
+ def apply_preview(path: str):
236
+ if not path:
237
+ return gr.update(value=None, visible=False), gr.update(value=None, visible=False), ""
238
+ try:
239
+ if any(path.lower().endswith(ext) for ext in IMAGE_EXTS):
240
+ return gr.update(value=path, visible=True), gr.update(value=None, visible=False), "Image preview updated."
241
+ if any(path.lower().endswith(ext) for ext in VIDEO_EXTS):
242
+ return gr.update(value=None, visible=False), gr.update(value=path, visible=True), "Video preview updated."
243
+ # Handle failure to load as image or video
244
+ Image.open(path)
245
+ return gr.update(value=path, visible=True), gr.update(value=None, visible=False), "Preview updated."
246
+ except Exception:
247
+ return gr.update(value=None, visible=False), gr.update(value=path, visible=True), "Preview updated."
 
 
 
248
 
249
  preview_path_state.change(fn=apply_preview, inputs=[preview_path_state], outputs=[preview_image, preview_video, preview_status])
250
 
 
251
  demo.queue()
252
  return demo
253