| """ |
| RoboMind VLA — Task 9: app.py |
| |
| FastAPI web UI for the RoboMind VLA reward judge. |
| Runs on Modal GPU with a public URL. |
| |
| Usage: |
| modal deploy app.py |
| |
| Then visit the public URL printed in the output. |
| """ |
|
|
| import json |
| import os |
| import tempfile |
| from typing import List |
|
|
| import modal |
|
|
| app_image = ( |
| modal.Image.debian_slim(python_version="3.11") |
| .apt_install("ffmpeg") |
| .pip_install( |
| "torch==2.4.0", |
| "torchvision==0.19.0", |
| "transformers==4.40.0", |
| "peft==0.11.1", |
| "accelerate==0.30.1", |
| "pillow", |
| "sentencepiece", |
| "huggingface_hub", |
| "fastapi==0.115.6", |
| "uvicorn", |
| "pydantic<2.13", |
| "numpy<2", |
| "opencv-python-headless", |
| "python-multipart", |
| "librosa==0.10.2", |
| "soundfile", |
| ) |
| .run_commands( |
| "python -c \"" |
| "import os, sys; " |
| "d = os.path.join(sys.prefix, 'lib/python3.11/site-packages/flash_attn'); " |
| "os.makedirs(d, exist_ok=True); " |
| "open(os.path.join(d, '__init__.py'), 'w').write(''); " |
| "open(os.path.join(d, 'flash_attn_interface.py'), 'w').write(" |
| "'def flash_attn_func(*a, **kw): raise NotImplementedError\\n" |
| "def flash_attn_varlen_func(*a, **kw): raise NotImplementedError\\n'); " |
| "print('flash_attn stub created')\"" |
| ) |
| .add_local_file("hybrid_judge.py", "/root/hybrid_judge.py") |
| ) |
|
|
| app = modal.App("robomind-gradio", image=app_image) |
| volume = modal.Volume.from_name("robomind-data", create_if_missing=True) |
| ADAPTER_REPO = "mitvho09/robomind-minicpm-loco-lora" |
|
|
| INSTRUCTION_PROMPT = ( |
| "You are RoboMind VLA, a vision-language reward model for robot locomotion. " |
| "You are shown keyframes from a MuJoCo locomotion rollout. " |
| "The robot was commanded to \"walk forward\". Analyze the rollout and " |
| "respond with ONLY a JSON object with these exact keys: timestep_range, " |
| "phase, command, command_followed, stability, fall_risk, gait_quality, " |
| "predicted_reward, anomaly, explanation." |
| ) |
|
|
| METADATA_PATH = "/data/rollouts/metadata.jsonl" |
|
|
| HTML_PAGE = """<!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| <title>RoboMind VLA — Locomotion Reward Judge</title> |
| <style> |
| *{box-sizing:border-box;margin:0;padding:0} |
| body{font-family:system-ui,-apple-system,sans-serif;background:#0f172a;color:#e2e8f0;min-height:100vh} |
| .container{max-width:960px;margin:0 auto;padding:2rem 1rem} |
| h1{font-size:2rem;margin-bottom:.5rem;color:#38bdf8} |
| .subtitle{color:#94a3b8;margin-bottom:2rem} |
| .tabs{display:flex;gap:.5rem;margin-bottom:1.5rem} |
| .tab{padding:.6rem 1.2rem;background:#1e293b;border:1px solid #334155;border-radius:8px;cursor:pointer;color:#94a3b8;transition:.2s} |
| .tab.active{background:#334155;color:#38bdf8;border-color:#38bdf8} |
| .tab-content{display:none} |
| .tab-content.active{display:block} |
| .upload-area{border:2px dashed #334155;border-radius:12px;padding:2rem;text-align:center;transition:.2s;cursor:pointer} |
| .upload-area:hover{border-color:#38bdf8} |
| .upload-area input[type=file]{display:none} |
| .upload-area label{cursor:pointer;color:#94a3b8;font-size:1.1rem} |
| .btn{background:#0ea5e9;color:white;border:none;padding:.8rem 1.5rem;border-radius:8px;font-size:1rem;cursor:pointer;margin-top:1rem;transition:.2s} |
| .btn:hover{background:#0284c7} |
| .btn:disabled{background:#334155;color:#64748b;cursor:not-allowed} |
| .result{margin-top:1.5rem;background:#1e293b;border:1px solid #334155;border-radius:12px;padding:1.5rem;white-space:pre-wrap;font-family:'Fira Code',monospace;font-size:.9rem;overflow-x:auto;max-height:500px;overflow-y:auto} |
| .raw{margin-top:1rem;color:#94a3b8;font-size:.85rem;white-space:pre-wrap} |
| .hybrid{margin-top:1rem;background:#0c1929;border:1px solid #0ea5e9;border-radius:12px;padding:1.2rem} |
| .hybrid h3{color:#38bdf8;margin-bottom:.5rem;font-size:1rem} |
| .hybrid-grid{display:grid;grid-template-columns:1fr 1fr 1fr;gap:.5rem} |
| .hybrid-item{background:#1e293b;padding:.5rem;border-radius:6px} |
| .hybrid-item .label{color:#64748b;font-size:.75rem;text-transform:uppercase} |
| .hybrid-item .value{color:#e2e8f0;font-size:1.1rem;font-weight:bold} |
| .hybrid-item .value.good{color:#22c55e} |
| .hybrid-item .value.warn{color:#f59e0b} |
| .hybrid-item .value.bad{color:#ef4444} |
| .url-input{width:100%;padding:.8rem;background:#1e293b;border:1px solid #334155;border-radius:8px;color:#e2e8f0;font-size:1rem;margin-top:.5rem} |
| .spinner{display:inline-block;width:20px;height:20px;border:3px solid #334155;border-top-color:#38bdf8;border-radius:50%;animation:spin .6s linear infinite;margin-right:.5rem;vertical-align:middle} |
| @keyframes spin{to{transform:rotate(360deg)}} |
| .preview{display:flex;gap:.5rem;flex-wrap:wrap;margin-top:1rem} |
| .preview img{max-width:120px;max-height:120px;border-radius:8px;border:1px solid #334155} |
| </style> |
| </head> |
| <body> |
| <div class="container"> |
| <h1>RoboMind VLA</h1> |
| <p class="subtitle">Humanoid Locomotion Reward Judge</p> |
| |
| <div class="tabs"> |
| <div class="tab active" onclick="switchTab(0)">Image Upload</div> |
| <div class="tab" onclick="switchTab(1)">Video Upload</div> |
| <div class="tab" onclick="switchTab(2)">URL</div> |
| </div> |
| |
| <div class="tab-content active" id="tab0"> |
| <div class="upload-area" id="img-drop"> |
| <label>Click or drag up to 6 keyframe images here</label><br> |
| <input type="file" id="img-input" accept="image/*" multiple> |
| </div> |
| <div class="preview" id="img-preview"></div> |
| <button class="btn" id="img-btn" onclick="judgeImages()" disabled>Judge Rollout</button> |
| <div class="hybrid" id="img-hybrid" style="display:none"></div> |
| <div class="result" id="img-result" style="display:none"></div> |
| <div class="raw" id="img-raw" style="display:none"></div> |
| </div> |
| |
| <div class="tab-content" id="tab1"> |
| <div class="upload-area" id="vid-drop"> |
| <label>Click or drag a rollout video here</label><br> |
| <input type="file" id="vid-input" accept="video/*"> |
| </div> |
| <p id="vid-name" style="margin-top:.5rem;color:#94a3b8"></p> |
| <button class="btn" id="vid-btn" onclick="judgeVideo()" disabled>Judge Rollout</button> |
| <div class="hybrid" id="vid-hybrid" style="display:none"></div> |
| <div class="result" id="vid-result" style="display:none"></div> |
| <div class="raw" id="vid-raw" style="display:none"></div> |
| </div> |
| |
| <div class="tab-content" id="tab2"> |
| <input class="url-input" id="url-input" placeholder="https://example.com/rollout.mp4"> |
| <button class="btn" id="url-btn" onclick="judgeUrl()">Judge Rollout</button> |
| <div class="hybrid" id="url-hybrid" style="display:none"></div> |
| <div class="result" id="url-result" style="display:none"></div> |
| <div class="raw" id="url-raw" style="display:none"></div> |
| </div> |
| </div> |
| |
| <script> |
| function switchTab(i){ |
| document.querySelectorAll('.tab').forEach((t,j)=>t.classList.toggle('active',j===i)); |
| document.querySelectorAll('.tab-content').forEach((t,j)=>t.classList.toggle('active',j===i)); |
| } |
| const imgInput=document.getElementById('img-input'); |
| const imgDrop=document.getElementById('img-drop'); |
| const imgPreview=document.getElementById('img-preview'); |
| imgDrop.onclick=()=>imgInput.click(); |
| imgInput.onchange=()=>{ |
| const files=[...imgInput.files]; |
| imgPreview.innerHTML=''; |
| files.forEach(f=>{ |
| const img=document.createElement('img'); |
| img.src=URL.createObjectURL(f); |
| imgPreview.appendChild(img); |
| }); |
| document.getElementById('img-btn').disabled=files.length===0; |
| }; |
| imgDrop.ondragover=e=>{e.preventDefault();imgDrop.style.borderColor='#38bdf8'}; |
| imgDrop.ondragleave=()=>{imgDrop.style.borderColor='#334155'}; |
| imgDrop.ondrop=e=>{ |
| e.preventDefault();imgDrop.style.borderColor='#334155'; |
| imgInput.files=e.dataTransfer.files;imgInput.onchange(); |
| }; |
| |
| const vidInput=document.getElementById('vid-input'); |
| const vidDrop=document.getElementById('vid-drop'); |
| vidDrop.onclick=()=>vidInput.click(); |
| vidInput.onchange=()=>{ |
| document.getElementById('vid-name').textContent=vidInput.files[0]?.name||''; |
| document.getElementById('vid-btn').disabled=!vidInput.files.length; |
| }; |
| vidDrop.ondragover=e=>{e.preventDefault();vidDrop.style.borderColor='#38bdf8'}; |
| vidDrop.ondragleave=()=>{vidDrop.style.borderColor='#334155'}; |
| vidDrop.ondrop=e=>{ |
| e.preventDefault();vidDrop.style.borderColor='#334155'; |
| vidInput.files=e.dataTransfer.files;vidInput.onchange(); |
| }; |
| |
| function showHybrid(containerId, hybrid) { |
| const el = document.getElementById(containerId); |
| if (!hybrid) { el.style.display='none'; return; } |
| const reward = hybrid.predicted_reward; |
| const cls = reward >= 0.7 ? 'good' : reward >= 0.3 ? 'warn' : 'bad'; |
| el.innerHTML = ` |
| <h3>Hybrid Score (VLM + Physics)</h3> |
| <div class="hybrid-grid"> |
| <div class="hybrid-item"><div class="label">Final Reward</div><div class="value ${cls}">${reward.toFixed(3)}</div></div> |
| <div class="hybrid-item"><div class="label">VLM Reward</div><div class="value">${hybrid.vlm_reward.toFixed(3)}</div></div> |
| <div class="hybrid-item"><div class="label">Rule Reward</div><div class="value">${hybrid.rule_reward.toFixed(3)}</div></div> |
| <div class="hybrid-item"><div class="label">Confidence</div><div class="value">${hybrid.confidence.toFixed(2)}</div></div> |
| <div class="hybrid-item"><div class="label">Stability</div><div class="value">${hybrid.stability}</div></div> |
| <div class="hybrid-item"><div class="label">Gait Quality</div><div class="value">${hybrid.gait_quality.toFixed(3)}</div></div> |
| </div> |
| ${hybrid.anomaly ? '<p style="color:#f59e0b;margin-top:.5rem">Anomaly: '+hybrid.anomaly+'</p>' : ''} |
| ${hybrid.sound_analysis && !hybrid.sound_analysis.error ? '<div style="margin-top:.8rem;padding:.5rem;background:#1e293b;border-radius:8px"><strong style="color:#38bdf8">Audio Analysis</strong><br>'+ |
| 'Fall detected: '+(hybrid.sound_analysis.has_fall ? '<span style="color:#ef4444">YES</span>' : 'No')+'<br>'+ |
| 'Impacts: '+hybrid.sound_analysis.impact_count+'<br>'+ |
| 'Gait rhythm: '+(hybrid.sound_analysis.has_rhythmic_gait ? '<span style="color:#22c55e">Regular</span>' : 'Irregular')+'<br>'+ |
| 'Motor strain: '+(hybrid.sound_analysis.has_motor_strain ? '<span style="color:#f59e0b">High</span>' : 'Normal')+ |
| '</div>' : ''} |
| `; |
| el.style.display='block'; |
| } |
| |
| async function judgeImages(){ |
| const btn=document.getElementById('img-btn'); |
| const res=document.getElementById('img-result'); |
| const raw=document.getElementById('img-raw'); |
| btn.disabled=true;btn.innerHTML='<span class="spinner"></span>Analyzing...'; |
| res.style.display='none';raw.style.display='none'; |
| const fd=new FormData(); |
| [...imgInput.files].forEach(f=>fd.append('files',f)); |
| try{ |
| const r=await fetch('/judge/images',{method:'POST',body:fd}); |
| const d=await r.json(); |
| res.textContent=JSON.stringify(d.parsed,null,2);res.style.display='block'; |
| showHybrid('img-hybrid', d.hybrid); |
| raw.textContent=d.raw;raw.style.display='block'; |
| }catch(e){res.textContent='Error: '+e.message;res.style.display='block';} |
| btn.disabled=false;btn.textContent='Judge Rollout'; |
| } |
| |
| async function judgeVideo(){ |
| const btn=document.getElementById('vid-btn'); |
| const res=document.getElementById('vid-result'); |
| const raw=document.getElementById('vid-raw'); |
| btn.disabled=true;btn.innerHTML='<span class="spinner"></span>Analyzing...'; |
| res.style.display='none';raw.style.display='none'; |
| const fd=new FormData(); |
| fd.append('file',vidInput.files[0]); |
| try{ |
| const r=await fetch('/judge/video',{method:'POST',body:fd}); |
| const d=await r.json(); |
| res.textContent=JSON.stringify(d.parsed,null,2);res.style.display='block'; |
| showHybrid('vid-hybrid', d.hybrid); |
| raw.textContent=d.raw;raw.style.display='block'; |
| }catch(e){res.textContent='Error: '+e.message;res.style.display='block';} |
| btn.disabled=false;btn.textContent='Judge Rollout'; |
| } |
| |
| async function judgeUrl(){ |
| const btn=document.getElementById('url-btn'); |
| const res=document.getElementById('url-result'); |
| const raw=document.getElementById('url-raw'); |
| btn.disabled=true;btn.innerHTML='<span class="spinner"></span>Analyzing...'; |
| res.style.display='none';raw.style.display='none'; |
| const url=document.getElementById('url-input').value; |
| try{ |
| const r=await fetch('/judge/url',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({url})}); |
| const d=await r.json(); |
| res.textContent=JSON.stringify(d.parsed,null,2);res.style.display='block'; |
| showHybrid('url-hybrid', d.hybrid); |
| raw.textContent=d.raw;raw.style.display='block'; |
| }catch(e){res.textContent='Error: '+e.message;res.style.display='block';} |
| btn.disabled=false;btn.textContent='Judge Rollout'; |
| } |
| </script> |
| </body> |
| </html>""" |
|
|
|
|
| def _extract_keyframes(video_path: str, n_frames: int = 6): |
| import cv2 |
| from PIL import Image |
|
|
| cap = cv2.VideoCapture(video_path) |
| total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) |
| if total <= 0: |
| cap.release() |
| return [] |
| indices = [int(i * total / n_frames) for i in range(n_frames)] |
| frames = [] |
| for idx in indices: |
| cap.set(cv2.CAP_PROP_POS_FRAMES, idx) |
| ret, frame = cap.read() |
| if ret: |
| frames.append(Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))) |
| cap.release() |
| return frames |
|
|
|
|
| _model_cache = {} |
|
|
|
|
| def _get_model(): |
| if "model" in _model_cache: |
| return _model_cache["model"], _model_cache["tokenizer"] |
|
|
| import torch |
| from transformers import AutoModel, AutoTokenizer |
| from peft import PeftModel |
| from huggingface_hub import login |
|
|
| hf_token = os.environ.get("HF_TOKEN") |
| if hf_token: |
| login(token=hf_token) |
|
|
| print("[robomind] loading model...") |
| tokenizer = AutoTokenizer.from_pretrained( |
| "openbmb/MiniCPM-V-2_6", trust_remote_code=True |
| ) |
| base_model = AutoModel.from_pretrained( |
| "openbmb/MiniCPM-V-2_6", |
| trust_remote_code=True, |
| torch_dtype=torch.bfloat16, |
| device_map="auto", |
| ) |
| model = PeftModel.from_pretrained(base_model, ADAPTER_REPO) |
| model.eval() |
|
|
| _model_cache["model"] = model |
| _model_cache["tokenizer"] = tokenizer |
| print("[robomind] model loaded") |
| return model, tokenizer |
|
|
|
|
| def _judge_images(model, tokenizer, images, n_images: int = 6): |
| n = min(len(images), n_images) |
| image_tokens = "\\n".join(f"<image_{k:02d}>" for k in range(n)) |
| user_content = f"{image_tokens}\\n{INSTRUCTION_PROMPT}" |
|
|
| response = model.chat( |
| image=images[:n], |
| msgs=[{"role": "user", "content": user_content}], |
| tokenizer=tokenizer, |
| max_new_tokens=512, |
| ) |
| return response if isinstance(response, str) else str(response) |
|
|
|
|
| def _parse_response(response: str) -> dict: |
| import re |
| response = response.strip() |
|
|
| |
| json_blocks = list(re.finditer(r'\{[^{}]*\}', response, re.DOTALL)) |
| if json_blocks: |
| for block in reversed(json_blocks): |
| try: |
| return json.loads(block.group()) |
| except json.JSONDecodeError: |
| continue |
|
|
| |
| try: |
| return json.loads(response) |
| except json.JSONDecodeError: |
| pass |
|
|
| return {"raw_response": response} |
|
|
|
|
| def _load_metadata(): |
| """Load rollout metadata for hybrid scoring.""" |
| import csv |
| meta = {} |
| if os.path.exists(METADATA_PATH): |
| with open(METADATA_PATH) as f: |
| for line in f: |
| r = json.loads(line.strip()) |
| key = (r["env"], r["tier"], r["episode_id"]) |
| meta[key] = r |
| return meta |
|
|
|
|
| def _lookup_metadata(video_name: str, metadata: dict): |
| """Try to find metadata for a video by parsing its filename. |
| |
| Tries exact match first, then fuzzy match on env+tier+episode. |
| """ |
| import re |
| |
| m = re.match(r"(\w+)_(\w+)_ep(\d+)\.mp4", video_name) |
| if m: |
| env, tier, ep_id = m.group(1), m.group(2), int(m.group(3)) |
| key = (env, tier, ep_id) |
| if key in metadata: |
| return metadata[key] |
|
|
| |
| for key, entry in metadata.items(): |
| env, tier, ep_id = key |
| if f"{env}_{tier}_ep{ep_id}" in video_name: |
| return entry |
|
|
| return None |
|
|
|
|
| def _compute_hybrid(parsed: dict, metadata_entry: dict = None, metadata: dict = None, sound_analysis: dict = None): |
| """Run hybrid judge combining VLM + rule-based scoring.""" |
| import sys |
| if "/root" not in sys.path: |
| sys.path.insert(0, "/root") |
| from hybrid_judge import hybrid_judge, hybrid_to_dict |
|
|
| if metadata_entry: |
| env = metadata_entry["env"] |
| all_metadata = metadata or {} |
| env_rets = [v["return"] for v in all_metadata.values() if v["env"] == env] |
| min_ret = min(env_rets) if env_rets else 0 |
| max_ret = max(env_rets) if env_rets else 1 |
|
|
| score = hybrid_judge( |
| vlm_parsed=parsed, |
| ep_return=metadata_entry["return"], |
| min_return=min_ret, |
| max_return=max_ret, |
| fell=metadata_entry.get("fell", False), |
| num_steps=metadata_entry.get("num_steps", 0), |
| tier=metadata_entry.get("tier", "unknown"), |
| env=metadata_entry.get("env", "unknown"), |
| ) |
| else: |
| score = hybrid_judge(vlm_parsed=parsed) |
|
|
| result = hybrid_to_dict(score) |
|
|
| if sound_analysis: |
| result["sound_analysis"] = sound_analysis |
| if sound_analysis.get("has_fall"): |
| confidence = sound_analysis.get("fall_confidence", 0.0) |
| penalty = confidence * 0.3 |
| result["predicted_reward"] = max(0.0, result["predicted_reward"] - penalty) |
| result["anomaly"] = (result.get("anomaly") or "") + f" [audio: fall detected, conf={confidence:.2f}]" |
|
|
| return result |
|
|
|
|
| @app.function( |
| image=app_image, |
| gpu="A100-40GB", |
| volumes={"/data": volume}, |
| secrets=[modal.Secret.from_name("huggingface-secret")], |
| timeout=3600, |
| ) |
| @modal.asgi_app() |
| def serve(): |
| from fastapi import FastAPI, UploadFile, File, Request |
| from fastapi.responses import HTMLResponse, JSONResponse |
| from typing import List |
|
|
| web_app = FastAPI() |
|
|
| metadata = _load_metadata() |
| print(f"[robomind] loaded {len(metadata)} metadata entries") |
|
|
| @web_app.get("/", response_class=HTMLResponse) |
| async def index(): |
| return HTML_PAGE |
|
|
| @web_app.post("/judge/images") |
| async def judge_images(files: List[UploadFile] = File(...)): |
| from PIL import Image |
| import io |
|
|
| images = [] |
| for f in files: |
| data = await f.read() |
| images.append(Image.open(io.BytesIO(data)).convert("RGB")) |
|
|
| model, tokenizer = _get_model() |
| response = _judge_images(model, tokenizer, images) |
| parsed = _parse_response(response) |
| hybrid = _compute_hybrid(parsed, metadata=metadata) |
| return JSONResponse({"parsed": parsed, "hybrid": hybrid, "raw": response}) |
|
|
| @web_app.post("/judge/video") |
| async def judge_video(file: UploadFile = File(...)): |
| data = await file.read() |
| original_name = file.filename or "unknown.mp4" |
| tmp = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) |
| tmp.write(data) |
| tmp.close() |
|
|
| frames = _extract_keyframes(tmp.name) |
|
|
| sound_result = None |
| try: |
| from robomind.sound import SoundAnalyzer |
| analyzer = SoundAnalyzer() |
| sa = analyzer.analyze_video(tmp.name) |
| sound_result = { |
| "has_fall": sa.has_fall, |
| "fall_confidence": round(sa.fall_confidence, 3), |
| "has_impact": sa.has_impact, |
| "impact_count": sa.impact_count, |
| "has_motor_strain": sa.has_motor_strain, |
| "has_rhythmic_gait": sa.has_rhythmic_gait, |
| "gait_quality": round(sa.gait_quality, 3), |
| "explanation": sa.explanation, |
| } |
| except Exception as e: |
| sound_result = {"error": str(e)} |
|
|
| os.unlink(tmp.name) |
|
|
| if not frames: |
| return JSONResponse({"error": "Failed to extract frames"}, status_code=400) |
|
|
| model, tokenizer = _get_model() |
| response = _judge_images(model, tokenizer, frames) |
| parsed = _parse_response(response) |
|
|
| meta_entry = _lookup_metadata(original_name, metadata) |
| hybrid = _compute_hybrid(parsed, meta_entry, metadata, sound_result) |
| return JSONResponse({"parsed": parsed, "hybrid": hybrid, "raw": response}) |
|
|
| @web_app.post("/judge/url") |
| async def judge_url(request: Request): |
| import urllib.request |
|
|
| body = await request.json() |
| url = body.get("url", "").strip() |
| if not url: |
| return JSONResponse({"error": "No URL provided"}, status_code=400) |
|
|
| from urllib.parse import urlparse |
| url_path = urlparse(url).path |
| original_name = os.path.basename(url_path) or "download.mp4" |
|
|
| tmp = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) |
| try: |
| urllib.request.urlretrieve(url, tmp.name) |
| except Exception as e: |
| os.unlink(tmp.name) |
| return JSONResponse({"error": f"Download failed: {e}"}, status_code=400) |
|
|
| frames = _extract_keyframes(tmp.name) |
|
|
| sound_result = None |
| try: |
| from robomind.sound import SoundAnalyzer |
| analyzer = SoundAnalyzer() |
| sa = analyzer.analyze_video(tmp.name) |
| sound_result = { |
| "has_fall": sa.has_fall, |
| "fall_confidence": round(sa.fall_confidence, 3), |
| "has_impact": sa.has_impact, |
| "impact_count": sa.impact_count, |
| "has_motor_strain": sa.has_motor_strain, |
| "has_rhythmic_gait": sa.has_rhythmic_gait, |
| "gait_quality": round(sa.gait_quality, 3), |
| "explanation": sa.explanation, |
| } |
| except Exception as e: |
| sound_result = {"error": str(e)} |
|
|
| os.unlink(tmp.name) |
|
|
| if not frames: |
| return JSONResponse({"error": "Failed to extract frames"}, status_code=400) |
|
|
| model, tokenizer = _get_model() |
| response = _judge_images(model, tokenizer, frames) |
| parsed = _parse_response(response) |
|
|
| meta_entry = _lookup_metadata(original_name, metadata) |
| hybrid = _compute_hybrid(parsed, meta_entry, metadata, sound_result) |
| return JSONResponse({"parsed": parsed, "hybrid": hybrid, "raw": response}) |
|
|
| return web_app |
|
|
|
|
| @app.local_entrypoint() |
| def main(): |
| serve.remote() |
|
|
