Spaces:
Running on Zero
Running on Zero
single-pass per-frame temporal-distance readout
Browse files
app.py
CHANGED
|
@@ -1,17 +1,18 @@
|
|
| 1 |
"""RynnValue-4B — robot-manipulation value model demo.
|
| 2 |
|
| 3 |
Given a manipulation video and the task instruction, RynnValue predicts the
|
| 4 |
-
*temporal distance* (remaining seconds until the task is done) for
|
| 5 |
-
|
| 6 |
-
video match the instruction /
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
|
|
|
| 15 |
"""
|
| 16 |
|
| 17 |
import os
|
|
@@ -65,13 +66,11 @@ EOS_TOKEN_ID = tokenizer.convert_tokens_to_ids("<|im_end|>")
|
|
| 65 |
|
| 66 |
DEFAULT_ROBOT = "a Franka single-arm robot"
|
| 67 |
DEFAULT_CAMERA = "the main camera"
|
| 68 |
-
DEFAULT_NUM_FRAMES =
|
| 69 |
-
DEFAULT_NUM_STEPS = 24
|
| 70 |
DEFAULT_MAX_IMAGE_SIDE = 384
|
| 71 |
DEFAULT_MAX_NEW_TOKENS = 128
|
| 72 |
-
BATCH_SIZE = 2
|
| 73 |
|
| 74 |
-
|
| 75 |
RENDER_MAX_SIDE = 640 # frames are downscaled to this for the rendered video
|
| 76 |
|
| 77 |
ROBOT_CHOICES = [
|
|
@@ -100,11 +99,11 @@ CAMERA_CHOICES = [
|
|
| 100 |
|
| 101 |
|
| 102 |
def load_video_frames(video_path):
|
| 103 |
-
"""Decode a video to RGB PIL frames, capped at
|
| 104 |
|
| 105 |
Returns (frames, effective_fps). Long videos are decoded with a stride so
|
| 106 |
-
memory
|
| 107 |
-
wall-clock timeline (and
|
| 108 |
"""
|
| 109 |
if not video_path or not os.path.isfile(video_path):
|
| 110 |
raise gr.Error("Please provide a video file.")
|
|
@@ -118,8 +117,8 @@ def load_video_frames(video_path):
|
|
| 118 |
estimated = int(duration * fps) if duration and np.isfinite(duration) else None
|
| 119 |
|
| 120 |
stride = 1
|
| 121 |
-
if estimated and estimated >
|
| 122 |
-
stride = int(np.ceil(estimated /
|
| 123 |
|
| 124 |
frames = []
|
| 125 |
try:
|
|
@@ -132,7 +131,7 @@ def load_video_frames(video_path):
|
|
| 132 |
s = RENDER_MAX_SIDE / max(w, h)
|
| 133 |
img = img.resize((max(1, round(w * s)), max(1, round(h * s))), Image.BICUBIC)
|
| 134 |
frames.append(img)
|
| 135 |
-
if len(frames) >=
|
| 136 |
break
|
| 137 |
finally:
|
| 138 |
reader.close()
|
|
@@ -154,16 +153,6 @@ def resize_frames(frames, max_side):
|
|
| 154 |
return [f.resize(new_size, resample=Image.BICUBIC) for f in frames]
|
| 155 |
|
| 156 |
|
| 157 |
-
def sample_frame_indices(total, num_frames):
|
| 158 |
-
"""Uniformly pick ``num_frames`` indices out of ``total`` frames."""
|
| 159 |
-
if num_frames <= 0 or num_frames >= total:
|
| 160 |
-
return list(range(total))
|
| 161 |
-
if num_frames == 1:
|
| 162 |
-
return [total - 1]
|
| 163 |
-
step = (total - 1) / (num_frames - 1)
|
| 164 |
-
return [int(round(j * step)) for j in range(num_frames)]
|
| 165 |
-
|
| 166 |
-
|
| 167 |
# ---------------------------------------------------------------------------
|
| 168 |
# Analysis parsing (same regexes as the reference script)
|
| 169 |
# ---------------------------------------------------------------------------
|
|
@@ -197,12 +186,12 @@ def _format_time(seconds):
|
|
| 197 |
return f"{minutes:02d}:{secs:02d}.{millis:03d}"
|
| 198 |
|
| 199 |
|
| 200 |
-
def _render_trend_frames(values, sampled_indices, fps, size,
|
| 201 |
"""Render one plot image per prediction step.
|
| 202 |
|
| 203 |
-
The plot only changes at the sampled indices, so we render
|
| 204 |
-
|
| 205 |
-
|
| 206 |
"""
|
| 207 |
w, h = size
|
| 208 |
dpi = 100
|
|
@@ -210,12 +199,13 @@ def _render_trend_frames(values, sampled_indices, fps, size, title, task_title):
|
|
| 210 |
|
| 211 |
x = np.asarray(sampled_indices, dtype=float)
|
| 212 |
y = np.asarray(values, dtype=float)
|
| 213 |
-
|
| 214 |
|
| 215 |
-
|
| 216 |
-
|
|
|
|
| 217 |
ax1.set_xlabel("Frame", fontsize=9)
|
| 218 |
-
ax1.set_ylabel("
|
| 219 |
ax1.tick_params(axis="x", labelsize=8)
|
| 220 |
ax1.tick_params(axis="y", labelcolor="tab:blue", labelsize=8)
|
| 221 |
ax1.grid(True, alpha=0.3)
|
|
@@ -229,13 +219,11 @@ def _render_trend_frames(values, sampled_indices, fps, size, title, task_title):
|
|
| 229 |
ax1.set_ylim(y_min - margin, y_max + margin)
|
| 230 |
|
| 231 |
ax2 = ax1.twinx()
|
| 232 |
-
ax2.plot(x,
|
| 233 |
-
pt2 = ax2.scatter(
|
| 234 |
-
|
| 235 |
-
)
|
| 236 |
-
ax2.set_ylabel("Remaining Time (s)", color="green", fontsize=9)
|
| 237 |
ax2.tick_params(axis="y", labelcolor="green", labelsize=8)
|
| 238 |
-
rt_min, rt_max = float(np.min(
|
| 239 |
if rt_min == rt_max:
|
| 240 |
rt_min -= 1.0
|
| 241 |
rt_max += 1.0
|
|
@@ -243,7 +231,7 @@ def _render_trend_frames(values, sampled_indices, fps, size, title, task_title):
|
|
| 243 |
ax2.set_ylim(rt_min - rt_margin, rt_max + rt_margin)
|
| 244 |
|
| 245 |
short_task = task_title if len(task_title) <= 42 else task_title[:39] + "..."
|
| 246 |
-
ax1.set_title(f"{short_task}\
|
| 247 |
|
| 248 |
lines1, labels1 = ax1.get_legend_handles_labels()
|
| 249 |
lines2, labels2 = ax2.get_legend_handles_labels()
|
|
@@ -253,10 +241,9 @@ def _render_trend_frames(values, sampled_indices, fps, size, title, task_title):
|
|
| 253 |
for i in range(len(y)):
|
| 254 |
line.set_data(x[: i + 1], y[: i + 1])
|
| 255 |
pt1.set_offsets(np.array([[x[i], y[i]]]))
|
| 256 |
-
pt2.set_offsets(np.array([[x[i],
|
| 257 |
fig.canvas.draw()
|
| 258 |
-
|
| 259 |
-
images.append(Image.fromarray(buf))
|
| 260 |
plt.close(fig)
|
| 261 |
return images
|
| 262 |
|
|
@@ -271,13 +258,13 @@ def _overlay_font(size):
|
|
| 271 |
def _draw_overlay_text(img, lines, font):
|
| 272 |
img = img.copy()
|
| 273 |
draw = ImageDraw.Draw(img)
|
| 274 |
-
x, y = 8,
|
| 275 |
for line in lines:
|
| 276 |
draw.text(
|
| 277 |
(x, y), line, fill=(255, 235, 59), font=font, stroke_width=2, stroke_fill=(0, 0, 0)
|
| 278 |
)
|
| 279 |
bbox = draw.textbbox((x, y), line, font=font)
|
| 280 |
-
y += (bbox[3] - bbox[1]) +
|
| 281 |
return img
|
| 282 |
|
| 283 |
|
|
@@ -293,27 +280,23 @@ def save_video_with_trend(frames, values, sampled_indices, fps, output_path, tas
|
|
| 293 |
base_w, base_h = int(round(base_w * scale)), 260
|
| 294 |
frames = [f.resize((base_w, base_h)) for f in frames]
|
| 295 |
|
| 296 |
-
plot_w = int(min(
|
| 297 |
plot_h = max(260, base_h)
|
| 298 |
|
| 299 |
-
plots = _render_trend_frames(
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
"Remaining Time (s)",
|
| 305 |
-
task_title,
|
| 306 |
-
)
|
| 307 |
-
if plots[0].height != base_h:
|
| 308 |
-
plots = [p.resize((plot_w, base_h)) for p in plots]
|
| 309 |
|
| 310 |
idx_to_pos = {idx: pos for pos, idx in enumerate(sampled_indices)}
|
| 311 |
n = len(frames)
|
| 312 |
-
|
| 313 |
|
| 314 |
font = _overlay_font(16)
|
| 315 |
canvas_w = _ceil_to_multiple(base_w + plot_w, 16)
|
| 316 |
canvas_h = _ceil_to_multiple(base_h, 16)
|
|
|
|
| 317 |
|
| 318 |
writer = imageio.get_writer(
|
| 319 |
output_path,
|
|
@@ -321,10 +304,11 @@ def save_video_with_trend(frames, values, sampled_indices, fps, output_path, tas
|
|
| 321 |
codec="libx264",
|
| 322 |
quality=7,
|
| 323 |
macro_block_size=None,
|
|
|
|
| 324 |
)
|
| 325 |
try:
|
| 326 |
pos = 0
|
| 327 |
-
short_task = task_title if len(task_title) <=
|
| 328 |
for i, img in enumerate(frames):
|
| 329 |
if i in idx_to_pos:
|
| 330 |
pos = idx_to_pos[i]
|
|
@@ -333,14 +317,13 @@ def save_video_with_trend(frames, values, sampled_indices, fps, output_path, tas
|
|
| 333 |
[
|
| 334 |
f"task: {short_task}",
|
| 335 |
f"predicted remaining: {values[pos]:.2f}s",
|
| 336 |
-
f"actual remaining: {_format_time(
|
| 337 |
],
|
| 338 |
font,
|
| 339 |
)
|
| 340 |
-
canvas =
|
| 341 |
-
canvas
|
| 342 |
-
|
| 343 |
-
writer.append_data(np.asarray(canvas))
|
| 344 |
finally:
|
| 345 |
writer.close()
|
| 346 |
return output_path
|
|
@@ -357,7 +340,6 @@ def _gpu_duration(
|
|
| 357 |
robot_description=DEFAULT_ROBOT,
|
| 358 |
camera_description=DEFAULT_CAMERA,
|
| 359 |
num_frames=DEFAULT_NUM_FRAMES,
|
| 360 |
-
num_steps=DEFAULT_NUM_STEPS,
|
| 361 |
max_image_side=DEFAULT_MAX_IMAGE_SIDE,
|
| 362 |
max_new_tokens=DEFAULT_MAX_NEW_TOKENS,
|
| 363 |
*args,
|
|
@@ -365,13 +347,14 @@ def _gpu_duration(
|
|
| 365 |
):
|
| 366 |
try:
|
| 367 |
nf = int(num_frames)
|
| 368 |
-
ns = int(num_steps)
|
| 369 |
side = int(max_image_side)
|
| 370 |
except (TypeError, ValueError):
|
| 371 |
-
nf,
|
| 372 |
-
#
|
| 373 |
-
|
| 374 |
-
|
|
|
|
|
|
|
| 375 |
|
| 376 |
|
| 377 |
@spaces.GPU(duration=_gpu_duration)
|
|
@@ -381,7 +364,6 @@ def analyze(
|
|
| 381 |
robot_description=DEFAULT_ROBOT,
|
| 382 |
camera_description=DEFAULT_CAMERA,
|
| 383 |
num_frames=DEFAULT_NUM_FRAMES,
|
| 384 |
-
num_steps=DEFAULT_NUM_STEPS,
|
| 385 |
max_image_side=DEFAULT_MAX_IMAGE_SIDE,
|
| 386 |
max_new_tokens=DEFAULT_MAX_NEW_TOKENS,
|
| 387 |
progress=gr.Progress(),
|
|
@@ -399,104 +381,52 @@ def analyze(
|
|
| 399 |
robot_description = DEFAULT_ROBOT
|
| 400 |
|
| 401 |
num_frames = int(num_frames)
|
| 402 |
-
num_steps = int(num_steps)
|
| 403 |
max_image_side = int(max_image_side)
|
| 404 |
max_new_tokens = int(max_new_tokens)
|
| 405 |
|
| 406 |
-
progress(0.
|
| 407 |
frames, fps = load_video_frames(video)
|
| 408 |
total = len(frames)
|
| 409 |
model_images = resize_frames(frames, max_image_side)
|
| 410 |
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
def build_prefix_sample(end_idx):
|
| 414 |
-
frame_idx = np.linspace(0, end_idx, num_frames, dtype=int)
|
| 415 |
-
prefix_images = [model_images[j] for j in frame_idx]
|
| 416 |
-
return processor.process_episode(
|
| 417 |
-
instruction=instruction,
|
| 418 |
-
images=prefix_images,
|
| 419 |
-
robot_description=robot_description,
|
| 420 |
-
camera_description=camera_description,
|
| 421 |
-
)
|
| 422 |
-
|
| 423 |
-
def run_batch(samples):
|
| 424 |
-
"""One forward pass over a batch of prefixes -> last-slot value each."""
|
| 425 |
-
batch_kwargs = dict(
|
| 426 |
-
input_ids=torch.cat([s["input_ids"] for s in samples], dim=0).to("cuda").long(),
|
| 427 |
-
attention_mask=torch.cat([s["attention_mask"] for s in samples], dim=0)
|
| 428 |
-
.to("cuda")
|
| 429 |
-
.long(),
|
| 430 |
-
pixel_values=torch.cat([s["pixel_values"].flatten(0, 1) for s in samples], dim=0)
|
| 431 |
-
.to("cuda")
|
| 432 |
-
.to(torch.bfloat16),
|
| 433 |
-
image_grid_thw=torch.cat([s["image_grid_thw"].flatten(0, 1) for s in samples], dim=0)
|
| 434 |
-
.to("cuda")
|
| 435 |
-
.long(),
|
| 436 |
-
)
|
| 437 |
-
with torch.inference_mode():
|
| 438 |
-
outputs = model(**batch_kwargs)
|
| 439 |
-
pred = outputs.value.pred_value
|
| 440 |
-
if pred.dim() == 2 and pred.shape[0] == 1:
|
| 441 |
-
pred = pred.reshape(len(samples), -1)
|
| 442 |
-
if pred.dim() == 3:
|
| 443 |
-
pred = pred.mean(dim=0)
|
| 444 |
-
if pred.dim() == 2 and pred.shape[-1] > 1:
|
| 445 |
-
pred = pred[:, -1]
|
| 446 |
-
elif pred.dim() == 2:
|
| 447 |
-
pred = pred[:, 0]
|
| 448 |
-
return pred.float().reshape(-1).tolist()
|
| 449 |
-
|
| 450 |
-
pred_value = []
|
| 451 |
-
final_sample = None
|
| 452 |
-
batch = []
|
| 453 |
-
for step, end_idx in enumerate(eval_indices):
|
| 454 |
-
sample = build_prefix_sample(end_idx)
|
| 455 |
-
if end_idx == eval_indices[-1]:
|
| 456 |
-
final_sample = sample
|
| 457 |
-
batch.append(sample)
|
| 458 |
-
if len(batch) >= BATCH_SIZE or step == len(eval_indices) - 1:
|
| 459 |
-
pred_value.extend(run_batch(batch))
|
| 460 |
-
batch = []
|
| 461 |
-
progress(
|
| 462 |
-
0.05 + 0.75 * len(pred_value) / len(eval_indices),
|
| 463 |
-
desc=f"Value pass {len(pred_value)}/{len(eval_indices)}",
|
| 464 |
-
)
|
| 465 |
|
| 466 |
-
|
| 467 |
-
|
| 468 |
-
dbg_sample = processor.process_episode(
|
| 469 |
instruction=instruction,
|
| 470 |
-
images=[model_images[j] for j in
|
| 471 |
robot_description=robot_description,
|
| 472 |
camera_description=camera_description,
|
| 473 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 474 |
with torch.inference_mode():
|
| 475 |
-
|
| 476 |
-
input_ids=
|
| 477 |
-
attention_mask=
|
| 478 |
-
pixel_values=
|
| 479 |
-
image_grid_thw=
|
| 480 |
)
|
| 481 |
-
|
| 482 |
-
|
| 483 |
-
|
| 484 |
-
|
| 485 |
-
|
| 486 |
-
|
| 487 |
-
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
]
|
| 491 |
-
|
| 492 |
-
progress(0.82, desc="Generating analysis...")
|
| 493 |
-
input_ids = final_sample["input_ids"].to("cuda").long()
|
| 494 |
with torch.inference_mode():
|
| 495 |
gen_out = model.generate(
|
| 496 |
input_ids=input_ids,
|
| 497 |
-
attention_mask=
|
| 498 |
-
pixel_values=
|
| 499 |
-
image_grid_thw=
|
| 500 |
max_new_tokens=max_new_tokens,
|
| 501 |
do_sample=False,
|
| 502 |
num_beams=1,
|
|
@@ -508,32 +438,32 @@ def analyze(
|
|
| 508 |
analysis = parse_analysis(analysis_text)
|
| 509 |
gpu_seconds = time.time() - started
|
| 510 |
|
| 511 |
-
progress(0.
|
| 512 |
out_path = os.path.join(tempfile.mkdtemp(), "rynnvalue_trend.mp4")
|
| 513 |
-
save_video_with_trend(frames, pred_value,
|
| 514 |
|
| 515 |
def _verdict(v):
|
| 516 |
if v is None:
|
| 517 |
return "—"
|
| 518 |
return "✅ Yes" if v.lower() == "yes" else "❌ No"
|
| 519 |
|
|
|
|
|
|
|
| 520 |
description = analysis["description"] or (analysis_text.strip() or "—")
|
| 521 |
summary = "\n".join(
|
| 522 |
[
|
| 523 |
-
f"**Task instruction** — {instruction}",
|
| 524 |
-
"",
|
| 525 |
f"**Video description** — {description}",
|
| 526 |
"",
|
| 527 |
f"**Matches the instruction?** {_verdict(analysis['match'])} · "
|
| 528 |
f"**Task completed?** {_verdict(analysis['success'])}",
|
| 529 |
"",
|
| 530 |
-
f"**
|
| 531 |
-
f"{pred_value[-1]:.2f} s at the last
|
| 532 |
-
f"(video is {(total - 1) / fps:.1f} s long, {total} frames decoded).",
|
| 533 |
"",
|
| 534 |
-
"
|
|
|
|
| 535 |
"",
|
| 536 |
-
f"<sub>{
|
| 537 |
f"{gpu_seconds:.1f} s on GPU</sub>",
|
| 538 |
]
|
| 539 |
)
|
|
@@ -549,21 +479,21 @@ DESCRIPTION = """
|
|
| 549 |
|
| 550 |
# 🤖 RynnValue-4B — how much longer until the robot is done?
|
| 551 |
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
|
| 556 |
|
| 557 |
</div>
|
| 558 |
|
| 559 |
-
**RynnValue** is a value foundation model for robot manipulation, built on RynnBrain (Qwen3-VL)
|
| 560 |
-
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
whether the task **succeeded**.
|
| 564 |
|
| 565 |
-
The
|
| 566 |
-
remaining time
|
| 567 |
"""
|
| 568 |
|
| 569 |
with gr.Blocks(title="RynnValue-4B") as demo:
|
|
@@ -571,7 +501,7 @@ with gr.Blocks(title="RynnValue-4B") as demo:
|
|
| 571 |
|
| 572 |
with gr.Row():
|
| 573 |
with gr.Column(scale=1):
|
| 574 |
-
video_in = gr.Video(label="Manipulation video", sources=["upload"], height=
|
| 575 |
instruction = gr.Textbox(
|
| 576 |
label="Task instruction",
|
| 577 |
placeholder="Put the box in the drawer and close it",
|
|
@@ -592,23 +522,20 @@ with gr.Blocks(title="RynnValue-4B") as demo:
|
|
| 592 |
allow_custom_value=True,
|
| 593 |
)
|
| 594 |
with gr.Column(scale=1):
|
| 595 |
-
video_out = gr.Video(label="Value trend", height=
|
| 596 |
analysis_md = gr.Markdown(label="Analysis")
|
| 597 |
|
| 598 |
with gr.Accordion("Advanced options", open=False):
|
| 599 |
gr.Markdown(
|
| 600 |
-
"The
|
| 601 |
-
"
|
| 602 |
-
"
|
| 603 |
)
|
| 604 |
num_frames = gr.Slider(
|
| 605 |
-
|
| 606 |
-
)
|
| 607 |
-
num_steps = gr.Slider(
|
| 608 |
-
8, 48, value=DEFAULT_NUM_STEPS, step=4, label="Curve points (prefix evaluations)"
|
| 609 |
)
|
| 610 |
max_image_side = gr.Slider(
|
| 611 |
-
256,
|
| 612 |
)
|
| 613 |
max_new_tokens = gr.Slider(
|
| 614 |
32, 256, value=DEFAULT_MAX_NEW_TOKENS, step=32, label="Analysis token budget"
|
|
@@ -620,7 +547,6 @@ with gr.Blocks(title="RynnValue-4B") as demo:
|
|
| 620 |
robot_desc,
|
| 621 |
camera_desc,
|
| 622 |
num_frames,
|
| 623 |
-
num_steps,
|
| 624 |
max_image_side,
|
| 625 |
max_new_tokens,
|
| 626 |
]
|
|
|
|
| 1 |
"""RynnValue-4B — robot-manipulation value model demo.
|
| 2 |
|
| 3 |
Given a manipulation video and the task instruction, RynnValue predicts the
|
| 4 |
+
*temporal distance* (remaining seconds until the task is done) for every
|
| 5 |
+
sampled frame, the signed time delta between consecutive frames, and a short
|
| 6 |
+
natural-language analysis (description / does the video match the instruction /
|
| 7 |
+
did it succeed).
|
| 8 |
+
|
| 9 |
+
Inference follows the model card's `process_episode` protocol: the video is
|
| 10 |
+
uniformly resampled to `num_frames` frames, the episode prompt interleaves each
|
| 11 |
+
frame with its `<value>` / `<relative_value>` query group, and one forward pass
|
| 12 |
+
yields one remaining-time prediction per frame (the LM is causal, so the value
|
| 13 |
+
at frame *i* only conditions on frames 0..i). A second `generate()` pass on the
|
| 14 |
+
same prompt produces the Analysis block. Rendering reproduces the authors'
|
| 15 |
+
`rynn_infer/plot_utils.py` layout: the clip next to a synchronized value trend.
|
| 16 |
"""
|
| 17 |
|
| 18 |
import os
|
|
|
|
| 66 |
|
| 67 |
DEFAULT_ROBOT = "a Franka single-arm robot"
|
| 68 |
DEFAULT_CAMERA = "the main camera"
|
| 69 |
+
DEFAULT_NUM_FRAMES = 48
|
|
|
|
| 70 |
DEFAULT_MAX_IMAGE_SIDE = 384
|
| 71 |
DEFAULT_MAX_NEW_TOKENS = 128
|
|
|
|
| 72 |
|
| 73 |
+
MAX_FRAMES = 450 # decoded/rendered frame cap (memory + render-time guard)
|
| 74 |
RENDER_MAX_SIDE = 640 # frames are downscaled to this for the rendered video
|
| 75 |
|
| 76 |
ROBOT_CHOICES = [
|
|
|
|
| 99 |
|
| 100 |
|
| 101 |
def load_video_frames(video_path):
|
| 102 |
+
"""Decode a video to RGB PIL frames, capped at MAX_FRAMES.
|
| 103 |
|
| 104 |
Returns (frames, effective_fps). Long videos are decoded with a stride so
|
| 105 |
+
memory and render time stay bounded; the effective fps is scaled to match,
|
| 106 |
+
which keeps the wall-clock timeline (and the seconds axis) correct.
|
| 107 |
"""
|
| 108 |
if not video_path or not os.path.isfile(video_path):
|
| 109 |
raise gr.Error("Please provide a video file.")
|
|
|
|
| 117 |
estimated = int(duration * fps) if duration and np.isfinite(duration) else None
|
| 118 |
|
| 119 |
stride = 1
|
| 120 |
+
if estimated and estimated > MAX_FRAMES:
|
| 121 |
+
stride = int(np.ceil(estimated / MAX_FRAMES))
|
| 122 |
|
| 123 |
frames = []
|
| 124 |
try:
|
|
|
|
| 131 |
s = RENDER_MAX_SIDE / max(w, h)
|
| 132 |
img = img.resize((max(1, round(w * s)), max(1, round(h * s))), Image.BICUBIC)
|
| 133 |
frames.append(img)
|
| 134 |
+
if len(frames) >= MAX_FRAMES:
|
| 135 |
break
|
| 136 |
finally:
|
| 137 |
reader.close()
|
|
|
|
| 153 |
return [f.resize(new_size, resample=Image.BICUBIC) for f in frames]
|
| 154 |
|
| 155 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
# ---------------------------------------------------------------------------
|
| 157 |
# Analysis parsing (same regexes as the reference script)
|
| 158 |
# ---------------------------------------------------------------------------
|
|
|
|
| 186 |
return f"{minutes:02d}:{secs:02d}.{millis:03d}"
|
| 187 |
|
| 188 |
|
| 189 |
+
def _render_trend_frames(values, sampled_indices, fps, size, task_title):
|
| 190 |
"""Render one plot image per prediction step.
|
| 191 |
|
| 192 |
+
The plot only changes at the sampled indices, so we render ``len(values)``
|
| 193 |
+
images and reuse them across the video instead of re-rendering a figure for
|
| 194 |
+
every video frame.
|
| 195 |
"""
|
| 196 |
w, h = size
|
| 197 |
dpi = 100
|
|
|
|
| 199 |
|
| 200 |
x = np.asarray(sampled_indices, dtype=float)
|
| 201 |
y = np.asarray(values, dtype=float)
|
| 202 |
+
actual = (x[-1] - x) / float(fps)
|
| 203 |
|
| 204 |
+
ax1.plot(x, y, color="tab:blue", linewidth=1.0, alpha=0.25)
|
| 205 |
+
(line,) = ax1.plot([], [], color="tab:blue", linewidth=2.0, label="predicted remaining")
|
| 206 |
+
pt1 = ax1.scatter([x[0]], [y[0]], color="red", s=32, zorder=3, label="current prediction")
|
| 207 |
ax1.set_xlabel("Frame", fontsize=9)
|
| 208 |
+
ax1.set_ylabel("Predicted value (s)", color="tab:blue", fontsize=9)
|
| 209 |
ax1.tick_params(axis="x", labelsize=8)
|
| 210 |
ax1.tick_params(axis="y", labelcolor="tab:blue", labelsize=8)
|
| 211 |
ax1.grid(True, alpha=0.3)
|
|
|
|
| 219 |
ax1.set_ylim(y_min - margin, y_max + margin)
|
| 220 |
|
| 221 |
ax2 = ax1.twinx()
|
| 222 |
+
ax2.plot(x, actual, color="green", linestyle="--", linewidth=2.0, label="actual remaining")
|
| 223 |
+
pt2 = ax2.scatter([x[0]], [actual[0]], color="green", s=26, zorder=3)
|
| 224 |
+
ax2.set_ylabel("Actual remaining (s)", color="green", fontsize=9)
|
|
|
|
|
|
|
| 225 |
ax2.tick_params(axis="y", labelcolor="green", labelsize=8)
|
| 226 |
+
rt_min, rt_max = float(np.min(actual)), float(np.max(actual))
|
| 227 |
if rt_min == rt_max:
|
| 228 |
rt_min -= 1.0
|
| 229 |
rt_max += 1.0
|
|
|
|
| 231 |
ax2.set_ylim(rt_min - rt_margin, rt_max + rt_margin)
|
| 232 |
|
| 233 |
short_task = task_title if len(task_title) <= 42 else task_title[:39] + "..."
|
| 234 |
+
ax1.set_title(f"{short_task}\nTemporal distance to completion", fontsize=10)
|
| 235 |
|
| 236 |
lines1, labels1 = ax1.get_legend_handles_labels()
|
| 237 |
lines2, labels2 = ax2.get_legend_handles_labels()
|
|
|
|
| 241 |
for i in range(len(y)):
|
| 242 |
line.set_data(x[: i + 1], y[: i + 1])
|
| 243 |
pt1.set_offsets(np.array([[x[i], y[i]]]))
|
| 244 |
+
pt2.set_offsets(np.array([[x[i], actual[i]]]))
|
| 245 |
fig.canvas.draw()
|
| 246 |
+
images.append(np.asarray(fig.canvas.buffer_rgba())[:, :, :3].copy())
|
|
|
|
| 247 |
plt.close(fig)
|
| 248 |
return images
|
| 249 |
|
|
|
|
| 258 |
def _draw_overlay_text(img, lines, font):
|
| 259 |
img = img.copy()
|
| 260 |
draw = ImageDraw.Draw(img)
|
| 261 |
+
x, y = 8, 6
|
| 262 |
for line in lines:
|
| 263 |
draw.text(
|
| 264 |
(x, y), line, fill=(255, 235, 59), font=font, stroke_width=2, stroke_fill=(0, 0, 0)
|
| 265 |
)
|
| 266 |
bbox = draw.textbbox((x, y), line, font=font)
|
| 267 |
+
y += (bbox[3] - bbox[1]) + 9
|
| 268 |
return img
|
| 269 |
|
| 270 |
|
|
|
|
| 280 |
base_w, base_h = int(round(base_w * scale)), 260
|
| 281 |
frames = [f.resize((base_w, base_h)) for f in frames]
|
| 282 |
|
| 283 |
+
plot_w = int(min(560, max(320, base_w * 0.8)))
|
| 284 |
plot_h = max(260, base_h)
|
| 285 |
|
| 286 |
+
plots = _render_trend_frames(values, sampled_indices, fps, (plot_w, plot_h), task_title)
|
| 287 |
+
if plots[0].shape[0] != base_h:
|
| 288 |
+
plots = [
|
| 289 |
+
np.asarray(Image.fromarray(p).resize((plot_w, base_h))) for p in plots
|
| 290 |
+
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 291 |
|
| 292 |
idx_to_pos = {idx: pos for pos, idx in enumerate(sampled_indices)}
|
| 293 |
n = len(frames)
|
| 294 |
+
actual_all = (n - 1 - np.arange(n)) / float(fps)
|
| 295 |
|
| 296 |
font = _overlay_font(16)
|
| 297 |
canvas_w = _ceil_to_multiple(base_w + plot_w, 16)
|
| 298 |
canvas_h = _ceil_to_multiple(base_h, 16)
|
| 299 |
+
canvas = np.full((canvas_h, canvas_w, 3), 255, dtype=np.uint8)
|
| 300 |
|
| 301 |
writer = imageio.get_writer(
|
| 302 |
output_path,
|
|
|
|
| 304 |
codec="libx264",
|
| 305 |
quality=7,
|
| 306 |
macro_block_size=None,
|
| 307 |
+
output_params=["-pix_fmt", "yuv420p"],
|
| 308 |
)
|
| 309 |
try:
|
| 310 |
pos = 0
|
| 311 |
+
short_task = task_title if len(task_title) <= 44 else task_title[:41] + "..."
|
| 312 |
for i, img in enumerate(frames):
|
| 313 |
if i in idx_to_pos:
|
| 314 |
pos = idx_to_pos[i]
|
|
|
|
| 317 |
[
|
| 318 |
f"task: {short_task}",
|
| 319 |
f"predicted remaining: {values[pos]:.2f}s",
|
| 320 |
+
f"actual remaining: {_format_time(actual_all[i])}",
|
| 321 |
],
|
| 322 |
font,
|
| 323 |
)
|
| 324 |
+
canvas[:base_h, :base_w] = np.asarray(left)
|
| 325 |
+
canvas[:base_h, base_w : base_w + plot_w] = plots[pos]
|
| 326 |
+
writer.append_data(canvas)
|
|
|
|
| 327 |
finally:
|
| 328 |
writer.close()
|
| 329 |
return output_path
|
|
|
|
| 340 |
robot_description=DEFAULT_ROBOT,
|
| 341 |
camera_description=DEFAULT_CAMERA,
|
| 342 |
num_frames=DEFAULT_NUM_FRAMES,
|
|
|
|
| 343 |
max_image_side=DEFAULT_MAX_IMAGE_SIDE,
|
| 344 |
max_new_tokens=DEFAULT_MAX_NEW_TOKENS,
|
| 345 |
*args,
|
|
|
|
| 347 |
):
|
| 348 |
try:
|
| 349 |
nf = int(num_frames)
|
|
|
|
| 350 |
side = int(max_image_side)
|
| 351 |
except (TypeError, ValueError):
|
| 352 |
+
nf, side = DEFAULT_NUM_FRAMES, DEFAULT_MAX_IMAGE_SIDE
|
| 353 |
+
# Sequence length scales with frames x tokens-per-frame, and the eager
|
| 354 |
+
# value-isolation attention makes the forward grow ~quadratically in it.
|
| 355 |
+
# Calibrated against a measured 32-frame @384px run.
|
| 356 |
+
n = nf * (side / 384.0) ** 2
|
| 357 |
+
return int(min(240, 20 + 0.6 * n + 0.012 * n**2))
|
| 358 |
|
| 359 |
|
| 360 |
@spaces.GPU(duration=_gpu_duration)
|
|
|
|
| 364 |
robot_description=DEFAULT_ROBOT,
|
| 365 |
camera_description=DEFAULT_CAMERA,
|
| 366 |
num_frames=DEFAULT_NUM_FRAMES,
|
|
|
|
| 367 |
max_image_side=DEFAULT_MAX_IMAGE_SIDE,
|
| 368 |
max_new_tokens=DEFAULT_MAX_NEW_TOKENS,
|
| 369 |
progress=gr.Progress(),
|
|
|
|
| 381 |
robot_description = DEFAULT_ROBOT
|
| 382 |
|
| 383 |
num_frames = int(num_frames)
|
|
|
|
| 384 |
max_image_side = int(max_image_side)
|
| 385 |
max_new_tokens = int(max_new_tokens)
|
| 386 |
|
| 387 |
+
progress(0.03, desc="Decoding video...")
|
| 388 |
frames, fps = load_video_frames(video)
|
| 389 |
total = len(frames)
|
| 390 |
model_images = resize_frames(frames, max_image_side)
|
| 391 |
|
| 392 |
+
num_frames = max(2, min(num_frames, total))
|
| 393 |
+
sampled_indices = np.linspace(0, total - 1, num_frames, dtype=int).tolist()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 394 |
|
| 395 |
+
progress(0.18, desc="Building episode prompt...")
|
| 396 |
+
sample = processor.process_episode(
|
|
|
|
| 397 |
instruction=instruction,
|
| 398 |
+
images=[model_images[j] for j in sampled_indices],
|
| 399 |
robot_description=robot_description,
|
| 400 |
camera_description=camera_description,
|
| 401 |
)
|
| 402 |
+
input_ids = sample["input_ids"].to("cuda").long()
|
| 403 |
+
attention_mask = sample["attention_mask"].to("cuda").long()
|
| 404 |
+
pixel_values = sample["pixel_values"].flatten(0, 1).to("cuda")
|
| 405 |
+
image_grid_thw = sample["image_grid_thw"].flatten(0, 1).to("cuda").long()
|
| 406 |
+
|
| 407 |
+
progress(0.28, desc="Predicting temporal distance...")
|
| 408 |
with torch.inference_mode():
|
| 409 |
+
outputs = model(
|
| 410 |
+
input_ids=input_ids,
|
| 411 |
+
attention_mask=attention_mask,
|
| 412 |
+
pixel_values=pixel_values,
|
| 413 |
+
image_grid_thw=image_grid_thw,
|
| 414 |
)
|
| 415 |
+
# (num_value_heads, num_frames) -> one remaining-time prediction per frame
|
| 416 |
+
pred_value = outputs.value.pred_value.float()
|
| 417 |
+
if pred_value.dim() > 1:
|
| 418 |
+
pred_value = pred_value.reshape(-1, len(sampled_indices)).mean(dim=0)
|
| 419 |
+
pred_value = pred_value.reshape(-1).tolist()
|
| 420 |
+
# relative head: signed time delta between consecutive sampled frames
|
| 421 |
+
rel_value = outputs.relative.pred_value.float().reshape(-1).tolist()
|
| 422 |
+
|
| 423 |
+
progress(0.6, desc="Generating analysis...")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 424 |
with torch.inference_mode():
|
| 425 |
gen_out = model.generate(
|
| 426 |
input_ids=input_ids,
|
| 427 |
+
attention_mask=attention_mask,
|
| 428 |
+
pixel_values=pixel_values,
|
| 429 |
+
image_grid_thw=image_grid_thw,
|
| 430 |
max_new_tokens=max_new_tokens,
|
| 431 |
do_sample=False,
|
| 432 |
num_beams=1,
|
|
|
|
| 438 |
analysis = parse_analysis(analysis_text)
|
| 439 |
gpu_seconds = time.time() - started
|
| 440 |
|
| 441 |
+
progress(0.85, desc="Rendering trend video...")
|
| 442 |
out_path = os.path.join(tempfile.mkdtemp(), "rynnvalue_trend.mp4")
|
| 443 |
+
save_video_with_trend(frames, pred_value, sampled_indices, fps, out_path, instruction)
|
| 444 |
|
| 445 |
def _verdict(v):
|
| 446 |
if v is None:
|
| 447 |
return "—"
|
| 448 |
return "✅ Yes" if v.lower() == "yes" else "❌ No"
|
| 449 |
|
| 450 |
+
clip_seconds = (total - 1) / fps
|
| 451 |
+
predicted_span = float(np.sum(rel_value)) if rel_value else float("nan")
|
| 452 |
description = analysis["description"] or (analysis_text.strip() or "—")
|
| 453 |
summary = "\n".join(
|
| 454 |
[
|
|
|
|
|
|
|
| 455 |
f"**Video description** — {description}",
|
| 456 |
"",
|
| 457 |
f"**Matches the instruction?** {_verdict(analysis['match'])} · "
|
| 458 |
f"**Task completed?** {_verdict(analysis['success'])}",
|
| 459 |
"",
|
| 460 |
+
f"**Temporal distance** — {pred_value[0]:.2f} s remaining at the first sampled frame → "
|
| 461 |
+
f"{pred_value[-1]:.2f} s at the last. The clip itself is {clip_seconds:.1f} s long.",
|
|
|
|
| 462 |
"",
|
| 463 |
+
f"**Relative head** — the predicted time deltas between the {num_frames} sampled "
|
| 464 |
+
f"frames sum to {predicted_span:.1f} s (actual clip length {clip_seconds:.1f} s).",
|
| 465 |
"",
|
| 466 |
+
f"<sub>{num_frames} frames · longest side {max_image_side}px · "
|
| 467 |
f"{gpu_seconds:.1f} s on GPU</sub>",
|
| 468 |
]
|
| 469 |
)
|
|
|
|
| 479 |
|
| 480 |
# 🤖 RynnValue-4B — how much longer until the robot is done?
|
| 481 |
|
| 482 |
+
<a href="https://huggingface.co/Alibaba-DAMO-Academy/RynnValue-4B">Model</a> ·
|
| 483 |
+
<a href="https://arxiv.org/abs/2608.09853">Paper</a> ·
|
| 484 |
+
<a href="https://github.com/alibaba-damo-academy/RynnValue">GitHub</a> ·
|
| 485 |
+
<a href="https://alibaba-damo-academy.github.io/RynnValue.github.io/">Project page</a>
|
| 486 |
|
| 487 |
</div>
|
| 488 |
|
| 489 |
+
**RynnValue** is a value foundation model for robot manipulation, built on RynnBrain (Qwen3-VL) and
|
| 490 |
+
trained on 7,000+ hours of embodied data. Give it a manipulation video **and** the task instruction,
|
| 491 |
+
and for every sampled frame it predicts the **temporal distance** — the remaining time in seconds
|
| 492 |
+
until the task is complete — plus a short analysis: what the video shows, whether it **matches** the
|
| 493 |
+
instruction, and whether the task **succeeded**.
|
| 494 |
|
| 495 |
+
The result video plays the clip beside the predicted value curve (blue), with the clip's actual
|
| 496 |
+
remaining time (dashed green) for reference.
|
| 497 |
"""
|
| 498 |
|
| 499 |
with gr.Blocks(title="RynnValue-4B") as demo:
|
|
|
|
| 501 |
|
| 502 |
with gr.Row():
|
| 503 |
with gr.Column(scale=1):
|
| 504 |
+
video_in = gr.Video(label="Manipulation video", sources=["upload"], height=300)
|
| 505 |
instruction = gr.Textbox(
|
| 506 |
label="Task instruction",
|
| 507 |
placeholder="Put the box in the drawer and close it",
|
|
|
|
| 522 |
allow_custom_value=True,
|
| 523 |
)
|
| 524 |
with gr.Column(scale=1):
|
| 525 |
+
video_out = gr.Video(label="Value trend", height=340, autoplay=True)
|
| 526 |
analysis_md = gr.Markdown(label="Analysis")
|
| 527 |
|
| 528 |
with gr.Accordion("Advanced options", open=False):
|
| 529 |
gr.Markdown(
|
| 530 |
+
"The video is uniformly resampled to *N* frames; the model emits one remaining-time "
|
| 531 |
+
"prediction per frame, so **frames sampled** is also the resolution of the curve. "
|
| 532 |
+
"More frames / larger frames give a finer, better-grounded curve and a longer run."
|
| 533 |
)
|
| 534 |
num_frames = gr.Slider(
|
| 535 |
+
16, 64, value=DEFAULT_NUM_FRAMES, step=8, label="Frames sampled from the video"
|
|
|
|
|
|
|
|
|
|
| 536 |
)
|
| 537 |
max_image_side = gr.Slider(
|
| 538 |
+
256, 512, value=DEFAULT_MAX_IMAGE_SIDE, step=64, label="Max frame side fed to the model"
|
| 539 |
)
|
| 540 |
max_new_tokens = gr.Slider(
|
| 541 |
32, 256, value=DEFAULT_MAX_NEW_TOKENS, step=32, label="Analysis token budget"
|
|
|
|
| 547 |
robot_desc,
|
| 548 |
camera_desc,
|
| 549 |
num_frames,
|
|
|
|
| 550 |
max_image_side,
|
| 551 |
max_new_tokens,
|
| 552 |
]
|