multimodalart HF Staff commited on
Commit
47d431f
·
verified ·
1 Parent(s): bbcdab3

single-pass per-frame temporal-distance readout

Browse files
Files changed (1) hide show
  1. app.py +114 -188
app.py CHANGED
@@ -1,17 +1,18 @@
1
  """RynnValue-4B — robot-manipulation value model demo.
2
 
3
  Given a manipulation video and the task instruction, RynnValue predicts the
4
- *temporal distance* (remaining seconds until the task is done) for a sequence of
5
- video prefixes, plus a short natural-language analysis (description / does the
6
- video match the instruction / did it succeed).
7
-
8
- The inference path mirrors the authors' reference script
9
- (`rynn_infer/inference.py` in github.com/alibaba-damo-academy/RynnValue):
10
- prefix-uniform sampling for each evaluated step `i` the prefix `frames[0:i+1]`
11
- is resampled to `num_frames` with `np.linspace` and the value of the *last*
12
- prediction slot is read out, so every score only conditions on frames seen so
13
- far followed by one `generate()` pass on the full-video prefix for the
14
- Analysis block, and the same side-by-side "video + trend plot" rendering.
 
15
  """
16
 
17
  import os
@@ -65,13 +66,11 @@ EOS_TOKEN_ID = tokenizer.convert_tokens_to_ids("<|im_end|>")
65
 
66
  DEFAULT_ROBOT = "a Franka single-arm robot"
67
  DEFAULT_CAMERA = "the main camera"
68
- DEFAULT_NUM_FRAMES = 32
69
- DEFAULT_NUM_STEPS = 24
70
  DEFAULT_MAX_IMAGE_SIDE = 384
71
  DEFAULT_MAX_NEW_TOKENS = 128
72
- BATCH_SIZE = 2
73
 
74
- MAX_DECODE_FRAMES = 900 # hard cap on decoded frames (memory guard)
75
  RENDER_MAX_SIDE = 640 # frames are downscaled to this for the rendered video
76
 
77
  ROBOT_CHOICES = [
@@ -100,11 +99,11 @@ CAMERA_CHOICES = [
100
 
101
 
102
  def load_video_frames(video_path):
103
- """Decode a video to RGB PIL frames, capped at MAX_DECODE_FRAMES.
104
 
105
  Returns (frames, effective_fps). Long videos are decoded with a stride so
106
- memory stays bounded; the effective fps is scaled to match, which keeps the
107
- wall-clock timeline (and therefore the seconds axis) correct.
108
  """
109
  if not video_path or not os.path.isfile(video_path):
110
  raise gr.Error("Please provide a video file.")
@@ -118,8 +117,8 @@ def load_video_frames(video_path):
118
  estimated = int(duration * fps) if duration and np.isfinite(duration) else None
119
 
120
  stride = 1
121
- if estimated and estimated > MAX_DECODE_FRAMES:
122
- stride = int(np.ceil(estimated / MAX_DECODE_FRAMES))
123
 
124
  frames = []
125
  try:
@@ -132,7 +131,7 @@ def load_video_frames(video_path):
132
  s = RENDER_MAX_SIDE / max(w, h)
133
  img = img.resize((max(1, round(w * s)), max(1, round(h * s))), Image.BICUBIC)
134
  frames.append(img)
135
- if len(frames) >= MAX_DECODE_FRAMES:
136
  break
137
  finally:
138
  reader.close()
@@ -154,16 +153,6 @@ def resize_frames(frames, max_side):
154
  return [f.resize(new_size, resample=Image.BICUBIC) for f in frames]
155
 
156
 
157
- def sample_frame_indices(total, num_frames):
158
- """Uniformly pick ``num_frames`` indices out of ``total`` frames."""
159
- if num_frames <= 0 or num_frames >= total:
160
- return list(range(total))
161
- if num_frames == 1:
162
- return [total - 1]
163
- step = (total - 1) / (num_frames - 1)
164
- return [int(round(j * step)) for j in range(num_frames)]
165
-
166
-
167
  # ---------------------------------------------------------------------------
168
  # Analysis parsing (same regexes as the reference script)
169
  # ---------------------------------------------------------------------------
@@ -197,12 +186,12 @@ def _format_time(seconds):
197
  return f"{minutes:02d}:{secs:02d}.{millis:03d}"
198
 
199
 
200
- def _render_trend_frames(values, sampled_indices, fps, size, title, task_title):
201
  """Render one plot image per prediction step.
202
 
203
- The plot only changes at the sampled indices, so we render
204
- ``len(values)`` images (a couple of dozen) and reuse them across the video
205
- instead of re-rendering a figure for every frame.
206
  """
207
  w, h = size
208
  dpi = 100
@@ -210,12 +199,13 @@ def _render_trend_frames(values, sampled_indices, fps, size, title, task_title):
210
 
211
  x = np.asarray(sampled_indices, dtype=float)
212
  y = np.asarray(values, dtype=float)
213
- remaining = (x[-1] - x) / float(fps)
214
 
215
- (line,) = ax1.plot([], [], color="tab:blue", linewidth=2.0, label="predicted value")
216
- pt1 = ax1.scatter([x[0]], [y[0]], color="red", s=30, zorder=3, label="current value")
 
217
  ax1.set_xlabel("Frame", fontsize=9)
218
- ax1.set_ylabel("Value", color="tab:blue", fontsize=9)
219
  ax1.tick_params(axis="x", labelsize=8)
220
  ax1.tick_params(axis="y", labelcolor="tab:blue", labelsize=8)
221
  ax1.grid(True, alpha=0.3)
@@ -229,13 +219,11 @@ def _render_trend_frames(values, sampled_indices, fps, size, title, task_title):
229
  ax1.set_ylim(y_min - margin, y_max + margin)
230
 
231
  ax2 = ax1.twinx()
232
- ax2.plot(x, remaining, color="green", linestyle="--", linewidth=2.0, label="remaining time")
233
- pt2 = ax2.scatter(
234
- [x[0]], [remaining[0]], color="green", s=25, zorder=3, label="current remaining"
235
- )
236
- ax2.set_ylabel("Remaining Time (s)", color="green", fontsize=9)
237
  ax2.tick_params(axis="y", labelcolor="green", labelsize=8)
238
- rt_min, rt_max = float(np.min(remaining)), float(np.max(remaining))
239
  if rt_min == rt_max:
240
  rt_min -= 1.0
241
  rt_max += 1.0
@@ -243,7 +231,7 @@ def _render_trend_frames(values, sampled_indices, fps, size, title, task_title):
243
  ax2.set_ylim(rt_min - rt_margin, rt_max + rt_margin)
244
 
245
  short_task = task_title if len(task_title) <= 42 else task_title[:39] + "..."
246
- ax1.set_title(f"{short_task}\n{title}", fontsize=10)
247
 
248
  lines1, labels1 = ax1.get_legend_handles_labels()
249
  lines2, labels2 = ax2.get_legend_handles_labels()
@@ -253,10 +241,9 @@ def _render_trend_frames(values, sampled_indices, fps, size, title, task_title):
253
  for i in range(len(y)):
254
  line.set_data(x[: i + 1], y[: i + 1])
255
  pt1.set_offsets(np.array([[x[i], y[i]]]))
256
- pt2.set_offsets(np.array([[x[i], remaining[i]]]))
257
  fig.canvas.draw()
258
- buf = np.asarray(fig.canvas.buffer_rgba())[:, :, :3].copy()
259
- images.append(Image.fromarray(buf))
260
  plt.close(fig)
261
  return images
262
 
@@ -271,13 +258,13 @@ def _overlay_font(size):
271
  def _draw_overlay_text(img, lines, font):
272
  img = img.copy()
273
  draw = ImageDraw.Draw(img)
274
- x, y = 8, 8
275
  for line in lines:
276
  draw.text(
277
  (x, y), line, fill=(255, 235, 59), font=font, stroke_width=2, stroke_fill=(0, 0, 0)
278
  )
279
  bbox = draw.textbbox((x, y), line, font=font)
280
- y += (bbox[3] - bbox[1]) + 8
281
  return img
282
 
283
 
@@ -293,27 +280,23 @@ def save_video_with_trend(frames, values, sampled_indices, fps, output_path, tas
293
  base_w, base_h = int(round(base_w * scale)), 260
294
  frames = [f.resize((base_w, base_h)) for f in frames]
295
 
296
- plot_w = int(min(520, max(300, base_w * 0.75)))
297
  plot_h = max(260, base_h)
298
 
299
- plots = _render_trend_frames(
300
- values,
301
- sampled_indices,
302
- fps,
303
- (plot_w, plot_h),
304
- "Remaining Time (s)",
305
- task_title,
306
- )
307
- if plots[0].height != base_h:
308
- plots = [p.resize((plot_w, base_h)) for p in plots]
309
 
310
  idx_to_pos = {idx: pos for pos, idx in enumerate(sampled_indices)}
311
  n = len(frames)
312
- remaining_all = (n - 1 - np.arange(n)) / float(fps)
313
 
314
  font = _overlay_font(16)
315
  canvas_w = _ceil_to_multiple(base_w + plot_w, 16)
316
  canvas_h = _ceil_to_multiple(base_h, 16)
 
317
 
318
  writer = imageio.get_writer(
319
  output_path,
@@ -321,10 +304,11 @@ def save_video_with_trend(frames, values, sampled_indices, fps, output_path, tas
321
  codec="libx264",
322
  quality=7,
323
  macro_block_size=None,
 
324
  )
325
  try:
326
  pos = 0
327
- short_task = task_title if len(task_title) <= 48 else task_title[:45] + "..."
328
  for i, img in enumerate(frames):
329
  if i in idx_to_pos:
330
  pos = idx_to_pos[i]
@@ -333,14 +317,13 @@ def save_video_with_trend(frames, values, sampled_indices, fps, output_path, tas
333
  [
334
  f"task: {short_task}",
335
  f"predicted remaining: {values[pos]:.2f}s",
336
- f"actual remaining: {_format_time(remaining_all[i])}",
337
  ],
338
  font,
339
  )
340
- canvas = Image.new("RGB", (canvas_w, canvas_h), (255, 255, 255))
341
- canvas.paste(left, (0, 0))
342
- canvas.paste(plots[pos], (base_w, 0))
343
- writer.append_data(np.asarray(canvas))
344
  finally:
345
  writer.close()
346
  return output_path
@@ -357,7 +340,6 @@ def _gpu_duration(
357
  robot_description=DEFAULT_ROBOT,
358
  camera_description=DEFAULT_CAMERA,
359
  num_frames=DEFAULT_NUM_FRAMES,
360
- num_steps=DEFAULT_NUM_STEPS,
361
  max_image_side=DEFAULT_MAX_IMAGE_SIDE,
362
  max_new_tokens=DEFAULT_MAX_NEW_TOKENS,
363
  *args,
@@ -365,13 +347,14 @@ def _gpu_duration(
365
  ):
366
  try:
367
  nf = int(num_frames)
368
- ns = int(num_steps)
369
  side = int(max_image_side)
370
  except (TypeError, ValueError):
371
- nf, ns, side = DEFAULT_NUM_FRAMES, DEFAULT_NUM_STEPS, DEFAULT_MAX_IMAGE_SIDE
372
- # base (load + render) + per prefix-forward cost, scaled by tokens per frame
373
- work = ns * nf * (side / 384.0) ** 2
374
- return int(min(300, 45 + 0.09 * work))
 
 
375
 
376
 
377
  @spaces.GPU(duration=_gpu_duration)
@@ -381,7 +364,6 @@ def analyze(
381
  robot_description=DEFAULT_ROBOT,
382
  camera_description=DEFAULT_CAMERA,
383
  num_frames=DEFAULT_NUM_FRAMES,
384
- num_steps=DEFAULT_NUM_STEPS,
385
  max_image_side=DEFAULT_MAX_IMAGE_SIDE,
386
  max_new_tokens=DEFAULT_MAX_NEW_TOKENS,
387
  progress=gr.Progress(),
@@ -399,104 +381,52 @@ def analyze(
399
  robot_description = DEFAULT_ROBOT
400
 
401
  num_frames = int(num_frames)
402
- num_steps = int(num_steps)
403
  max_image_side = int(max_image_side)
404
  max_new_tokens = int(max_new_tokens)
405
 
406
- progress(0.02, desc="Decoding video...")
407
  frames, fps = load_video_frames(video)
408
  total = len(frames)
409
  model_images = resize_frames(frames, max_image_side)
410
 
411
- eval_indices = sample_frame_indices(total, min(num_steps, total))
412
-
413
- def build_prefix_sample(end_idx):
414
- frame_idx = np.linspace(0, end_idx, num_frames, dtype=int)
415
- prefix_images = [model_images[j] for j in frame_idx]
416
- return processor.process_episode(
417
- instruction=instruction,
418
- images=prefix_images,
419
- robot_description=robot_description,
420
- camera_description=camera_description,
421
- )
422
-
423
- def run_batch(samples):
424
- """One forward pass over a batch of prefixes -> last-slot value each."""
425
- batch_kwargs = dict(
426
- input_ids=torch.cat([s["input_ids"] for s in samples], dim=0).to("cuda").long(),
427
- attention_mask=torch.cat([s["attention_mask"] for s in samples], dim=0)
428
- .to("cuda")
429
- .long(),
430
- pixel_values=torch.cat([s["pixel_values"].flatten(0, 1) for s in samples], dim=0)
431
- .to("cuda")
432
- .to(torch.bfloat16),
433
- image_grid_thw=torch.cat([s["image_grid_thw"].flatten(0, 1) for s in samples], dim=0)
434
- .to("cuda")
435
- .long(),
436
- )
437
- with torch.inference_mode():
438
- outputs = model(**batch_kwargs)
439
- pred = outputs.value.pred_value
440
- if pred.dim() == 2 and pred.shape[0] == 1:
441
- pred = pred.reshape(len(samples), -1)
442
- if pred.dim() == 3:
443
- pred = pred.mean(dim=0)
444
- if pred.dim() == 2 and pred.shape[-1] > 1:
445
- pred = pred[:, -1]
446
- elif pred.dim() == 2:
447
- pred = pred[:, 0]
448
- return pred.float().reshape(-1).tolist()
449
-
450
- pred_value = []
451
- final_sample = None
452
- batch = []
453
- for step, end_idx in enumerate(eval_indices):
454
- sample = build_prefix_sample(end_idx)
455
- if end_idx == eval_indices[-1]:
456
- final_sample = sample
457
- batch.append(sample)
458
- if len(batch) >= BATCH_SIZE or step == len(eval_indices) - 1:
459
- pred_value.extend(run_batch(batch))
460
- batch = []
461
- progress(
462
- 0.05 + 0.75 * len(pred_value) / len(eval_indices),
463
- desc=f"Value pass {len(pred_value)}/{len(eval_indices)}",
464
- )
465
 
466
- # --- DEBUG: single-pass per-frame readout (model card "Programmatic Usage") ---
467
- dbg_idx = np.linspace(0, total - 1, num_frames, dtype=int)
468
- dbg_sample = processor.process_episode(
469
  instruction=instruction,
470
- images=[model_images[j] for j in dbg_idx],
471
  robot_description=robot_description,
472
  camera_description=camera_description,
473
  )
 
 
 
 
 
 
474
  with torch.inference_mode():
475
- dbg_out = model(
476
- input_ids=dbg_sample["input_ids"].to("cuda").long(),
477
- attention_mask=dbg_sample["attention_mask"].to("cuda").long(),
478
- pixel_values=dbg_sample["pixel_values"].flatten(0, 1).to("cuda"),
479
- image_grid_thw=dbg_sample["image_grid_thw"].flatten(0, 1).to("cuda").long(),
480
  )
481
- dbg_pred = dbg_out.value.pred_value.float().reshape(-1).tolist()
482
- dbg_rel = dbg_out.relative.pred_value.float().reshape(-1).tolist()
483
- debug_lines = [
484
- "```",
485
- "SINGLE-PASS values: " + ", ".join(f"{v:.2f}" for v in dbg_pred),
486
- "SINGLE-PASS deltas: " + ", ".join(f"{v:.2f}" for v in dbg_rel),
487
- "PREFIX last-slot : " + ", ".join(f"{v:.2f}" for v in pred_value),
488
- f"fps={fps:.2f} total={total}",
489
- "```",
490
- ]
491
-
492
- progress(0.82, desc="Generating analysis...")
493
- input_ids = final_sample["input_ids"].to("cuda").long()
494
  with torch.inference_mode():
495
  gen_out = model.generate(
496
  input_ids=input_ids,
497
- attention_mask=final_sample["attention_mask"].to("cuda").long(),
498
- pixel_values=final_sample["pixel_values"].flatten(0, 1).to("cuda").to(torch.bfloat16),
499
- image_grid_thw=final_sample["image_grid_thw"].flatten(0, 1).to("cuda").long(),
500
  max_new_tokens=max_new_tokens,
501
  do_sample=False,
502
  num_beams=1,
@@ -508,32 +438,32 @@ def analyze(
508
  analysis = parse_analysis(analysis_text)
509
  gpu_seconds = time.time() - started
510
 
511
- progress(0.88, desc="Rendering trend video...")
512
  out_path = os.path.join(tempfile.mkdtemp(), "rynnvalue_trend.mp4")
513
- save_video_with_trend(frames, pred_value, eval_indices, fps, out_path, instruction)
514
 
515
  def _verdict(v):
516
  if v is None:
517
  return "—"
518
  return "✅ Yes" if v.lower() == "yes" else "❌ No"
519
 
 
 
520
  description = analysis["description"] or (analysis_text.strip() or "—")
521
  summary = "\n".join(
522
  [
523
- f"**Task instruction** — {instruction}",
524
- "",
525
  f"**Video description** — {description}",
526
  "",
527
  f"**Matches the instruction?** {_verdict(analysis['match'])} &nbsp;&nbsp;·&nbsp;&nbsp; "
528
  f"**Task completed?** {_verdict(analysis['success'])}",
529
  "",
530
- f"**Predicted remaining time** — {pred_value[0]:.2f} s at the first frame → "
531
- f"{pred_value[-1]:.2f} s at the last frame "
532
- f"(video is {(total - 1) / fps:.1f} s long, {total} frames decoded).",
533
  "",
534
- "\n".join(debug_lines),
 
535
  "",
536
- f"<sub>{len(eval_indices)} prefix evaluations · {num_frames} frames per prefix · "
537
  f"{gpu_seconds:.1f} s on GPU</sub>",
538
  ]
539
  )
@@ -549,21 +479,21 @@ DESCRIPTION = """
549
 
550
  # 🤖 RynnValue-4B — how much longer until the robot is done?
551
 
552
- [Model](https://huggingface.co/Alibaba-DAMO-Academy/RynnValue-4B) ·
553
- [Paper](https://arxiv.org/abs/2608.09853) ·
554
- [GitHub](https://github.com/alibaba-damo-academy/RynnValue) ·
555
- [Project page](https://alibaba-damo-academy.github.io/RynnValue.github.io/)
556
 
557
  </div>
558
 
559
- **RynnValue** is a value foundation model for robot manipulation, built on RynnBrain (Qwen3-VL).
560
- Given a manipulation video *and* a task instruction, it predicts the **temporal distance**
561
- the remaining time in seconds until the task is complete for a series of video prefixes, and
562
- generates a short analysis: what the video shows, whether it **matches** the instruction, and
563
- whether the task **succeeded**.
564
 
565
- The output video plays the clip next to the predicted value curve (blue) alongside the actual
566
- remaining time of the clip (dashed green).
567
  """
568
 
569
  with gr.Blocks(title="RynnValue-4B") as demo:
@@ -571,7 +501,7 @@ with gr.Blocks(title="RynnValue-4B") as demo:
571
 
572
  with gr.Row():
573
  with gr.Column(scale=1):
574
- video_in = gr.Video(label="Manipulation video", sources=["upload"], height=320)
575
  instruction = gr.Textbox(
576
  label="Task instruction",
577
  placeholder="Put the box in the drawer and close it",
@@ -592,23 +522,20 @@ with gr.Blocks(title="RynnValue-4B") as demo:
592
  allow_custom_value=True,
593
  )
594
  with gr.Column(scale=1):
595
- video_out = gr.Video(label="Value trend", height=360, autoplay=True)
596
  analysis_md = gr.Markdown(label="Analysis")
597
 
598
  with gr.Accordion("Advanced options", open=False):
599
  gr.Markdown(
600
- "The model is queried once per curve point, on the prefix of the video seen so far "
601
- "(the authors' `rynn_infer/inference.py` protocol). More curve points and more frames "
602
- "per prefix mean a smoother, better-conditioned curve and a longer run."
603
  )
604
  num_frames = gr.Slider(
605
- 8, 64, value=DEFAULT_NUM_FRAMES, step=8, label="Frames per prefix (context length)"
606
- )
607
- num_steps = gr.Slider(
608
- 8, 48, value=DEFAULT_NUM_STEPS, step=4, label="Curve points (prefix evaluations)"
609
  )
610
  max_image_side = gr.Slider(
611
- 256, 640, value=DEFAULT_MAX_IMAGE_SIDE, step=64, label="Max frame side fed to the model"
612
  )
613
  max_new_tokens = gr.Slider(
614
  32, 256, value=DEFAULT_MAX_NEW_TOKENS, step=32, label="Analysis token budget"
@@ -620,7 +547,6 @@ with gr.Blocks(title="RynnValue-4B") as demo:
620
  robot_desc,
621
  camera_desc,
622
  num_frames,
623
- num_steps,
624
  max_image_side,
625
  max_new_tokens,
626
  ]
 
1
  """RynnValue-4B — robot-manipulation value model demo.
2
 
3
  Given a manipulation video and the task instruction, RynnValue predicts the
4
+ *temporal distance* (remaining seconds until the task is done) for every
5
+ sampled frame, the signed time delta between consecutive frames, and a short
6
+ natural-language analysis (description / does the video match the instruction /
7
+ did it succeed).
8
+
9
+ Inference follows the model card's `process_episode` protocol: the video is
10
+ uniformly resampled to `num_frames` frames, the episode prompt interleaves each
11
+ frame with its `<value>` / `<relative_value>` query group, and one forward pass
12
+ yields one remaining-time prediction per frame (the LM is causal, so the value
13
+ at frame *i* only conditions on frames 0..i). A second `generate()` pass on the
14
+ same prompt produces the Analysis block. Rendering reproduces the authors'
15
+ `rynn_infer/plot_utils.py` layout: the clip next to a synchronized value trend.
16
  """
17
 
18
  import os
 
66
 
67
  DEFAULT_ROBOT = "a Franka single-arm robot"
68
  DEFAULT_CAMERA = "the main camera"
69
+ DEFAULT_NUM_FRAMES = 48
 
70
  DEFAULT_MAX_IMAGE_SIDE = 384
71
  DEFAULT_MAX_NEW_TOKENS = 128
 
72
 
73
+ MAX_FRAMES = 450 # decoded/rendered frame cap (memory + render-time guard)
74
  RENDER_MAX_SIDE = 640 # frames are downscaled to this for the rendered video
75
 
76
  ROBOT_CHOICES = [
 
99
 
100
 
101
  def load_video_frames(video_path):
102
+ """Decode a video to RGB PIL frames, capped at MAX_FRAMES.
103
 
104
  Returns (frames, effective_fps). Long videos are decoded with a stride so
105
+ memory and render time stay bounded; the effective fps is scaled to match,
106
+ which keeps the wall-clock timeline (and the seconds axis) correct.
107
  """
108
  if not video_path or not os.path.isfile(video_path):
109
  raise gr.Error("Please provide a video file.")
 
117
  estimated = int(duration * fps) if duration and np.isfinite(duration) else None
118
 
119
  stride = 1
120
+ if estimated and estimated > MAX_FRAMES:
121
+ stride = int(np.ceil(estimated / MAX_FRAMES))
122
 
123
  frames = []
124
  try:
 
131
  s = RENDER_MAX_SIDE / max(w, h)
132
  img = img.resize((max(1, round(w * s)), max(1, round(h * s))), Image.BICUBIC)
133
  frames.append(img)
134
+ if len(frames) >= MAX_FRAMES:
135
  break
136
  finally:
137
  reader.close()
 
153
  return [f.resize(new_size, resample=Image.BICUBIC) for f in frames]
154
 
155
 
 
 
 
 
 
 
 
 
 
 
156
  # ---------------------------------------------------------------------------
157
  # Analysis parsing (same regexes as the reference script)
158
  # ---------------------------------------------------------------------------
 
186
  return f"{minutes:02d}:{secs:02d}.{millis:03d}"
187
 
188
 
189
+ def _render_trend_frames(values, sampled_indices, fps, size, task_title):
190
  """Render one plot image per prediction step.
191
 
192
+ The plot only changes at the sampled indices, so we render ``len(values)``
193
+ images and reuse them across the video instead of re-rendering a figure for
194
+ every video frame.
195
  """
196
  w, h = size
197
  dpi = 100
 
199
 
200
  x = np.asarray(sampled_indices, dtype=float)
201
  y = np.asarray(values, dtype=float)
202
+ actual = (x[-1] - x) / float(fps)
203
 
204
+ ax1.plot(x, y, color="tab:blue", linewidth=1.0, alpha=0.25)
205
+ (line,) = ax1.plot([], [], color="tab:blue", linewidth=2.0, label="predicted remaining")
206
+ pt1 = ax1.scatter([x[0]], [y[0]], color="red", s=32, zorder=3, label="current prediction")
207
  ax1.set_xlabel("Frame", fontsize=9)
208
+ ax1.set_ylabel("Predicted value (s)", color="tab:blue", fontsize=9)
209
  ax1.tick_params(axis="x", labelsize=8)
210
  ax1.tick_params(axis="y", labelcolor="tab:blue", labelsize=8)
211
  ax1.grid(True, alpha=0.3)
 
219
  ax1.set_ylim(y_min - margin, y_max + margin)
220
 
221
  ax2 = ax1.twinx()
222
+ ax2.plot(x, actual, color="green", linestyle="--", linewidth=2.0, label="actual remaining")
223
+ pt2 = ax2.scatter([x[0]], [actual[0]], color="green", s=26, zorder=3)
224
+ ax2.set_ylabel("Actual remaining (s)", color="green", fontsize=9)
 
 
225
  ax2.tick_params(axis="y", labelcolor="green", labelsize=8)
226
+ rt_min, rt_max = float(np.min(actual)), float(np.max(actual))
227
  if rt_min == rt_max:
228
  rt_min -= 1.0
229
  rt_max += 1.0
 
231
  ax2.set_ylim(rt_min - rt_margin, rt_max + rt_margin)
232
 
233
  short_task = task_title if len(task_title) <= 42 else task_title[:39] + "..."
234
+ ax1.set_title(f"{short_task}\nTemporal distance to completion", fontsize=10)
235
 
236
  lines1, labels1 = ax1.get_legend_handles_labels()
237
  lines2, labels2 = ax2.get_legend_handles_labels()
 
241
  for i in range(len(y)):
242
  line.set_data(x[: i + 1], y[: i + 1])
243
  pt1.set_offsets(np.array([[x[i], y[i]]]))
244
+ pt2.set_offsets(np.array([[x[i], actual[i]]]))
245
  fig.canvas.draw()
246
+ images.append(np.asarray(fig.canvas.buffer_rgba())[:, :, :3].copy())
 
247
  plt.close(fig)
248
  return images
249
 
 
258
  def _draw_overlay_text(img, lines, font):
259
  img = img.copy()
260
  draw = ImageDraw.Draw(img)
261
+ x, y = 8, 6
262
  for line in lines:
263
  draw.text(
264
  (x, y), line, fill=(255, 235, 59), font=font, stroke_width=2, stroke_fill=(0, 0, 0)
265
  )
266
  bbox = draw.textbbox((x, y), line, font=font)
267
+ y += (bbox[3] - bbox[1]) + 9
268
  return img
269
 
270
 
 
280
  base_w, base_h = int(round(base_w * scale)), 260
281
  frames = [f.resize((base_w, base_h)) for f in frames]
282
 
283
+ plot_w = int(min(560, max(320, base_w * 0.8)))
284
  plot_h = max(260, base_h)
285
 
286
+ plots = _render_trend_frames(values, sampled_indices, fps, (plot_w, plot_h), task_title)
287
+ if plots[0].shape[0] != base_h:
288
+ plots = [
289
+ np.asarray(Image.fromarray(p).resize((plot_w, base_h))) for p in plots
290
+ ]
 
 
 
 
 
291
 
292
  idx_to_pos = {idx: pos for pos, idx in enumerate(sampled_indices)}
293
  n = len(frames)
294
+ actual_all = (n - 1 - np.arange(n)) / float(fps)
295
 
296
  font = _overlay_font(16)
297
  canvas_w = _ceil_to_multiple(base_w + plot_w, 16)
298
  canvas_h = _ceil_to_multiple(base_h, 16)
299
+ canvas = np.full((canvas_h, canvas_w, 3), 255, dtype=np.uint8)
300
 
301
  writer = imageio.get_writer(
302
  output_path,
 
304
  codec="libx264",
305
  quality=7,
306
  macro_block_size=None,
307
+ output_params=["-pix_fmt", "yuv420p"],
308
  )
309
  try:
310
  pos = 0
311
+ short_task = task_title if len(task_title) <= 44 else task_title[:41] + "..."
312
  for i, img in enumerate(frames):
313
  if i in idx_to_pos:
314
  pos = idx_to_pos[i]
 
317
  [
318
  f"task: {short_task}",
319
  f"predicted remaining: {values[pos]:.2f}s",
320
+ f"actual remaining: {_format_time(actual_all[i])}",
321
  ],
322
  font,
323
  )
324
+ canvas[:base_h, :base_w] = np.asarray(left)
325
+ canvas[:base_h, base_w : base_w + plot_w] = plots[pos]
326
+ writer.append_data(canvas)
 
327
  finally:
328
  writer.close()
329
  return output_path
 
340
  robot_description=DEFAULT_ROBOT,
341
  camera_description=DEFAULT_CAMERA,
342
  num_frames=DEFAULT_NUM_FRAMES,
 
343
  max_image_side=DEFAULT_MAX_IMAGE_SIDE,
344
  max_new_tokens=DEFAULT_MAX_NEW_TOKENS,
345
  *args,
 
347
  ):
348
  try:
349
  nf = int(num_frames)
 
350
  side = int(max_image_side)
351
  except (TypeError, ValueError):
352
+ nf, side = DEFAULT_NUM_FRAMES, DEFAULT_MAX_IMAGE_SIDE
353
+ # Sequence length scales with frames x tokens-per-frame, and the eager
354
+ # value-isolation attention makes the forward grow ~quadratically in it.
355
+ # Calibrated against a measured 32-frame @384px run.
356
+ n = nf * (side / 384.0) ** 2
357
+ return int(min(240, 20 + 0.6 * n + 0.012 * n**2))
358
 
359
 
360
  @spaces.GPU(duration=_gpu_duration)
 
364
  robot_description=DEFAULT_ROBOT,
365
  camera_description=DEFAULT_CAMERA,
366
  num_frames=DEFAULT_NUM_FRAMES,
 
367
  max_image_side=DEFAULT_MAX_IMAGE_SIDE,
368
  max_new_tokens=DEFAULT_MAX_NEW_TOKENS,
369
  progress=gr.Progress(),
 
381
  robot_description = DEFAULT_ROBOT
382
 
383
  num_frames = int(num_frames)
 
384
  max_image_side = int(max_image_side)
385
  max_new_tokens = int(max_new_tokens)
386
 
387
+ progress(0.03, desc="Decoding video...")
388
  frames, fps = load_video_frames(video)
389
  total = len(frames)
390
  model_images = resize_frames(frames, max_image_side)
391
 
392
+ num_frames = max(2, min(num_frames, total))
393
+ sampled_indices = np.linspace(0, total - 1, num_frames, dtype=int).tolist()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
394
 
395
+ progress(0.18, desc="Building episode prompt...")
396
+ sample = processor.process_episode(
 
397
  instruction=instruction,
398
+ images=[model_images[j] for j in sampled_indices],
399
  robot_description=robot_description,
400
  camera_description=camera_description,
401
  )
402
+ input_ids = sample["input_ids"].to("cuda").long()
403
+ attention_mask = sample["attention_mask"].to("cuda").long()
404
+ pixel_values = sample["pixel_values"].flatten(0, 1).to("cuda")
405
+ image_grid_thw = sample["image_grid_thw"].flatten(0, 1).to("cuda").long()
406
+
407
+ progress(0.28, desc="Predicting temporal distance...")
408
  with torch.inference_mode():
409
+ outputs = model(
410
+ input_ids=input_ids,
411
+ attention_mask=attention_mask,
412
+ pixel_values=pixel_values,
413
+ image_grid_thw=image_grid_thw,
414
  )
415
+ # (num_value_heads, num_frames) -> one remaining-time prediction per frame
416
+ pred_value = outputs.value.pred_value.float()
417
+ if pred_value.dim() > 1:
418
+ pred_value = pred_value.reshape(-1, len(sampled_indices)).mean(dim=0)
419
+ pred_value = pred_value.reshape(-1).tolist()
420
+ # relative head: signed time delta between consecutive sampled frames
421
+ rel_value = outputs.relative.pred_value.float().reshape(-1).tolist()
422
+
423
+ progress(0.6, desc="Generating analysis...")
 
 
 
 
424
  with torch.inference_mode():
425
  gen_out = model.generate(
426
  input_ids=input_ids,
427
+ attention_mask=attention_mask,
428
+ pixel_values=pixel_values,
429
+ image_grid_thw=image_grid_thw,
430
  max_new_tokens=max_new_tokens,
431
  do_sample=False,
432
  num_beams=1,
 
438
  analysis = parse_analysis(analysis_text)
439
  gpu_seconds = time.time() - started
440
 
441
+ progress(0.85, desc="Rendering trend video...")
442
  out_path = os.path.join(tempfile.mkdtemp(), "rynnvalue_trend.mp4")
443
+ save_video_with_trend(frames, pred_value, sampled_indices, fps, out_path, instruction)
444
 
445
  def _verdict(v):
446
  if v is None:
447
  return "—"
448
  return "✅ Yes" if v.lower() == "yes" else "❌ No"
449
 
450
+ clip_seconds = (total - 1) / fps
451
+ predicted_span = float(np.sum(rel_value)) if rel_value else float("nan")
452
  description = analysis["description"] or (analysis_text.strip() or "—")
453
  summary = "\n".join(
454
  [
 
 
455
  f"**Video description** — {description}",
456
  "",
457
  f"**Matches the instruction?** {_verdict(analysis['match'])} &nbsp;&nbsp;·&nbsp;&nbsp; "
458
  f"**Task completed?** {_verdict(analysis['success'])}",
459
  "",
460
+ f"**Temporal distance** — {pred_value[0]:.2f} s remaining at the first sampled frame → "
461
+ f"{pred_value[-1]:.2f} s at the last. The clip itself is {clip_seconds:.1f} s long.",
 
462
  "",
463
+ f"**Relative head** — the predicted time deltas between the {num_frames} sampled "
464
+ f"frames sum to {predicted_span:.1f} s (actual clip length {clip_seconds:.1f} s).",
465
  "",
466
+ f"<sub>{num_frames} frames · longest side {max_image_side}px · "
467
  f"{gpu_seconds:.1f} s on GPU</sub>",
468
  ]
469
  )
 
479
 
480
  # 🤖 RynnValue-4B — how much longer until the robot is done?
481
 
482
+ <a href="https://huggingface.co/Alibaba-DAMO-Academy/RynnValue-4B">Model</a> ·
483
+ <a href="https://arxiv.org/abs/2608.09853">Paper</a> ·
484
+ <a href="https://github.com/alibaba-damo-academy/RynnValue">GitHub</a> ·
485
+ <a href="https://alibaba-damo-academy.github.io/RynnValue.github.io/">Project page</a>
486
 
487
  </div>
488
 
489
+ **RynnValue** is a value foundation model for robot manipulation, built on RynnBrain (Qwen3-VL) and
490
+ trained on 7,000+ hours of embodied data. Give it a manipulation video **and** the task instruction,
491
+ and for every sampled frame it predicts the **temporal distance**the remaining time in seconds
492
+ until the task is complete — plus a short analysis: what the video shows, whether it **matches** the
493
+ instruction, and whether the task **succeeded**.
494
 
495
+ The result video plays the clip beside the predicted value curve (blue), with the clip's actual
496
+ remaining time (dashed green) for reference.
497
  """
498
 
499
  with gr.Blocks(title="RynnValue-4B") as demo:
 
501
 
502
  with gr.Row():
503
  with gr.Column(scale=1):
504
+ video_in = gr.Video(label="Manipulation video", sources=["upload"], height=300)
505
  instruction = gr.Textbox(
506
  label="Task instruction",
507
  placeholder="Put the box in the drawer and close it",
 
522
  allow_custom_value=True,
523
  )
524
  with gr.Column(scale=1):
525
+ video_out = gr.Video(label="Value trend", height=340, autoplay=True)
526
  analysis_md = gr.Markdown(label="Analysis")
527
 
528
  with gr.Accordion("Advanced options", open=False):
529
  gr.Markdown(
530
+ "The video is uniformly resampled to *N* frames; the model emits one remaining-time "
531
+ "prediction per frame, so **frames sampled** is also the resolution of the curve. "
532
+ "More frames / larger frames give a finer, better-grounded curve and a longer run."
533
  )
534
  num_frames = gr.Slider(
535
+ 16, 64, value=DEFAULT_NUM_FRAMES, step=8, label="Frames sampled from the video"
 
 
 
536
  )
537
  max_image_side = gr.Slider(
538
+ 256, 512, value=DEFAULT_MAX_IMAGE_SIDE, step=64, label="Max frame side fed to the model"
539
  )
540
  max_new_tokens = gr.Slider(
541
  32, 256, value=DEFAULT_MAX_NEW_TOKENS, step=32, label="Analysis token budget"
 
547
  robot_desc,
548
  camera_desc,
549
  num_frames,
 
550
  max_image_side,
551
  max_new_tokens,
552
  ]