Fix skipped videos without a runtime limit; calibrate video pricing

#2
by pepijn223 HF Staff - opened
Files changed (8) hide show
  1. README.md +15 -11
  2. frontend/src/main.tsx +12 -10
  3. human_video.py +28 -11
  4. model_catalog.py +24 -5
  5. provider.py +5 -4
  6. studio.py +14 -12
  7. test_human_video.py +84 -24
  8. worker.py +13 -1
README.md CHANGED
@@ -121,17 +121,21 @@ a first-person human demonstration, following LeRobot's `--human_video` annotati
121
  Both media models run on fal through HF Inference Providers with the signed-in user's
122
  token. Selecting videos also selects Subtasks, since there is one video per subtask.
123
 
124
- The estimate follows the selected episodes' lengths: it expects about one subtask per 5 s of
125
- each episode (at least one per episode), allows twice that in the upper bound, and applies the
126
- per-episode cap. **Parallel generations** (1–32, default 8) limits simultaneous media
127
- requests across the whole job. The review warns when the expected videos at that parallelism
128
- would outlast the job timeout (about 3 min per video).
129
-
130
- Videos start only after an episode's
131
- annotations pass validation, and each request reserves its quoted price
132
- (`model_catalog.HUMAN_VIDEO`) from the spending target first. A failed video is recorded
133
- and the others continue. Reaching the target or the runtime limit stops further videos but
134
- keeps the episode's annotations. The launch checks that both fal mappings are live.
 
 
 
 
135
 
136
  The output holds `annotation_studio/human_videos/episode_XXXXXX/segment_XXX{.mp4,
137
  _first_frame.png,_robot_frame.png}` (named by source episode) and `index.json`, which lists
 
121
  Both media models run on fal through HF Inference Providers with the signed-in user's
122
  token. Selecting videos also selects Subtasks, since there is one video per subtask.
123
 
124
+ The estimate uses the selected episodes' total seconds and count, calibrated on the first
125
+ video run (3 ReBot episodes, 159.7 s, 33 subtasks): **1 subtask per episode + 1 per 5.3 s**,
126
+ up to 1.5× that in the upper bound, with the per-episode cap applied. Each video is priced at
127
+ fal's list prices: the edit at $0.03 per output megapixel (1024×1024) and the 5 s 480p video
128
+ at $0.05/s, about $0.28 in total, plus about 1,750 input and 550 output VLM tokens.
129
+ **Parallel generations** (1–32, default 8) limits simultaneous media requests across the
130
+ whole job. The review warns when the expected videos at that parallelism would outlast the
131
+ job timeout (about 3 min per video).
132
+
133
+ Videos start only after an episode's annotations pass validation. Each request holds a
134
+ conservative reservation (`model_catalog.HUMAN_VIDEO`) from the spending target while in
135
+ flight and is reconciled to its list price when it returns. A failed video is recorded and
136
+ the others continue. Videos stop at the spending target, or before the runtime limit (the job
137
+ timeout when none is set), without paying for an edit whose video could not finish; the
138
+ episode's annotations are kept. The launch checks that both fal mappings are live.
139
 
140
  The output holds `annotation_studio/human_videos/episode_XXXXXX/segment_XXX{.mp4,
141
  _first_frame.png,_robot_frame.png}` (named by source episode) and `index.json`, which lists
frontend/src/main.tsx CHANGED
@@ -94,13 +94,15 @@ type HumanVideoProfile = {
94
  video_model: string;
95
  resolution: string;
96
  duration_s: number;
97
- edit_usd: number;
98
- video_usd: number;
 
99
  checked_at: string;
100
  pricing_url: string;
101
  segment_choices: number[];
102
  parallel_choices: number[];
103
  seconds_per_subtask: number;
 
104
  minutes_per_video: number;
105
  };
106
  type ModelCatalog = {
@@ -1129,14 +1131,14 @@ function App() {
1129
  {modelCatalog.human_video.resolution} via{" "}
1130
  {modelCatalog.human_video.provider_name} on Inference
1131
  Providers · about{" "}
1132
- {money(
1133
- modelCatalog.human_video.edit_usd +
1134
- modelCatalog.human_video.video_usd,
1135
- )}{" "}
1136
- per video, billed to your account. The estimate counts
1137
- about one subtask per{" "}
1138
- {modelCatalog.human_video.seconds_per_subtask} s of each
1139
- episode.
1140
  {options.features.includes(HUMAN_VIDEOS) &&
1141
  " One video per subtask, so Subtasks are included."}{" "}
1142
  <a
 
94
  video_model: string;
95
  resolution: string;
96
  duration_s: number;
97
+ segment_usd: number;
98
+ edit_usd_per_megapixel: number;
99
+ video_usd_per_second: number;
100
  checked_at: string;
101
  pricing_url: string;
102
  segment_choices: number[];
103
  parallel_choices: number[];
104
  seconds_per_subtask: number;
105
+ subtasks_per_episode: number;
106
  minutes_per_video: number;
107
  };
108
  type ModelCatalog = {
 
1131
  {modelCatalog.human_video.resolution} via{" "}
1132
  {modelCatalog.human_video.provider_name} on Inference
1133
  Providers · about{" "}
1134
+ {money(modelCatalog.human_video.segment_usd)} per video
1135
+ (edit ${modelCatalog.human_video.edit_usd_per_megapixel}
1136
+ /MP + video $
1137
+ {modelCatalog.human_video.video_usd_per_second}/s), billed
1138
+ to your account. The estimate counts{" "}
1139
+ {modelCatalog.human_video.subtasks_per_episode} + one
1140
+ subtask per {modelCatalog.human_video.seconds_per_subtask}{" "}
1141
+ s of each episode.
1142
  {options.features.includes(HUMAN_VIDEOS) &&
1143
  " One video per subtask, so Subtasks are included."}{" "}
1144
  <a
human_video.py CHANGED
@@ -52,20 +52,32 @@ def subtask_spans(record, staging):
52
 
53
 
54
  class FalMedia:
55
- """Qwen-Image-Edit + MiniMax-H3 on fal through the HF router, charged to ``budget``."""
56
 
57
- def __init__(self, token, budget, remaining_time, profile=None, concurrency=None, poll_interval=2.0):
58
- self.token, self.budget, self.remaining_time = token, budget, remaining_time
59
- self.profile = profile or human_video_profile()
 
 
 
 
60
  self.poll_interval = poll_interval
61
  # Job-wide limit on simultaneous media requests, shared by every episode thread.
62
  self.concurrency = max(1, min(int(concurrency or self.profile["concurrency"]), 32))
63
  self.semaphore = threading.BoundedSemaphore(self.concurrency)
64
 
 
 
 
 
 
 
 
 
65
  def edit_image(self, image, prompt):
66
  from huggingface_hub import InferenceClient
67
 
68
- self.remaining_time()
69
  with self.semaphore:
70
  reservation = self.budget.reserve_media(self.profile["edit_usd"])
71
  client = InferenceClient(provider=self.profile["provider"], api_key=self.token)
@@ -75,7 +87,8 @@ class FalMedia:
75
  model=self.profile["edit_model"],
76
  guidance_scale=self.profile["edit_guidance_scale"],
77
  )
78
- self.budget.record_media(reservation)
 
79
  # fal returns a square image; keep the camera's size and aspect ratio.
80
  return edited.convert("RGB").resize(image.size, Image.LANCZOS)
81
 
@@ -83,8 +96,7 @@ class FalMedia:
83
  from huggingface_hub.inference._providers.fal_ai import FalAIImageToVideoTask
84
  from huggingface_hub.utils import get_session, hf_raise_for_status
85
 
86
- if self.remaining_time() < VIDEO_SECONDS_NEEDED:
87
- raise DeadlineExceededError("Not enough runtime left for another video.")
88
  # InferenceClient has no image-text-to-video method yet; fal's image-to-video helper speaks
89
  # the same queue protocol once its task matches the model's mapping.
90
  helper = FalAIImageToVideoTask()
@@ -119,7 +131,8 @@ class FalMedia:
119
  while status != "COMPLETED":
120
  if status in ("FAILED", "ERROR", "CANCELLED"):
121
  raise RuntimeError(f"The video provider reported {status}.")
122
- self.remaining_time()
 
123
  time.sleep(self.poll_interval)
124
  polled = session.get(f"{base}{path}/status{query}", headers=request.headers, timeout=30)
125
  hf_raise_for_status(polled)
@@ -130,7 +143,9 @@ class FalMedia:
130
  # The result is a public CDN link; the token is never sent there.
131
  video = session.get(url, timeout=120)
132
  hf_raise_for_status(video)
133
- self.budget.record_media(reservation)
 
 
134
  return video.content
135
 
136
 
@@ -199,6 +214,7 @@ def generate_human_videos(record, staging, vlm, frames, full_frames, media, out_
199
  entry["status"] = "skipped"
200
  return entry
201
  try:
 
202
  if not isinstance(plan, dict) or not plan.get("edit_prompt"):
203
  raise ValueError("The model returned no human-video plan.")
204
  robot = full_frames.frames_at(record, [_nearest(ts, span["start"])])[0].convert("RGB")
@@ -230,7 +246,8 @@ def generate_human_videos(record, staging, vlm, frames, full_frames, media, out_
230
  stop.set()
231
  entry["status"] = "time_stopped" if isinstance(exc, DeadlineExceededError) else "budget_stopped"
232
  except Exception as exc:
233
- entry.update(status="failed", error_type=type(exc).__name__)
 
234
  return entry
235
 
236
  with ThreadPoolExecutor(max_workers=min(len(spans), getattr(media, "concurrency", 1))) as pool:
 
52
 
53
 
54
  class FalMedia:
55
+ """Qwen-Image-Edit + MiniMax-H3 on fal through the HF router, charged to ``budget``.
56
 
57
+ ``deadline`` is the epoch by which the job must finish (None = no limit).
58
+ """
59
+
60
+ def __init__(self, token, budget, deadline=None, profile=None, concurrency=None, poll_interval=2.0):
61
+ self.token, self.budget, self.deadline = token, budget, deadline
62
+ # Pinned manifests may predate newer price fields; fill them from the current catalog.
63
+ self.profile = {**human_video_profile(), **(profile or {})}
64
  self.poll_interval = poll_interval
65
  # Job-wide limit on simultaneous media requests, shared by every episode thread.
66
  self.concurrency = max(1, min(int(concurrency or self.profile["concurrency"]), 32))
67
  self.semaphore = threading.BoundedSemaphore(self.concurrency)
68
 
69
+ def time_left(self):
70
+ return self.deadline - time.time() if self.deadline else float("inf")
71
+
72
+ def check_time(self):
73
+ """Raise before paying for a segment that could not finish before the deadline."""
74
+ if self.time_left() < VIDEO_SECONDS_NEEDED:
75
+ raise DeadlineExceededError("Not enough runtime left for another video.")
76
+
77
  def edit_image(self, image, prompt):
78
  from huggingface_hub import InferenceClient
79
 
80
+ self.check_time()
81
  with self.semaphore:
82
  reservation = self.budget.reserve_media(self.profile["edit_usd"])
83
  client = InferenceClient(provider=self.profile["provider"], api_key=self.token)
 
87
  model=self.profile["edit_model"],
88
  guidance_scale=self.profile["edit_guidance_scale"],
89
  )
90
+ megapixels = edited.width * edited.height / 1e6
91
+ self.budget.record_media(reservation, megapixels * self.profile["edit_usd_per_megapixel"])
92
  # fal returns a square image; keep the camera's size and aspect ratio.
93
  return edited.convert("RGB").resize(image.size, Image.LANCZOS)
94
 
 
96
  from huggingface_hub.inference._providers.fal_ai import FalAIImageToVideoTask
97
  from huggingface_hub.utils import get_session, hf_raise_for_status
98
 
99
+ self.check_time()
 
100
  # InferenceClient has no image-text-to-video method yet; fal's image-to-video helper speaks
101
  # the same queue protocol once its task matches the model's mapping.
102
  helper = FalAIImageToVideoTask()
 
131
  while status != "COMPLETED":
132
  if status in ("FAILED", "ERROR", "CANCELLED"):
133
  raise RuntimeError(f"The video provider reported {status}.")
134
+ if self.time_left() <= 5:
135
+ raise DeadlineExceededError("Runtime limit reached while a video was generating.")
136
  time.sleep(self.poll_interval)
137
  polled = session.get(f"{base}{path}/status{query}", headers=request.headers, timeout=30)
138
  hf_raise_for_status(polled)
 
143
  # The result is a public CDN link; the token is never sent there.
144
  video = session.get(url, timeout=120)
145
  hf_raise_for_status(video)
146
+ self.budget.record_media(
147
+ reservation, self.profile["video_usd_per_second"] * self.profile["duration_s"]
148
+ )
149
  return video.content
150
 
151
 
 
214
  entry["status"] = "skipped"
215
  return entry
216
  try:
217
+ media.check_time()
218
  if not isinstance(plan, dict) or not plan.get("edit_prompt"):
219
  raise ValueError("The model returned no human-video plan.")
220
  robot = full_frames.frames_at(record, [_nearest(ts, span["start"])])[0].convert("RGB")
 
246
  stop.set()
247
  entry["status"] = "time_stopped" if isinstance(exc, DeadlineExceededError) else "budget_stopped"
248
  except Exception as exc:
249
+ # A short reason for index.json; provider errors carry no credentials.
250
+ entry.update(status="failed", error_type=type(exc).__name__, error=str(exc)[:300])
251
  return entry
252
 
253
  with ThreadPoolExecutor(max_workers=min(len(spans), getattr(media, "concurrency", 1))) as pool:
model_catalog.py CHANGED
@@ -219,15 +219,23 @@ HUMAN_VIDEO = {
219
  "video_task": "image-text-to-video",
220
  "resolution": "480P",
221
  "duration_s": 5,
222
- # $0.03 per megapixel; fal returns about 1 MP, reserved as 1.33 MP.
 
 
 
 
 
223
  "edit_usd": 0.04,
224
- # $0.05 per second at 480p; the clip is 5 s, reserved as 6 s.
225
  "video_usd": 0.30,
226
  # Default simultaneous media requests per job (user-selectable up to 32).
227
  "concurrency": 8,
228
- # Estimation only: the action-phase prompt yields about one subtask per 5 s of episode
229
- # (the same rate the memory estimate uses), and a video takes up to ~3 min on fal.
230
- "seconds_per_subtask": 5,
 
 
 
 
231
  "minutes_per_video": 3,
232
  "checked_at": "2026-09-30",
233
  "pricing_url": "https://fal.ai/models/minimax/h3/image-to-video",
@@ -248,6 +256,16 @@ def human_video_profile():
248
 
249
 
250
  def human_video_segment_cost(profile=None):
 
 
 
 
 
 
 
 
 
 
251
  profile = profile or HUMAN_VIDEO
252
  return profile["edit_usd"] + profile["video_usd"]
253
 
@@ -261,5 +279,6 @@ def public_models():
261
  **human_video_profile(),
262
  "segment_choices": list(HUMAN_VIDEO_SEGMENT_CHOICES),
263
  "parallel_choices": list(HUMAN_VIDEO_PARALLEL_CHOICES),
 
264
  },
265
  }
 
219
  "video_task": "image-text-to-video",
220
  "resolution": "480P",
221
  "duration_s": 5,
222
+ # fal list prices. The edit is billed per output megapixel (fal returns 1024x1024), the video
223
+ # per second of output.
224
+ "edit_usd_per_megapixel": 0.03,
225
+ "edit_megapixels": 1.05,
226
+ "video_usd_per_second": 0.05,
227
+ # Reserved before each request and reconciled to the price above on success.
228
  "edit_usd": 0.04,
 
229
  "video_usd": 0.30,
230
  # Default simultaneous media requests per job (user-selectable up to 32).
231
  "concurrency": 8,
232
+ # Estimation only, calibrated on the first Studio video run (3 ReBot episodes, 159.7 s,
233
+ # 33 subtasks): subtasks ≈ 1 per episode + 1 per 5.3 s. The two VLM calls per video used
234
+ # about 1,750 input and 550 output tokens together. A video takes up to ~3 min on fal.
235
+ "subtasks_per_episode": 1,
236
+ "seconds_per_subtask": 5.3,
237
+ "vlm_input_tokens": 1750,
238
+ "vlm_output_tokens": 550,
239
  "minutes_per_video": 3,
240
  "checked_at": "2026-09-30",
241
  "pricing_url": "https://fal.ai/models/minimax/h3/image-to-video",
 
256
 
257
 
258
  def human_video_segment_cost(profile=None):
259
+ """Expected fal charge for one video (edit + animation) at list prices."""
260
+ profile = profile or HUMAN_VIDEO
261
+ return (
262
+ profile["edit_usd_per_megapixel"] * profile["edit_megapixels"]
263
+ + profile["video_usd_per_second"] * profile["duration_s"]
264
+ )
265
+
266
+
267
+ def human_video_segment_reservation(profile=None):
268
+ """Conservative amount held from the spending target while one video is in flight."""
269
  profile = profile or HUMAN_VIDEO
270
  return profile["edit_usd"] + profile["video_usd"]
271
 
 
279
  **human_video_profile(),
280
  "segment_choices": list(HUMAN_VIDEO_SEGMENT_CHOICES),
281
  "parallel_choices": list(HUMAN_VIDEO_PARALLEL_CHOICES),
282
+ "segment_usd": human_video_segment_cost(),
283
  },
284
  }
provider.py CHANGED
@@ -65,11 +65,12 @@ class Budget:
65
  self.media_attempts += 1
66
  return amount
67
 
68
- def record_media(self, amount):
69
- """A media request returned its output; count its quoted price as measured."""
70
  with self.lock:
71
- self.measured += amount
72
- self.media_usd += amount
 
73
  self.media_completed += 1
74
 
75
  def reconcile(self, reservation, usage):
 
65
  self.media_attempts += 1
66
  return amount
67
 
68
+ def record_media(self, reservation, cost):
69
+ """A media request returned its output: replace its reservation with its list price."""
70
  with self.lock:
71
+ self.reserved += cost - reservation
72
+ self.measured += cost
73
+ self.media_usd += cost
74
  self.media_completed += 1
75
 
76
  def reconcile(self, reservation, usage):
studio.py CHANGED
@@ -20,6 +20,7 @@ from model_catalog import (
20
  HUMAN_VIDEO_SEGMENT_CHOICES,
21
  human_video_profile,
22
  human_video_segment_cost,
 
23
  model_profile,
24
  )
25
  from prompt_config import TEMPLATES, validate_prompts
@@ -215,20 +216,21 @@ def estimate(
215
  videos = videos_max = media = media_max = video_minutes = 0
216
  if HUMAN_VIDEOS in features:
217
  video_profile = human_video_profile()
218
- # One video per subtask, and subtask counts scale with episode length: expect one per
219
- # `seconds_per_subtask`, allow twice that in the upper bound, and apply the per-episode cap.
220
  cap = human_video_segments or math.inf
221
- per = video_profile["seconds_per_subtask"]
222
  fps = selection["info"]["fps"]
223
- lengths = [r["length"] / fps for r in selection["episodes"]]
224
- videos = sum(min(cap, max(1.0, s / per)) for s in lengths)
225
- videos_max = sum(min(cap, max(1, math.ceil(2 * s / per))) for s in lengths)
226
  media = videos * human_video_segment_cost()
227
  media_max = videos_max * human_video_segment_cost()
228
  video_minutes = math.ceil(videos / human_video_parallel) * video_profile["minutes_per_video"]
229
- # Two VLM calls per video: plan + edit prompt (6 frames), video prompt (edited frame + 6).
230
- inputs += videos * (2600 + 3500)
231
- outputs += videos * (300 + 350)
 
232
  calls += videos * 2
233
  if "plan_render" in prompts:
234
  plan_calls = max(n, seconds / 5) + (n * 3 if "Interjections & speech" in features else 0)
@@ -252,12 +254,12 @@ def estimate(
252
  cpu_max = math.ceil(hours * 60 + 1) * cpu_rate
253
  # Range includes retries, grounding, variable segment counts and recovery.
254
  return {
255
- "low": inference * 0.6 + media * 0.6 + cpu_max,
256
  "high": inference * 2.5 + media_max + cpu_max,
257
  "inference": inference,
258
  "media": media,
259
  "media_max": media_max,
260
- "videos": math.ceil(videos),
261
  "videos_max": videos_max,
262
  "video_minutes": video_minutes,
263
  "video_parallel": human_video_parallel if videos else 0,
@@ -507,7 +509,7 @@ def launch(
507
  if human_video["profile"] != human_video_profile():
508
  raise ValueError("Video pricing changed. Calculate a fresh estimate.")
509
  # One video must fit next to one conservative text reservation.
510
- minimum_reservation += human_video_segment_cost()
511
  if budget - plan["costs"]["cpu_max"] < minimum_reservation:
512
  raise ValueError(
513
  f"This model needs at least ${minimum_reservation + plan['costs']['cpu_max']:.2f} for one conservative request reservation. Increase the target or choose a lower-cost model."
 
20
  HUMAN_VIDEO_SEGMENT_CHOICES,
21
  human_video_profile,
22
  human_video_segment_cost,
23
+ human_video_segment_reservation,
24
  model_profile,
25
  )
26
  from prompt_config import TEMPLATES, validate_prompts
 
216
  videos = videos_max = media = media_max = video_minutes = 0
217
  if HUMAN_VIDEOS in features:
218
  video_profile = human_video_profile()
219
+ # One video per subtask. Subtasks grow with each episode's length (calibrated rate in
220
+ # model_catalog.HUMAN_VIDEO); the upper bound allows 1.5x that; the per-episode cap applies.
221
  cap = human_video_segments or math.inf
222
+ base, per = video_profile["subtasks_per_episode"], video_profile["seconds_per_subtask"]
223
  fps = selection["info"]["fps"]
224
+ expected = [base + r["length"] / fps / per for r in selection["episodes"]]
225
+ videos = sum(min(cap, e) for e in expected)
226
+ videos_max = sum(min(cap, math.ceil(1.5 * e)) for e in expected)
227
  media = videos * human_video_segment_cost()
228
  media_max = videos_max * human_video_segment_cost()
229
  video_minutes = math.ceil(videos / human_video_parallel) * video_profile["minutes_per_video"]
230
+ # Two VLM calls per video: the plan and edit prompt (6 thumbnails), then the video prompt
231
+ # (edited frame + 6 thumbnails). The plan call runs for every capped subtask.
232
+ inputs += videos * video_profile["vlm_input_tokens"]
233
+ outputs += videos * video_profile["vlm_output_tokens"]
234
  calls += videos * 2
235
  if "plan_render" in prompts:
236
  plan_calls = max(n, seconds / 5) + (n * 3 if "Interjections & speech" in features else 0)
 
254
  cpu_max = math.ceil(hours * 60 + 1) * cpu_rate
255
  # Range includes retries, grounding, variable segment counts and recovery.
256
  return {
257
+ "low": inference * 0.6 + media * 0.8 + cpu_max,
258
  "high": inference * 2.5 + media_max + cpu_max,
259
  "inference": inference,
260
  "media": media,
261
  "media_max": media_max,
262
+ "videos": round(videos),
263
  "videos_max": videos_max,
264
  "video_minutes": video_minutes,
265
  "video_parallel": human_video_parallel if videos else 0,
 
509
  if human_video["profile"] != human_video_profile():
510
  raise ValueError("Video pricing changed. Calculate a fresh estimate.")
511
  # One video must fit next to one conservative text reservation.
512
+ minimum_reservation += human_video_segment_reservation()
513
  if budget - plan["costs"]["cpu_max"] < minimum_reservation:
514
  raise ValueError(
515
  f"This model needs at least ${minimum_reservation + plan['costs']['cpu_max']:.2f} for one conservative request reservation. Increase the target or choose a lower-cost model."
test_human_video.py CHANGED
@@ -13,7 +13,11 @@ from PIL import Image
13
  sys.path.insert(0, str(Path(__file__).parent))
14
  import human_video # noqa: E402
15
  import studio # noqa: E402
16
- from model_catalog import human_video_profile, human_video_segment_cost # noqa: E402
 
 
 
 
17
  from provider import Budget, BudgetExceededError, DeadlineExceededError # noqa: E402
18
 
19
  from lerobot.annotations.steerable_pipeline.reader import EpisodeRecord
@@ -64,29 +68,41 @@ def episodes(*seconds, fps=30):
64
  }
65
 
66
 
67
- def test_video_estimate_scales_with_episode_length_and_count():
 
 
 
 
 
 
 
 
 
 
68
  base = studio.estimate(selection(), ["Subtasks"], 0.1, 4, 0.0005)
69
  assert base["media"] == 0 and base["videos"] == 0
70
- # 10 s and 20 s episodes: about one subtask per 5 s -> 2 + 4 videos, up to 4 + 8.
71
  every = video_estimate(selection())
72
- assert every["videos"] == 6 and every["videos_max"] == 12
73
- assert every["media"] == pytest.approx(6 * human_video_segment_cost())
74
- assert every["media_max"] == pytest.approx(12 * human_video_segment_cost())
 
75
  assert every["high"] - base["high"] > every["media_max"]
76
- assert every["calls"] == base["calls"] + 12
77
- # Longer episodes and more episodes cost more; very short episodes still get one video.
78
  assert video_estimate(episodes(40, 40))["media"] > every["media"]
79
- assert video_estimate(episodes(10, 20, 10, 20))["videos"] == 12
80
- assert video_estimate(episodes(1, 2))["videos"] == 2
 
81
  # A per-episode cap bounds both the expected and maximum count.
82
  capped = video_estimate(selection(), segments=3)
83
- assert capped["videos"] == 5 and capped["videos_max"] == 6
84
 
85
 
86
  def test_parallel_generations_shorten_the_video_time_estimate():
87
- long = episodes(*[60] * 20) # 240 videos expected
88
  slow, fast = video_estimate(long, parallel=1), video_estimate(long, parallel=32)
89
- assert slow["video_minutes"] == 240 * 3 and fast["video_minutes"] == 8 * 3
90
  assert slow["media"] == fast["media"]
91
  with pytest.raises(ValueError, match="parallel"):
92
  video_estimate(selection(), parallel=64)
@@ -179,9 +195,11 @@ def test_launch_refuses_offline_fal_models_or_stale_video_prices():
179
  def test_media_budget_reserves_before_request_and_stops_at_target():
180
  budget = Budget(0.5, 0.15, 0.5)
181
  reservation = budget.reserve_media(0.3)
182
- budget.record_media(reservation)
183
- assert budget.summary()["media_usd"] == pytest.approx(0.3)
184
- assert budget.summary()["measured_inference_usd"] == pytest.approx(0.3)
 
 
185
  with pytest.raises(BudgetExceededError):
186
  budget.reserve_media(0.3)
187
  assert budget.summary()["media_requests"] == 1
@@ -190,11 +208,15 @@ def test_media_budget_reserves_before_request_and_stops_at_target():
190
  class FakeMedia:
191
  concurrency = 4
192
 
193
- def __init__(self, fail_first=False, budget_after=None):
194
  self.profile = human_video_profile()
195
- self.fail_first, self.budget_after = fail_first, budget_after
196
  self.edits, self.videos = [], []
197
 
 
 
 
 
198
  def edit_image(self, image, prompt):
199
  self.edits.append((image.size, prompt))
200
  return image.transpose(Image.FLIP_LEFT_RIGHT)
@@ -313,9 +335,9 @@ def test_parallel_setting_bounds_simultaneous_media_requests():
313
  import threading
314
 
315
  budget = Budget(5, 0.15, 0.5)
316
- assert human_video.FalMedia("t", budget, lambda: 3600).concurrency == human_video_profile()["concurrency"]
317
- assert human_video.FalMedia("t", budget, lambda: 3600, concurrency=64).concurrency == 32
318
- media = human_video.FalMedia("t", budget, lambda: 3600, concurrency=3)
319
  active, peak, lock = [0], [0], threading.Lock()
320
 
321
  def request():
@@ -415,7 +437,7 @@ def test_video_polls_router_charges_quoted_price_and_never_sends_token_to_cdn(fa
415
  session = FakeSession(["IN_PROGRESS", "COMPLETED"])
416
  use(session)
417
  budget = Budget(5, 0.15, 0.5)
418
- media = human_video.FalMedia("hf_token", budget, lambda: 3600, poll_interval=0)
419
  assert media.image_to_video(Image.new("RGB", (64, 48)), "a prompt") == b"mp4-bytes"
420
  assert seen["task"] == "image-text-to-video" and seen["model"] == "MiniMaxAI/MiniMax-H3"
421
  assert seen["parameters"] == {"prompt": "a prompt", "resolution": "480P", "duration": 5}
@@ -423,7 +445,8 @@ def test_video_polls_router_charges_quoted_price_and_never_sends_token_to_cdn(fa
423
  assert [c[1] for c in session.calls if c[0] == "get"][:2] == [status_url, status_url]
424
  cdn = [c for c in session.calls if c[1].startswith("https://cdn.fal.media")]
425
  assert cdn and cdn[0][2] is None
426
- assert budget.summary()["media_usd"] == pytest.approx(human_video_profile()["video_usd"])
 
427
 
428
 
429
  def test_video_is_not_started_without_runtime_for_it(fal):
@@ -431,7 +454,7 @@ def test_video_is_not_started_without_runtime_for_it(fal):
431
  session = FakeSession([])
432
  use(session)
433
  budget = Budget(5, 0.15, 0.5)
434
- media = human_video.FalMedia("hf_token", budget, lambda: 60, poll_interval=0)
435
  with pytest.raises(DeadlineExceededError):
436
  media.image_to_video(Image.new("RGB", (64, 48)), "a prompt")
437
  assert session.calls == [] and budget.summary()["media_requests"] == 0
@@ -483,3 +506,40 @@ def test_media_endpoint_serves_only_generated_files(monkeypatch):
483
  assert "secret-token" not in json.dumps(listed)
484
  anonymous = TestClient(server.app).get("/api/jobs/job1/media?path=" + listed[0]["video"].split("=")[1])
485
  assert anonymous.status_code == 401
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13
  sys.path.insert(0, str(Path(__file__).parent))
14
  import human_video # noqa: E402
15
  import studio # noqa: E402
16
+ from model_catalog import ( # noqa: E402
17
+ human_video_profile,
18
+ human_video_segment_cost,
19
+ human_video_segment_reservation,
20
+ )
21
  from provider import Budget, BudgetExceededError, DeadlineExceededError # noqa: E402
22
 
23
  from lerobot.annotations.steerable_pipeline.reader import EpisodeRecord
 
68
  }
69
 
70
 
71
+ def test_video_estimate_matches_the_calibration_run():
72
+ # First Studio video run: 3 ReBot episodes (68.6 s, 46.8 s, 44.4 s) had 33 subtasks.
73
+ run = video_estimate(episodes(68.6, 46.8, 44.4))
74
+ assert run["videos"] == 33
75
+ # fal list prices: $0.03/MP for a 1024x1024 edit + $0.05/s for a 5 s 480p video.
76
+ assert human_video_segment_cost() == pytest.approx(0.03 * 1.05 + 0.05 * 5)
77
+ assert run["media"] == pytest.approx(run["videos"] * human_video_segment_cost(), rel=0.03)
78
+ assert human_video_segment_reservation() > human_video_segment_cost()
79
+
80
+
81
+ def test_video_estimate_scales_with_total_seconds_and_episode_count():
82
  base = studio.estimate(selection(), ["Subtasks"], 0.1, 4, 0.0005)
83
  assert base["media"] == 0 and base["videos"] == 0
84
+ # 10 s and 20 s episodes: 1 + s / 5.3 subtasks each -> 7.66 videos, up to 5 + 8.
85
  every = video_estimate(selection())
86
+ expected = (1 + 10 / 5.3) + (1 + 20 / 5.3)
87
+ assert every["videos"] == 8 and every["videos_max"] == 13
88
+ assert every["media"] == pytest.approx(expected * human_video_segment_cost())
89
+ assert every["media_max"] == pytest.approx(13 * human_video_segment_cost())
90
  assert every["high"] - base["high"] > every["media_max"]
91
+ assert every["calls"] == pytest.approx(base["calls"] + expected * 2, abs=1)
92
+ # More seconds and more episodes both add videos; every episode gets at least one.
93
  assert video_estimate(episodes(40, 40))["media"] > every["media"]
94
+ assert video_estimate(episodes(10, 20, 10, 20))["media"] == pytest.approx(2 * every["media"])
95
+ assert video_estimate(episodes(15, 15, 15))["videos"] > video_estimate(episodes(45))["videos"]
96
+ assert video_estimate(episodes(1, 2))["videos"] == 3
97
  # A per-episode cap bounds both the expected and maximum count.
98
  capped = video_estimate(selection(), segments=3)
99
+ assert capped["videos"] == 6 and capped["videos_max"] == 6
100
 
101
 
102
  def test_parallel_generations_shorten_the_video_time_estimate():
103
+ long = episodes(*[60] * 20) # 20 * (1 + 60 / 5.3) = 246.4 videos expected
104
  slow, fast = video_estimate(long, parallel=1), video_estimate(long, parallel=32)
105
+ assert slow["video_minutes"] == 247 * 3 and fast["video_minutes"] == 8 * 3
106
  assert slow["media"] == fast["media"]
107
  with pytest.raises(ValueError, match="parallel"):
108
  video_estimate(selection(), parallel=64)
 
195
  def test_media_budget_reserves_before_request_and_stops_at_target():
196
  budget = Budget(0.5, 0.15, 0.5)
197
  reservation = budget.reserve_media(0.3)
198
+ budget.record_media(reservation, 0.25)
199
+ # Success replaces the reservation with the list price.
200
+ assert budget.summary()["media_usd"] == pytest.approx(0.25)
201
+ assert budget.summary()["measured_inference_usd"] == pytest.approx(0.25)
202
+ assert budget.reserved == pytest.approx(0.25)
203
  with pytest.raises(BudgetExceededError):
204
  budget.reserve_media(0.3)
205
  assert budget.summary()["media_requests"] == 1
 
208
  class FakeMedia:
209
  concurrency = 4
210
 
211
+ def __init__(self, fail_first=False, budget_after=None, out_of_time=False):
212
  self.profile = human_video_profile()
213
+ self.fail_first, self.budget_after, self.out_of_time = fail_first, budget_after, out_of_time
214
  self.edits, self.videos = [], []
215
 
216
+ def check_time(self):
217
+ if self.out_of_time:
218
+ raise DeadlineExceededError("Not enough runtime left for another video.")
219
+
220
  def edit_image(self, image, prompt):
221
  self.edits.append((image.size, prompt))
222
  return image.transpose(Image.FLIP_LEFT_RIGHT)
 
335
  import threading
336
 
337
  budget = Budget(5, 0.15, 0.5)
338
+ assert human_video.FalMedia("t", budget).concurrency == human_video_profile()["concurrency"]
339
+ assert human_video.FalMedia("t", budget, concurrency=64).concurrency == 32
340
+ media = human_video.FalMedia("t", budget, concurrency=3)
341
  active, peak, lock = [0], [0], threading.Lock()
342
 
343
  def request():
 
437
  session = FakeSession(["IN_PROGRESS", "COMPLETED"])
438
  use(session)
439
  budget = Budget(5, 0.15, 0.5)
440
+ media = human_video.FalMedia("hf_token", budget, None, poll_interval=0)
441
  assert media.image_to_video(Image.new("RGB", (64, 48)), "a prompt") == b"mp4-bytes"
442
  assert seen["task"] == "image-text-to-video" and seen["model"] == "MiniMaxAI/MiniMax-H3"
443
  assert seen["parameters"] == {"prompt": "a prompt", "resolution": "480P", "duration": 5}
 
445
  assert [c[1] for c in session.calls if c[0] == "get"][:2] == [status_url, status_url]
446
  cdn = [c for c in session.calls if c[1].startswith("https://cdn.fal.media")]
447
  assert cdn and cdn[0][2] is None
448
+ assert budget.summary()["media_usd"] == pytest.approx(0.05 * 5)
449
+ assert budget.reserved == pytest.approx(0.05 * 5)
450
 
451
 
452
  def test_video_is_not_started_without_runtime_for_it(fal):
 
454
  session = FakeSession([])
455
  use(session)
456
  budget = Budget(5, 0.15, 0.5)
457
+ media = human_video.FalMedia("hf_token", budget, time.time() + 60, poll_interval=0)
458
  with pytest.raises(DeadlineExceededError):
459
  media.image_to_video(Image.new("RGB", (64, 48)), "a prompt")
460
  assert session.calls == [] and budget.summary()["media_requests"] == 0
 
506
  assert "secret-token" not in json.dumps(listed)
507
  anonymous = TestClient(server.app).get("/api/jobs/job1/media?path=" + listed[0]["video"].split("=")[1])
508
  assert anonymous.status_code == 401
509
+
510
+
511
+ def test_jobs_without_a_runtime_limit_still_generate_videos():
512
+ """Regression: the first run skipped every video because no runtime limit was set."""
513
+ from worker import video_deadline
514
+
515
+ manifest = {"costs": {"hours": 4}}
516
+ deadline = video_deadline(None, manifest, now=1000.0)
517
+ assert deadline == 1000.0 + 4 * 3600 - 900
518
+ assert video_deadline(5000.0, manifest) == 5000.0
519
+ media = human_video.FalMedia("t", Budget(5, 0.15, 0.5), video_deadline(None, manifest))
520
+ media.check_time() # hours of runtime left: no DeadlineExceededError
521
+ assert human_video.FalMedia("t", Budget(5, 0.15, 0.5), None).time_left() == float("inf")
522
+
523
+
524
+ def test_no_edit_is_paid_for_when_the_video_could_not_finish(tmp_path, episode):
525
+ media = FakeMedia(out_of_time=True)
526
+ result = annotate(tmp_path, episode, media, segments=0)
527
+ assert result["status"] == "completed"
528
+ assert media.edits == [] and media.videos == []
529
+ assert set(result["human_videos"]) <= {"time_stopped", "skipped"}
530
+
531
+
532
+ def test_older_pinned_profiles_get_current_price_fields():
533
+ old = {
534
+ k: v
535
+ for k, v in human_video_profile().items()
536
+ if k not in ("edit_usd_per_megapixel", "video_usd_per_second")
537
+ }
538
+ media = human_video.FalMedia("t", Budget(5, 0.15, 0.5), None, old)
539
+ assert media.profile["edit_usd_per_megapixel"] == 0.03 and media.profile["video_usd_per_second"] == 0.05
540
+
541
+
542
+ def test_failed_segments_record_a_short_reason(tmp_path, episode):
543
+ result = annotate(tmp_path, episode, FakeMedia(fail_first=True))
544
+ failed = next(e for e in result["human_video_entries"] if e["status"] == "failed")
545
+ assert failed["error_type"] == "RuntimeError" and failed["error"] == "provider error"
worker.py CHANGED
@@ -301,6 +301,18 @@ def _annotate_episode(record, row, info, root, manifest, provider, staging_dir,
301
  HUMAN_VIDEO_DIR = "annotation_studio/human_videos"
302
 
303
 
 
 
 
 
 
 
 
 
 
 
 
 
304
  def human_video_operations(entries, index):
305
  """Upload an episode's media and the updated index (source episode IDs until materialized)."""
306
  operations = []
@@ -422,7 +434,7 @@ def run(manifest, work=Path("/workspace/run")):
422
  FalMedia(
423
  token,
424
  budget,
425
- provider.remaining_time,
426
  manifest["human_video"]["profile"],
427
  concurrency=manifest["human_video"].get("parallel"),
428
  )
 
301
  HUMAN_VIDEO_DIR = "annotation_studio/human_videos"
302
 
303
 
304
+ def video_deadline(deadline, manifest, now=None):
305
+ """When videos must stop: the runtime limit if one was set, else the job timeout.
306
+
307
+ Without a runtime limit the job still stops at its timeout (counted from job start, before
308
+ setup), so keep a 15 minute margin for setup and the final commits.
309
+ """
310
+ if deadline:
311
+ return deadline
312
+ now = time.time() if now is None else now
313
+ return now + manifest["costs"]["hours"] * 3600 - 900
314
+
315
+
316
  def human_video_operations(entries, index):
317
  """Upload an episode's media and the updated index (source episode IDs until materialized)."""
318
  operations = []
 
434
  FalMedia(
435
  token,
436
  budget,
437
+ video_deadline(deadline, manifest),
438
  manifest["human_video"]["profile"],
439
  concurrency=manifest["human_video"].get("parallel"),
440
  )