futurefantasy commited on
Commit
f8c2464
·
verified ·
1 Parent(s): 9dabf65

Simplify quick_start to inference-only generic video entrypoint

Browse files
Files changed (2) hide show
  1. quick_start/README.md +30 -18
  2. quick_start/run_example.py +134 -259
quick_start/README.md CHANGED
@@ -1,7 +1,7 @@
1
  # Quick Start
2
 
3
- This quick start runs the released VLAC model on one bundled example video with the `chunk_all` prompt.
4
- The bundled `mp4` is sampled on the fly into a fixed `2 Hz` frame list, then the predicted progress points are aligned to the bundled dense GT curve for full-trajectory comparison.
5
 
6
  ## Requirements
7
 
@@ -12,7 +12,6 @@ The bundled `mp4` is sampled on the fly into a fixed `2 Hz` frame list, then the
12
  - `decord`
13
  - `numpy`
14
  - `Pillow`
15
- - `matplotlib` optional, only for automatic curve plots
16
  - at least `1` GPU with enough memory for this `30B` checkpoint
17
 
18
  ## Run
@@ -20,9 +19,26 @@ The bundled `mp4` is sampled on the fly into a fixed `2 Hz` frame list, then the
20
  From the model release root:
21
 
22
  ```bash
23
- python quick_start/run_example.py --example-id example_01
24
- python quick_start/run_example.py --example-id example_02
25
- python quick_start/run_example.py --example-id example_03
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
  ```
27
 
28
  If the model directory is not the parent of `quick_start/`, pass it explicitly:
@@ -30,27 +46,23 @@ If the model directory is not the parent of `quick_start/`, pass it explicitly:
30
  ```bash
31
  python quick_start/run_example.py \
32
  --model-path /path/to/VLAC2-Qwen3VL-30B-A3B-Progress \
33
- --example-id example_01
 
34
  ```
35
 
36
  ## Output
37
 
38
  By default the script writes:
39
 
40
- - `quick_start/outputs/example_01.jsonl`
41
- - `quick_start/outputs/example_02.jsonl`
42
- - `quick_start/outputs/example_03.jsonl`
43
 
44
  Each output file contains one JSONL row with:
45
 
46
- - example id
47
- - task metadata
48
- - `2 Hz` sampled input frame indices and timestamps
49
- - chunk_all prompt text
50
  - raw model response
51
  - parsed predicted key points: `时间 / 进度`
52
- - benchmark dense GT progress and semantic anchors
53
- - predicted dense curve aligned to the GT timeline
54
- - full-curve comparison metrics and an optional comparison plot path
55
 
56
- If `matplotlib` is unavailable, the JSONL output is still written and only the comparison plot is skipped.
 
1
  # Quick Start
2
 
3
+ This quick start runs VLAC progress inference on an arbitrary input video.
4
+ It samples the video at `2 Hz` by default, sends the sampled frames and prompt to the model, and writes one JSONL row with the raw response and parsed predicted progress points.
5
 
6
  ## Requirements
7
 
 
12
  - `decord`
13
  - `numpy`
14
  - `Pillow`
 
15
  - at least `1` GPU with enough memory for this `30B` checkpoint
16
 
17
  ## Run
 
19
  From the model release root:
20
 
21
  ```bash
22
+ python quick_start/run_example.py \
23
+ --video-path /path/to/video.mp4 \
24
+ --task-instruction "把洋葱放进快递箱里。"
25
+ ```
26
+
27
+ If you have a more detailed plan, pass it as text or file:
28
+
29
+ ```bash
30
+ python quick_start/run_example.py \
31
+ --video-path /path/to/video.mp4 \
32
+ --task-instruction "将三角烧杯放在三脚架上。" \
33
+ --task-plan-file /path/to/task_plan.txt
34
+ ```
35
+
36
+ If you want full control over the prompt, pass it directly:
37
+
38
+ ```bash
39
+ python quick_start/run_example.py \
40
+ --video-path /path/to/video.mp4 \
41
+ --prompt-file /path/to/prompt.txt
42
  ```
43
 
44
  If the model directory is not the parent of `quick_start/`, pass it explicitly:
 
46
  ```bash
47
  python quick_start/run_example.py \
48
  --model-path /path/to/VLAC2-Qwen3VL-30B-A3B-Progress \
49
+ --video-path /path/to/video.mp4 \
50
+ --task-instruction "把洋葱放进快递箱里。"
51
  ```
52
 
53
  ## Output
54
 
55
  By default the script writes:
56
 
57
+ - `quick_start/outputs/<video_stem>.jsonl`
 
 
58
 
59
  Each output file contains one JSONL row with:
60
 
61
+ - input video path
62
+ - sampled frame indices and timestamps
63
+ - decoded video statistics
64
+ - prompt text
65
  - raw model response
66
  - parsed predicted key points: `时间 / 进度`
 
 
 
67
 
68
+ No ground truth, alignment, metrics, or comparison plots are produced by this quick start script.
quick_start/run_example.py CHANGED
@@ -3,21 +3,17 @@ from __future__ import annotations
3
 
4
  import argparse
5
  import json
6
- import math
7
  import os
8
  import re
9
  import tempfile
10
  from pathlib import Path
11
- from typing import Any
12
 
13
  import numpy as np
14
  from PIL import Image
15
 
16
  THIS_DIR = Path(__file__).resolve().parent
17
  MODEL_ROOT = THIS_DIR.parent
18
- EXAMPLES_ROOT = MODEL_ROOT / "examples"
19
- MANIFEST_PATH = EXAMPLES_ROOT / "examples_manifest.json"
20
- CHUNK_ALL_SAMPLE_HZ = 2.0
21
 
22
  SIGNED_NUM = r"([+-]?[0-9]+(?:\.[0-9]+)?)"
23
  POINT_TIME_RE = re.compile(r"(?:Time|时间)[::]?\s*([0-9]+(?:\.[0-9]+)?)\s*s?", re.IGNORECASE)
@@ -30,7 +26,7 @@ INLINE_POINT_RE = re.compile(
30
 
31
  def parse_args() -> argparse.Namespace:
32
  parser = argparse.ArgumentParser(
33
- description="Run the released VLAC model on one bundled example with chunk_all prompt and fixed 2hz video input."
34
  )
35
  parser.add_argument(
36
  "--model-path",
@@ -39,16 +35,52 @@ def parse_args() -> argparse.Namespace:
39
  help="Path to the released VLAC model directory.",
40
  )
41
  parser.add_argument(
42
- "--example-id",
43
- type=str,
44
  required=True,
45
- help="Bundled example id, for example: example_01",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
46
  )
47
  parser.add_argument(
48
  "--output-jsonl",
49
  type=Path,
50
  default=None,
51
- help="Optional output path. Defaults to quick_start/outputs/<example_id>.jsonl",
52
  )
53
  parser.add_argument(
54
  "--max-new-tokens",
@@ -59,36 +91,22 @@ def parse_args() -> argparse.Namespace:
59
  return parser.parse_args()
60
 
61
 
62
- def load_manifest() -> dict[str, dict]:
63
- payload = json.loads(MANIFEST_PATH.read_text(encoding="utf-8"))
64
- return {item["example_id"]: item for item in payload.get("examples", [])}
 
 
 
 
 
 
 
65
 
66
 
67
- def strip_code_fence(text: str) -> str:
68
- cleaned = str(text or "").strip()
69
- if cleaned.startswith("```") and cleaned.endswith("```"):
70
- lines = cleaned.splitlines()
71
- if len(lines) >= 3:
72
- return "\n".join(lines[1:-1]).strip()
73
- return cleaned
74
-
75
-
76
- def extract_plan_text(task_instruction: str, task_description: str) -> str:
77
- lines = [line.strip() for line in str(task_description or "").splitlines() if line.strip()]
78
- if not lines:
79
- return ""
80
- first_line = lines[0].rstrip("::")
81
- normalized_instruction = str(task_instruction or "").strip().rstrip("::")
82
- if normalized_instruction and first_line == normalized_instruction:
83
- lines = lines[1:]
84
- return "\n".join(lines).strip()
85
-
86
-
87
- def build_chunk_all_prompt(metadata: dict[str, Any]) -> str:
88
- task_instruction = str(metadata.get("task_instruction") or "").strip()
89
- task_description = str(metadata.get("task_description") or "").strip()
90
- plan_text = extract_plan_text(task_instruction, task_description)
91
- task_and_plan = task_instruction if not plan_text else f"{task_instruction}\n{plan_text}"
92
  return (
93
  f"任务描述和具体规划: {task_and_plan}\n\n"
94
  "请根据任务描���和具体规划,找到并逐点生成视频中的关键动作点和相应的进度标注。"
@@ -98,14 +116,34 @@ def build_chunk_all_prompt(metadata: dict[str, Any]) -> str:
98
  )
99
 
100
 
101
- def compute_sampled_indices_2hz(num_frames: int, fps: float, sample_hz: float) -> tuple[list[int], list[float]]:
102
- if num_frames <= 0 or fps <= 0 or sample_hz <= 0:
103
- return [], []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
  frame_ids = list(range(num_frames))
105
  eligible_arr = np.array(frame_ids, dtype=float)
106
- start_frame = frame_ids[0]
107
- end_frame = frame_ids[-1]
108
- duration_sec = max(0.0, (end_frame - start_frame) / fps)
109
  step_sec = 1.0 / sample_hz
110
 
111
  target_times: list[float] = []
@@ -121,38 +159,38 @@ def compute_sampled_indices_2hz(num_frames: int, fps: float, sample_hz: float) -
121
  sampled_timestamps_sec: list[float] = []
122
  seen = set()
123
  for target_time in target_times:
124
- target_frame = start_frame + target_time * fps
125
  pos = int(np.argmin(np.abs(eligible_arr - target_frame)))
126
  idx = int(eligible_arr[pos])
127
  if idx in seen:
128
  continue
129
  seen.add(idx)
130
  sampled_indices.append(idx)
131
- sampled_timestamps_sec.append(round((idx - start_frame) / fps, 6))
132
- if not sampled_indices:
133
- sampled_indices = [0]
134
- sampled_timestamps_sec = [0.0]
135
  return sampled_indices, sampled_timestamps_sec
136
 
137
 
138
  def extract_sampled_frame_paths(
139
  video_path: Path,
140
- sampled_indices: list[int],
141
  *,
142
  temp_dir: Path,
143
- ) -> tuple[list[str], dict[str, float]]:
144
  try:
145
  from decord import VideoReader, cpu
146
  except ImportError as exc:
147
- raise SystemExit("Missing dependency: decord is required for 2hz frame sampling in quick_start.") from exc
148
 
149
  vr = VideoReader(str(video_path), ctx=cpu(0), num_threads=1)
150
  actual_frame_count = len(vr)
 
 
151
  if not sampled_indices:
152
  raise SystemExit("No sampled frame indices were generated.")
153
  if sampled_indices[-1] >= actual_frame_count:
154
  raise SystemExit(
155
- f"Bundled video is shorter than expected. Need frame index {sampled_indices[-1]}, got {actual_frame_count} frames."
 
156
  )
157
 
158
  batch = vr.get_batch(sampled_indices).asnumpy()
@@ -164,9 +202,10 @@ def extract_sampled_frame_paths(
164
 
165
  video_stats = {
166
  "decoded_frame_count": float(actual_frame_count),
167
- "decoded_avg_fps": float(vr.get_avg_fps()),
 
168
  }
169
- return frame_paths, video_stats
170
 
171
 
172
  def load_swift_runtime():
@@ -175,12 +214,26 @@ def load_swift_runtime():
175
  os.environ.setdefault("VIDEO_MAX_TOKEN_NUM", "256")
176
  os.environ.setdefault("VIDEO_MIN_TOKEN_NUM", "4")
177
  os.environ.setdefault("QWEN_VL_UTILS_MAX_FRAME_LIST", "0")
178
- import torch
 
 
 
 
 
179
  try:
180
  from swift.llm import InferRequest, PtEngine, RequestConfig
181
  except ImportError:
182
  from swift.infer_engine import InferRequest, RequestConfig, TransformersEngine as PtEngine # type: ignore
183
- return torch, InferRequest, PtEngine, RequestConfig
 
 
 
 
 
 
 
 
 
184
 
185
 
186
  def dedupe_sorted_points(times: list[float], values: list[float]) -> tuple[list[float], list[float]]:
@@ -192,7 +245,7 @@ def dedupe_sorted_points(times: list[float], values: list[float]) -> tuple[list[
192
  cur_time = pairs[0][0]
193
  bucket: list[float] = []
194
  for time_val, progress_val in pairs:
195
- if not math.isclose(time_val, cur_time, rel_tol=0.0, abs_tol=1e-9):
196
  out_times.append(float(cur_time))
197
  out_values.append(float(sum(bucket) / len(bucket)))
198
  cur_time = time_val
@@ -231,177 +284,32 @@ def parse_point_blocks(text: str) -> tuple[list[float], list[float]]:
231
  return dedupe_sorted_points(times, values)
232
 
233
 
234
- def align_curve_to_gt_dense(
235
- point_times_sec: list[float],
236
- point_values: list[float],
237
- gt_times_sec: list[float],
238
- ) -> list[float]:
239
- if not gt_times_sec or not point_values:
240
- return []
241
- if len(point_values) == 1:
242
- return [float(point_values[0])] * len(gt_times_sec)
243
- times, values = dedupe_sorted_points(point_times_sec, point_values)
244
- if len(values) == 1:
245
- return [float(values[0])] * len(gt_times_sec)
246
- aligned = np.interp(
247
- np.array(gt_times_sec, dtype=float),
248
- np.array(times, dtype=float),
249
- np.array(values, dtype=float),
250
- left=float(values[0]),
251
- right=float(values[-1]),
252
- )
253
- return [float(x) for x in aligned.tolist()]
254
-
255
-
256
- def pearson_corr(xs: list[float], ys: list[float]) -> float | None:
257
- if len(xs) != len(ys) or len(xs) < 2:
258
- return None
259
- x = np.array(xs, dtype=float)
260
- y = np.array(ys, dtype=float)
261
- if np.allclose(x, x[0]) or np.allclose(y, y[0]):
262
- return None
263
- value = float(np.corrcoef(x, y)[0, 1])
264
- if math.isnan(value) or not math.isfinite(value):
265
- return None
266
- return value
267
-
268
-
269
- def average_ranks(values: list[float]) -> list[float]:
270
- arr = np.array(values, dtype=float)
271
- order = np.argsort(arr, kind="mergesort")
272
- ranks = np.zeros(arr.shape[0], dtype=float)
273
- idx = 0
274
- while idx < len(order):
275
- next_idx = idx + 1
276
- while next_idx < len(order) and math.isclose(
277
- arr[order[next_idx]],
278
- arr[order[idx]],
279
- rel_tol=0.0,
280
- abs_tol=1e-9,
281
- ):
282
- next_idx += 1
283
- ranks[order[idx:next_idx]] = (idx + next_idx - 1) / 2.0 + 1.0
284
- idx = next_idx
285
- return ranks.tolist()
286
-
287
-
288
- def spearman_corr(xs: list[float], ys: list[float]) -> float | None:
289
- return pearson_corr(average_ranks(xs), average_ranks(ys))
290
-
291
-
292
- def compute_curve_metrics(gt_dense_progress: list[float], pred_dense_progress: list[float]) -> dict[str, float | int | None]:
293
- if not gt_dense_progress or len(gt_dense_progress) != len(pred_dense_progress):
294
- return {
295
- "point_count": 0,
296
- "mae": None,
297
- "rmse": None,
298
- "pearson": None,
299
- "spearman": None,
300
- }
301
- gt_arr = np.array(gt_dense_progress, dtype=float)
302
- pred_arr = np.array(pred_dense_progress, dtype=float)
303
- diff = pred_arr - gt_arr
304
- return {
305
- "point_count": int(len(gt_dense_progress)),
306
- "mae": float(np.mean(np.abs(diff))),
307
- "rmse": float(np.sqrt(np.mean(diff ** 2))),
308
- "pearson": pearson_corr(pred_arr.tolist(), gt_arr.tolist()),
309
- "spearman": spearman_corr(pred_arr.tolist(), gt_arr.tolist()),
310
- }
311
-
312
-
313
- def maybe_write_curve_plot(
314
- *,
315
- output_path: Path,
316
- gt_times_sec: list[float],
317
- gt_dense_progress: list[float],
318
- pred_dense_progress: list[float],
319
- pred_point_times_sec: list[float],
320
- pred_point_progress: list[float],
321
- ) -> str | None:
322
- try:
323
- import matplotlib.pyplot as plt
324
- except ImportError:
325
- return None
326
-
327
- plot_path = output_path.with_name(f"{output_path.stem}_curve_compare.png")
328
- fig, ax = plt.subplots(figsize=(10, 4.8))
329
- ax.plot(gt_times_sec, gt_dense_progress, color="#2563eb", linewidth=2.2, label="GT dense progress")
330
- if pred_dense_progress:
331
- ax.plot(gt_times_sec, pred_dense_progress, color="#f97316", linewidth=2.2, label="Pred aligned curve")
332
- if pred_point_progress:
333
- ax.scatter(
334
- pred_point_times_sec,
335
- pred_point_progress,
336
- color="#111827",
337
- s=28,
338
- zorder=3,
339
- label="Pred chunk_all points",
340
- )
341
- ax.set_xlabel("Time (s)")
342
- ax.set_ylabel("Progress (%)")
343
- ax.grid(alpha=0.2, linewidth=0.8)
344
- ax.legend(loc="best")
345
- fig.tight_layout()
346
- fig.savefig(plot_path, dpi=180)
347
- plt.close(fig)
348
- return str(plot_path.resolve())
349
-
350
-
351
- def format_metric(value: float | None) -> str:
352
- if value is None:
353
- return "N/A"
354
- return f"{value:.4f}"
355
-
356
-
357
  def main() -> int:
358
  args = parse_args()
359
- manifest = load_manifest()
360
- if args.example_id not in manifest:
361
- raise SystemExit(f"Unknown example id: {args.example_id}")
362
-
363
- example_info = manifest[args.example_id]
364
- metadata_path = MODEL_ROOT / example_info["metadata_path"]
365
- video_path = MODEL_ROOT / example_info["video_path"]
366
- if not metadata_path.exists():
367
- raise SystemExit(f"Missing metadata: {metadata_path}")
368
  if not video_path.exists():
369
  raise SystemExit(f"Missing video: {video_path}")
 
 
370
 
371
- metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
372
- gt_dense_progress = [float(x) for x in list(metadata.get("benchmark_dense_progress") or [])]
373
- fps = float(metadata.get("fps") or 0.0)
374
- num_frames = int(metadata.get("num_frames") or len(gt_dense_progress))
375
- if not gt_dense_progress:
376
- raise SystemExit("Missing benchmark_dense_progress in example metadata.")
377
- if fps <= 0 or num_frames <= 0:
378
- raise SystemExit(f"Invalid metadata fps/num_frames: fps={fps}, num_frames={num_frames}")
379
-
380
- prompt = build_chunk_all_prompt(metadata)
381
- output_path = args.output_jsonl or (THIS_DIR / "outputs" / f"{args.example_id}.jsonl")
382
  output_path.parent.mkdir(parents=True, exist_ok=True)
383
 
384
- sampled_indices_2hz, sampled_timestamps_sec_2hz = compute_sampled_indices_2hz(
385
- num_frames=num_frames,
386
- fps=fps,
387
- sample_hz=CHUNK_ALL_SAMPLE_HZ,
388
- )
389
- gt_times_sec = [round(frame_idx / fps, 6) for frame_idx in range(len(gt_dense_progress))]
390
-
391
- torch, InferRequest, PtEngine, RequestConfig = load_swift_runtime()
392
- device_map = "auto"
393
  engine = PtEngine(
394
- str(args.model_path.resolve()),
395
  model_type="qwen3_moe_vl",
396
  max_batch_size=1,
397
- device_map=device_map,
398
  )
399
 
400
- with tempfile.TemporaryDirectory(prefix=f"{args.example_id}_2hz_frames_") as temp_dir_str:
401
  temp_dir = Path(temp_dir_str)
402
- frame_paths, video_stats = extract_sampled_frame_paths(
403
  video_path,
404
- sampled_indices_2hz,
405
  temp_dir=temp_dir,
406
  )
407
  request = InferRequest(
@@ -421,63 +329,30 @@ def main() -> int:
421
  )[0].choices[0].message.content
422
 
423
  pred_point_times_sec, pred_point_progress = parse_point_blocks(response)
424
- pred_dense_progress = align_curve_to_gt_dense(
425
- point_times_sec=pred_point_times_sec,
426
- point_values=pred_point_progress,
427
- gt_times_sec=gt_times_sec,
428
- )
429
- curve_metrics = compute_curve_metrics(gt_dense_progress, pred_dense_progress)
430
- plot_path = maybe_write_curve_plot(
431
- output_path=output_path,
432
- gt_times_sec=gt_times_sec,
433
- gt_dense_progress=gt_dense_progress,
434
- pred_dense_progress=pred_dense_progress,
435
- pred_point_times_sec=pred_point_times_sec,
436
- pred_point_progress=pred_point_progress,
437
- )
438
-
439
  output_row = {
440
- "example_id": args.example_id,
441
- "bucket": metadata.get("bucket"),
442
- "global_episode_id": metadata.get("global_episode_id"),
443
- "task_instruction": metadata.get("task_instruction"),
444
- "task_description": metadata.get("task_description"),
445
- "video_path": str(video_path.resolve()),
446
  "prompt_variant": "chunk_all",
447
- "input_sample_hz": CHUNK_ALL_SAMPLE_HZ,
448
- "input_frame_indices_2hz": sampled_indices_2hz,
449
- "input_timestamps_sec_2hz": sampled_timestamps_sec_2hz,
450
- "input_frame_count_2hz": len(sampled_indices_2hz),
451
  "decoded_video_stats": video_stats,
452
  "prompt": prompt,
453
  "response": response,
454
- "benchmark_progress_type": metadata.get("benchmark_progress_type"),
455
- "benchmark_progress_source": metadata.get("benchmark_progress_source"),
456
- "benchmark_semantic_anchors": metadata.get("benchmark_semantic_anchors"),
457
- "gt_dense_progress": gt_dense_progress,
458
- "gt_dense_timestamps_sec": gt_times_sec,
459
  "pred_curve_parse_ok": bool(pred_point_progress),
460
  "pred_curve_point_times_sec": pred_point_times_sec,
461
  "pred_curve_point_progress": pred_point_progress,
462
- "pred_dense_progress_aligned_to_gt": pred_dense_progress,
463
- "curve_compare_metrics": curve_metrics,
464
- "curve_compare_plot": plot_path,
465
  }
466
  output_path.write_text(json.dumps(output_row, ensure_ascii=False) + "\n", encoding="utf-8")
467
 
468
- print(f"example_id={args.example_id}")
469
- print(f"video_path={video_path.resolve()}")
470
  print(f"output_jsonl={output_path.resolve()}")
471
- print(f"input_sample_hz={CHUNK_ALL_SAMPLE_HZ}")
472
- print(f"input_frame_count_2hz={len(sampled_indices_2hz)}")
473
  print(f"pred_keypoint_count={len(pred_point_progress)}")
474
- print(f"gt_dense_point_count={len(gt_dense_progress)}")
475
- print(f"curve_mae={format_metric(curve_metrics['mae'])}")
476
- print(f"curve_rmse={format_metric(curve_metrics['rmse'])}")
477
- print(f"curve_pearson={format_metric(curve_metrics['pearson'])}")
478
- print(f"curve_spearman={format_metric(curve_metrics['spearman'])}")
479
- if plot_path is not None:
480
- print(f"curve_compare_plot={plot_path}")
481
  print()
482
  print(response)
483
  return 0
 
3
 
4
  import argparse
5
  import json
 
6
  import os
7
  import re
8
  import tempfile
9
  from pathlib import Path
 
10
 
11
  import numpy as np
12
  from PIL import Image
13
 
14
  THIS_DIR = Path(__file__).resolve().parent
15
  MODEL_ROOT = THIS_DIR.parent
16
+ DEFAULT_SAMPLE_HZ = 2.0
 
 
17
 
18
  SIGNED_NUM = r"([+-]?[0-9]+(?:\.[0-9]+)?)"
19
  POINT_TIME_RE = re.compile(r"(?:Time|时间)[::]?\s*([0-9]+(?:\.[0-9]+)?)\s*s?", re.IGNORECASE)
 
26
 
27
  def parse_args() -> argparse.Namespace:
28
  parser = argparse.ArgumentParser(
29
+ description="Run VLAC progress inference on an arbitrary input video.",
30
  )
31
  parser.add_argument(
32
  "--model-path",
 
35
  help="Path to the released VLAC model directory.",
36
  )
37
  parser.add_argument(
38
+ "--video-path",
39
+ type=Path,
40
  required=True,
41
+ help="Path to the input video.",
42
+ )
43
+ parser.add_argument(
44
+ "--task-instruction",
45
+ type=str,
46
+ default=None,
47
+ help="Task instruction used to build the default chunk_all prompt.",
48
+ )
49
+ parser.add_argument(
50
+ "--task-plan",
51
+ type=str,
52
+ default=None,
53
+ help="Optional task plan text appended after the task instruction.",
54
+ )
55
+ parser.add_argument(
56
+ "--task-plan-file",
57
+ type=Path,
58
+ default=None,
59
+ help="Optional file containing task plan text.",
60
+ )
61
+ parser.add_argument(
62
+ "--prompt",
63
+ type=str,
64
+ default=None,
65
+ help="Optional full prompt override. If set, task instruction and plan are ignored.",
66
+ )
67
+ parser.add_argument(
68
+ "--prompt-file",
69
+ type=Path,
70
+ default=None,
71
+ help="Optional file containing the full prompt override.",
72
+ )
73
+ parser.add_argument(
74
+ "--sample-hz",
75
+ type=float,
76
+ default=DEFAULT_SAMPLE_HZ,
77
+ help="Video sampling rate in Hz. Defaults to 2.0.",
78
  )
79
  parser.add_argument(
80
  "--output-jsonl",
81
  type=Path,
82
  default=None,
83
+ help="Optional output path. Defaults to quick_start/outputs/<video_stem>.jsonl",
84
  )
85
  parser.add_argument(
86
  "--max-new-tokens",
 
91
  return parser.parse_args()
92
 
93
 
94
+ def read_text_value(text_value: str | None, file_value: Path | None, *, field_name: str) -> str | None:
95
+ if text_value is not None and file_value is not None:
96
+ raise SystemExit(f"Only one of --{field_name} and --{field_name}-file may be set.")
97
+ if file_value is not None:
98
+ if not file_value.exists():
99
+ raise SystemExit(f"Missing {field_name} file: {file_value}")
100
+ return file_value.read_text(encoding="utf-8").strip()
101
+ if text_value is not None:
102
+ return text_value.strip()
103
+ return None
104
 
105
 
106
+ def build_chunk_all_prompt(task_instruction: str, task_plan: str | None) -> str:
107
+ task_instruction = task_instruction.strip()
108
+ task_plan = (task_plan or "").strip()
109
+ task_and_plan = task_instruction if not task_plan else f"{task_instruction}\n{task_plan}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
110
  return (
111
  f"任务描述和具体规划: {task_and_plan}\n\n"
112
  "请根据任务描���和具体规划,找到并逐点生成视频中的关键动作点和相应的进度标注。"
 
116
  )
117
 
118
 
119
+ def resolve_prompt(args: argparse.Namespace) -> tuple[str, str, str | None, str | None]:
120
+ prompt = read_text_value(args.prompt, args.prompt_file, field_name="prompt")
121
+ task_plan = read_text_value(args.task_plan, args.task_plan_file, field_name="task-plan")
122
+ task_instruction = args.task_instruction.strip() if args.task_instruction else None
123
+
124
+ if prompt:
125
+ return prompt, "raw_prompt", task_instruction, task_plan
126
+
127
+ if not task_instruction:
128
+ raise SystemExit(
129
+ "Either provide --prompt/--prompt-file, or provide --task-instruction "
130
+ "with optional --task-plan/--task-plan-file."
131
+ )
132
+
133
+ return build_chunk_all_prompt(task_instruction, task_plan), "task_instruction_plus_plan", task_instruction, task_plan
134
+
135
+
136
+ def compute_sampled_indices(num_frames: int, fps: float, sample_hz: float) -> tuple[list[int], list[float]]:
137
+ if num_frames <= 0:
138
+ raise SystemExit(f"Invalid decoded frame count: {num_frames}")
139
+ if fps <= 0:
140
+ raise SystemExit(f"Invalid decoded fps: {fps}")
141
+ if sample_hz <= 0:
142
+ raise SystemExit(f"sample_hz must be positive, got {sample_hz}")
143
+
144
  frame_ids = list(range(num_frames))
145
  eligible_arr = np.array(frame_ids, dtype=float)
146
+ duration_sec = max(0.0, (num_frames - 1) / fps)
 
 
147
  step_sec = 1.0 / sample_hz
148
 
149
  target_times: list[float] = []
 
159
  sampled_timestamps_sec: list[float] = []
160
  seen = set()
161
  for target_time in target_times:
162
+ target_frame = target_time * fps
163
  pos = int(np.argmin(np.abs(eligible_arr - target_frame)))
164
  idx = int(eligible_arr[pos])
165
  if idx in seen:
166
  continue
167
  seen.add(idx)
168
  sampled_indices.append(idx)
169
+ sampled_timestamps_sec.append(round(idx / fps, 6))
 
 
 
170
  return sampled_indices, sampled_timestamps_sec
171
 
172
 
173
  def extract_sampled_frame_paths(
174
  video_path: Path,
175
+ sample_hz: float,
176
  *,
177
  temp_dir: Path,
178
+ ) -> tuple[list[str], dict[str, float], list[int], list[float]]:
179
  try:
180
  from decord import VideoReader, cpu
181
  except ImportError as exc:
182
+ raise SystemExit("Missing dependency: decord is required for video sampling in quick_start.") from exc
183
 
184
  vr = VideoReader(str(video_path), ctx=cpu(0), num_threads=1)
185
  actual_frame_count = len(vr)
186
+ actual_fps = float(vr.get_avg_fps())
187
+ sampled_indices, sampled_timestamps_sec = compute_sampled_indices(actual_frame_count, actual_fps, sample_hz)
188
  if not sampled_indices:
189
  raise SystemExit("No sampled frame indices were generated.")
190
  if sampled_indices[-1] >= actual_frame_count:
191
  raise SystemExit(
192
+ f"Decoded video is shorter than expected. Need frame index {sampled_indices[-1]}, "
193
+ f"got {actual_frame_count} frames."
194
  )
195
 
196
  batch = vr.get_batch(sampled_indices).asnumpy()
 
202
 
203
  video_stats = {
204
  "decoded_frame_count": float(actual_frame_count),
205
+ "decoded_avg_fps": actual_fps,
206
+ "decoded_duration_sec": round(max(0.0, (actual_frame_count - 1) / actual_fps), 6),
207
  }
208
+ return frame_paths, video_stats, sampled_indices, sampled_timestamps_sec
209
 
210
 
211
  def load_swift_runtime():
 
214
  os.environ.setdefault("VIDEO_MAX_TOKEN_NUM", "256")
215
  os.environ.setdefault("VIDEO_MIN_TOKEN_NUM", "4")
216
  os.environ.setdefault("QWEN_VL_UTILS_MAX_FRAME_LIST", "0")
217
+
218
+ try:
219
+ import torch # noqa: F401
220
+ except ImportError as exc:
221
+ raise SystemExit("Missing dependency: torch is required for quick_start inference.") from exc
222
+
223
  try:
224
  from swift.llm import InferRequest, PtEngine, RequestConfig
225
  except ImportError:
226
  from swift.infer_engine import InferRequest, RequestConfig, TransformersEngine as PtEngine # type: ignore
227
+ return InferRequest, PtEngine, RequestConfig
228
+
229
+
230
+ def strip_code_fence(text: str) -> str:
231
+ cleaned = str(text or "").strip()
232
+ if cleaned.startswith("```") and cleaned.endswith("```"):
233
+ lines = cleaned.splitlines()
234
+ if len(lines) >= 3:
235
+ return "\n".join(lines[1:-1]).strip()
236
+ return cleaned
237
 
238
 
239
  def dedupe_sorted_points(times: list[float], values: list[float]) -> tuple[list[float], list[float]]:
 
245
  cur_time = pairs[0][0]
246
  bucket: list[float] = []
247
  for time_val, progress_val in pairs:
248
+ if abs(time_val - cur_time) > 1e-9:
249
  out_times.append(float(cur_time))
250
  out_values.append(float(sum(bucket) / len(bucket)))
251
  cur_time = time_val
 
284
  return dedupe_sorted_points(times, values)
285
 
286
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
287
  def main() -> int:
288
  args = parse_args()
289
+ video_path = args.video_path.resolve()
290
+ model_path = args.model_path.resolve()
 
 
 
 
 
 
 
291
  if not video_path.exists():
292
  raise SystemExit(f"Missing video: {video_path}")
293
+ if not model_path.exists():
294
+ raise SystemExit(f"Missing model path: {model_path}")
295
 
296
+ prompt, prompt_source, task_instruction, task_plan = resolve_prompt(args)
297
+ output_path = args.output_jsonl or (THIS_DIR / "outputs" / f"{video_path.stem}.jsonl")
 
 
 
 
 
 
 
 
 
298
  output_path.parent.mkdir(parents=True, exist_ok=True)
299
 
300
+ InferRequest, PtEngine, RequestConfig = load_swift_runtime()
 
 
 
 
 
 
 
 
301
  engine = PtEngine(
302
+ str(model_path),
303
  model_type="qwen3_moe_vl",
304
  max_batch_size=1,
305
+ device_map="auto",
306
  )
307
 
308
+ with tempfile.TemporaryDirectory(prefix="vlac_quick_start_frames_") as temp_dir_str:
309
  temp_dir = Path(temp_dir_str)
310
+ frame_paths, video_stats, sampled_indices, sampled_timestamps_sec = extract_sampled_frame_paths(
311
  video_path,
312
+ args.sample_hz,
313
  temp_dir=temp_dir,
314
  )
315
  request = InferRequest(
 
329
  )[0].choices[0].message.content
330
 
331
  pred_point_times_sec, pred_point_progress = parse_point_blocks(response)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
332
  output_row = {
333
+ "video_path": str(video_path),
334
+ "task_instruction": task_instruction,
335
+ "task_plan": task_plan,
336
+ "prompt_source": prompt_source,
 
 
337
  "prompt_variant": "chunk_all",
338
+ "input_sample_hz": float(args.sample_hz),
339
+ "input_frame_indices": sampled_indices,
340
+ "input_timestamps_sec": sampled_timestamps_sec,
341
+ "input_frame_count": len(sampled_indices),
342
  "decoded_video_stats": video_stats,
343
  "prompt": prompt,
344
  "response": response,
 
 
 
 
 
345
  "pred_curve_parse_ok": bool(pred_point_progress),
346
  "pred_curve_point_times_sec": pred_point_times_sec,
347
  "pred_curve_point_progress": pred_point_progress,
 
 
 
348
  }
349
  output_path.write_text(json.dumps(output_row, ensure_ascii=False) + "\n", encoding="utf-8")
350
 
351
+ print(f"video_path={video_path}")
 
352
  print(f"output_jsonl={output_path.resolve()}")
353
+ print(f"input_sample_hz={float(args.sample_hz):.4f}")
354
+ print(f"input_frame_count={len(sampled_indices)}")
355
  print(f"pred_keypoint_count={len(pred_point_progress)}")
 
 
 
 
 
 
 
356
  print()
357
  print(response)
358
  return 0