| |
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import math |
| import os |
| import re |
| import tempfile |
| from pathlib import Path |
| from typing import Any |
|
|
| import numpy as np |
| from PIL import Image |
|
|
| THIS_DIR = Path(__file__).resolve().parent |
| MODEL_ROOT = THIS_DIR.parent |
| EXAMPLES_ROOT = MODEL_ROOT / "examples" |
| MANIFEST_PATH = EXAMPLES_ROOT / "examples_manifest.json" |
| CHUNK_ALL_SAMPLE_HZ = 2.0 |
|
|
| SIGNED_NUM = r"([+-]?[0-9]+(?:\.[0-9]+)?)" |
| POINT_TIME_RE = re.compile(r"(?:Time|时间)[:：]?\s*([0-9]+(?:\.[0-9]+)?)\s*s?", re.IGNORECASE) |
| POINT_PROGRESS_LINE_RE = re.compile(rf"(?im)^\s*(?:Progress|进度)[:：]?\s*{SIGNED_NUM}\s*%") |
| INLINE_POINT_RE = re.compile( |
| rf"(?:Time|时间)[:：]?\s*([0-9]+(?:\.[0-9]+)?)\s*s?\s*[,，]?\s*(?:Progress|进度)[:：]?\s*{SIGNED_NUM}\s*%", |
| re.IGNORECASE, |
| ) |
|
|
|
|
| def parse_args() -> argparse.Namespace: |
| parser = argparse.ArgumentParser( |
| description="Run the released VLAC model on one bundled example with chunk_all prompt and fixed 2hz video input." |
| ) |
| parser.add_argument( |
| "--model-path", |
| type=Path, |
| default=MODEL_ROOT, |
| help="Path to the released VLAC model directory.", |
| ) |
| parser.add_argument( |
| "--example-id", |
| type=str, |
| required=True, |
| help="Bundled example id, for example: example_01", |
| ) |
| parser.add_argument( |
| "--output-jsonl", |
| type=Path, |
| default=None, |
| help="Optional output path. Defaults to quick_start/outputs/<example_id>.jsonl", |
| ) |
| parser.add_argument( |
| "--max-new-tokens", |
| type=int, |
| default=1024, |
| help="Generation cap for the response.", |
| ) |
| return parser.parse_args() |
|
|
|
|
| def load_manifest() -> dict[str, dict]: |
| payload = json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) |
| return {item["example_id"]: item for item in payload.get("examples", [])} |
|
|
|
|
| def strip_code_fence(text: str) -> str: |
| cleaned = str(text or "").strip() |
| if cleaned.startswith("```") and cleaned.endswith("```"): |
| lines = cleaned.splitlines() |
| if len(lines) >= 3: |
| return "\n".join(lines[1:-1]).strip() |
| return cleaned |
|
|
|
|
| def extract_plan_text(task_instruction: str, task_description: str) -> str: |
| lines = [line.strip() for line in str(task_description or "").splitlines() if line.strip()] |
| if not lines: |
| return "" |
| first_line = lines[0].rstrip("：:") |
| normalized_instruction = str(task_instruction or "").strip().rstrip("：:") |
| if normalized_instruction and first_line == normalized_instruction: |
| lines = lines[1:] |
| return "\n".join(lines).strip() |
|
|
|
|
| def build_chunk_all_prompt(metadata: dict[str, Any]) -> str: |
| task_instruction = str(metadata.get("task_instruction") or "").strip() |
| task_description = str(metadata.get("task_description") or "").strip() |
| plan_text = extract_plan_text(task_instruction, task_description) |
| task_and_plan = task_instruction if not plan_text else f"{task_instruction}\n{plan_text}" |
| return ( |
| f"任务描述和具体规划: {task_and_plan}\n\n" |
| "请根据任务描述和具体规划，找到并逐点生成视频中的关键动作点和相应的进度标注。" |
| "输出格式要求：每个关键点一行，格式为：\n" |
| "时间: X.Xs, 进度: Y%\n\n" |
| "请严格按照上述格式输出，不要输出额外说明。" |
| ) |
|
|
|
|
| def compute_sampled_indices_2hz(num_frames: int, fps: float, sample_hz: float) -> tuple[list[int], list[float]]: |
| if num_frames <= 0 or fps <= 0 or sample_hz <= 0: |
| return [], [] |
| frame_ids = list(range(num_frames)) |
| eligible_arr = np.array(frame_ids, dtype=float) |
| start_frame = frame_ids[0] |
| end_frame = frame_ids[-1] |
| duration_sec = max(0.0, (end_frame - start_frame) / fps) |
| step_sec = 1.0 / sample_hz |
|
|
| target_times: list[float] = [] |
| current = 0.0 |
| eps = 1e-9 |
| while current <= duration_sec + eps: |
| target_times.append(round(current, 6)) |
| current += step_sec |
| if not target_times: |
| target_times = [0.0] |
|
|
| sampled_indices: list[int] = [] |
| sampled_timestamps_sec: list[float] = [] |
| seen = set() |
| for target_time in target_times: |
| target_frame = start_frame + target_time * fps |
| pos = int(np.argmin(np.abs(eligible_arr - target_frame))) |
| idx = int(eligible_arr[pos]) |
| if idx in seen: |
| continue |
| seen.add(idx) |
| sampled_indices.append(idx) |
| sampled_timestamps_sec.append(round((idx - start_frame) / fps, 6)) |
| if not sampled_indices: |
| sampled_indices = [0] |
| sampled_timestamps_sec = [0.0] |
| return sampled_indices, sampled_timestamps_sec |
|
|
|
|
| def extract_sampled_frame_paths( |
| video_path: Path, |
| sampled_indices: list[int], |
| *, |
| temp_dir: Path, |
| ) -> tuple[list[str], dict[str, float]]: |
| try: |
| from decord import VideoReader, cpu |
| except ImportError as exc: |
| raise SystemExit("Missing dependency: decord is required for 2hz frame sampling in quick_start.") from exc |
|
|
| vr = VideoReader(str(video_path), ctx=cpu(0), num_threads=1) |
| actual_frame_count = len(vr) |
| if not sampled_indices: |
| raise SystemExit("No sampled frame indices were generated.") |
| if sampled_indices[-1] >= actual_frame_count: |
| raise SystemExit( |
| f"Bundled video is shorter than expected. Need frame index {sampled_indices[-1]}, got {actual_frame_count} frames." |
| ) |
|
|
| batch = vr.get_batch(sampled_indices).asnumpy() |
| frame_paths: list[str] = [] |
| for idx, frame in zip(sampled_indices, batch, strict=True): |
| frame_path = temp_dir / f"frame_{idx:06d}.jpg" |
| Image.fromarray(frame).save(frame_path, quality=95) |
| frame_paths.append(str(frame_path.resolve())) |
|
|
| video_stats = { |
| "decoded_frame_count": float(actual_frame_count), |
| "decoded_avg_fps": float(vr.get_avg_fps()), |
| } |
| return frame_paths, video_stats |
|
|
|
|
| def load_swift_runtime(): |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") |
| os.environ.setdefault("IMAGE_MAX_TOKEN_NUM", "256") |
| os.environ.setdefault("VIDEO_MAX_TOKEN_NUM", "256") |
| os.environ.setdefault("VIDEO_MIN_TOKEN_NUM", "4") |
| os.environ.setdefault("QWEN_VL_UTILS_MAX_FRAME_LIST", "0") |
| import torch |
| try: |
| from swift.llm import InferRequest, PtEngine, RequestConfig |
| except ImportError: |
| from swift.infer_engine import InferRequest, RequestConfig, TransformersEngine as PtEngine |
| return torch, InferRequest, PtEngine, RequestConfig |
|
|
|
|
| def dedupe_sorted_points(times: list[float], values: list[float]) -> tuple[list[float], list[float]]: |
| if not times: |
| return [], [] |
| pairs = sorted(zip(times, values), key=lambda item: (item[0], item[1])) |
| out_times: list[float] = [] |
| out_values: list[float] = [] |
| cur_time = pairs[0][0] |
| bucket: list[float] = [] |
| for time_val, progress_val in pairs: |
| if not math.isclose(time_val, cur_time, rel_tol=0.0, abs_tol=1e-9): |
| out_times.append(float(cur_time)) |
| out_values.append(float(sum(bucket) / len(bucket))) |
| cur_time = time_val |
| bucket = [float(progress_val)] |
| else: |
| bucket.append(float(progress_val)) |
| out_times.append(float(cur_time)) |
| out_values.append(float(sum(bucket) / len(bucket))) |
| return out_times, out_values |
|
|
|
|
| def parse_point_blocks(text: str) -> tuple[list[float], list[float]]: |
| cleaned = strip_code_fence(text) |
| if not cleaned: |
| return [], [] |
|
|
| inline_matches = INLINE_POINT_RE.findall(cleaned) |
| if inline_matches: |
| return dedupe_sorted_points( |
| [float(time_val) for time_val, _ in inline_matches], |
| [float(progress_val) for _, progress_val in inline_matches], |
| ) |
|
|
| blocks = re.split(r"(?=(?:Time|时间)[:：]?\s*[0-9])", cleaned, flags=re.IGNORECASE) |
| times: list[float] = [] |
| values: list[float] = [] |
| for block in blocks: |
| block = block.strip() |
| if not block: |
| continue |
| time_match = POINT_TIME_RE.search(block) |
| progress_match = POINT_PROGRESS_LINE_RE.search(block) |
| if time_match and progress_match: |
| times.append(float(time_match.group(1))) |
| values.append(float(progress_match.group(1))) |
| return dedupe_sorted_points(times, values) |
|
|
|
|
| def align_curve_to_gt_dense( |
| point_times_sec: list[float], |
| point_values: list[float], |
| gt_times_sec: list[float], |
| ) -> list[float]: |
| if not gt_times_sec or not point_values: |
| return [] |
| if len(point_values) == 1: |
| return [float(point_values[0])] * len(gt_times_sec) |
| times, values = dedupe_sorted_points(point_times_sec, point_values) |
| if len(values) == 1: |
| return [float(values[0])] * len(gt_times_sec) |
| aligned = np.interp( |
| np.array(gt_times_sec, dtype=float), |
| np.array(times, dtype=float), |
| np.array(values, dtype=float), |
| left=float(values[0]), |
| right=float(values[-1]), |
| ) |
| return [float(x) for x in aligned.tolist()] |
|
|
|
|
| def pearson_corr(xs: list[float], ys: list[float]) -> float | None: |
| if len(xs) != len(ys) or len(xs) < 2: |
| return None |
| x = np.array(xs, dtype=float) |
| y = np.array(ys, dtype=float) |
| if np.allclose(x, x[0]) or np.allclose(y, y[0]): |
| return None |
| value = float(np.corrcoef(x, y)[0, 1]) |
| if math.isnan(value) or not math.isfinite(value): |
| return None |
| return value |
|
|
|
|
| def average_ranks(values: list[float]) -> list[float]: |
| arr = np.array(values, dtype=float) |
| order = np.argsort(arr, kind="mergesort") |
| ranks = np.zeros(arr.shape[0], dtype=float) |
| idx = 0 |
| while idx < len(order): |
| next_idx = idx + 1 |
| while next_idx < len(order) and math.isclose( |
| arr[order[next_idx]], |
| arr[order[idx]], |
| rel_tol=0.0, |
| abs_tol=1e-9, |
| ): |
| next_idx += 1 |
| ranks[order[idx:next_idx]] = (idx + next_idx - 1) / 2.0 + 1.0 |
| idx = next_idx |
| return ranks.tolist() |
|
|
|
|
| def spearman_corr(xs: list[float], ys: list[float]) -> float | None: |
| return pearson_corr(average_ranks(xs), average_ranks(ys)) |
|
|
|
|
| def compute_curve_metrics(gt_dense_progress: list[float], pred_dense_progress: list[float]) -> dict[str, float | int | None]: |
| if not gt_dense_progress or len(gt_dense_progress) != len(pred_dense_progress): |
| return { |
| "point_count": 0, |
| "mae": None, |
| "rmse": None, |
| "pearson": None, |
| "spearman": None, |
| } |
| gt_arr = np.array(gt_dense_progress, dtype=float) |
| pred_arr = np.array(pred_dense_progress, dtype=float) |
| diff = pred_arr - gt_arr |
| return { |
| "point_count": int(len(gt_dense_progress)), |
| "mae": float(np.mean(np.abs(diff))), |
| "rmse": float(np.sqrt(np.mean(diff ** 2))), |
| "pearson": pearson_corr(pred_arr.tolist(), gt_arr.tolist()), |
| "spearman": spearman_corr(pred_arr.tolist(), gt_arr.tolist()), |
| } |
|
|
|
|
| def maybe_write_curve_plot( |
| *, |
| output_path: Path, |
| gt_times_sec: list[float], |
| gt_dense_progress: list[float], |
| pred_dense_progress: list[float], |
| pred_point_times_sec: list[float], |
| pred_point_progress: list[float], |
| ) -> str | None: |
| try: |
| import matplotlib.pyplot as plt |
| except ImportError: |
| return None |
|
|
| plot_path = output_path.with_name(f"{output_path.stem}_curve_compare.png") |
| fig, ax = plt.subplots(figsize=(10, 4.8)) |
| ax.plot(gt_times_sec, gt_dense_progress, color="#2563eb", linewidth=2.2, label="GT dense progress") |
| if pred_dense_progress: |
| ax.plot(gt_times_sec, pred_dense_progress, color="#f97316", linewidth=2.2, label="Pred aligned curve") |
| if pred_point_progress: |
| ax.scatter( |
| pred_point_times_sec, |
| pred_point_progress, |
| color="#111827", |
| s=28, |
| zorder=3, |
| label="Pred chunk_all points", |
| ) |
| ax.set_xlabel("Time (s)") |
| ax.set_ylabel("Progress (%)") |
| ax.grid(alpha=0.2, linewidth=0.8) |
| ax.legend(loc="best") |
| fig.tight_layout() |
| fig.savefig(plot_path, dpi=180) |
| plt.close(fig) |
| return str(plot_path.resolve()) |
|
|
|
|
| def format_metric(value: float | None) -> str: |
| if value is None: |
| return "N/A" |
| return f"{value:.4f}" |
|
|
|
|
| def main() -> int: |
| args = parse_args() |
| manifest = load_manifest() |
| if args.example_id not in manifest: |
| raise SystemExit(f"Unknown example id: {args.example_id}") |
|
|
| example_info = manifest[args.example_id] |
| metadata_path = MODEL_ROOT / example_info["metadata_path"] |
| video_path = MODEL_ROOT / example_info["video_path"] |
| if not metadata_path.exists(): |
| raise SystemExit(f"Missing metadata: {metadata_path}") |
| if not video_path.exists(): |
| raise SystemExit(f"Missing video: {video_path}") |
|
|
| metadata = json.loads(metadata_path.read_text(encoding="utf-8")) |
| gt_dense_progress = [float(x) for x in list(metadata.get("benchmark_dense_progress") or [])] |
| fps = float(metadata.get("fps") or 0.0) |
| num_frames = int(metadata.get("num_frames") or len(gt_dense_progress)) |
| if not gt_dense_progress: |
| raise SystemExit("Missing benchmark_dense_progress in example metadata.") |
| if fps <= 0 or num_frames <= 0: |
| raise SystemExit(f"Invalid metadata fps/num_frames: fps={fps}, num_frames={num_frames}") |
|
|
| prompt = build_chunk_all_prompt(metadata) |
| output_path = args.output_jsonl or (THIS_DIR / "outputs" / f"{args.example_id}.jsonl") |
| output_path.parent.mkdir(parents=True, exist_ok=True) |
|
|
| sampled_indices_2hz, sampled_timestamps_sec_2hz = compute_sampled_indices_2hz( |
| num_frames=num_frames, |
| fps=fps, |
| sample_hz=CHUNK_ALL_SAMPLE_HZ, |
| ) |
| gt_times_sec = [round(frame_idx / fps, 6) for frame_idx in range(len(gt_dense_progress))] |
|
|
| torch, InferRequest, PtEngine, RequestConfig = load_swift_runtime() |
| device_map = "auto" |
| engine = PtEngine( |
| str(args.model_path.resolve()), |
| model_type="qwen3_moe_vl", |
| max_batch_size=1, |
| device_map=device_map, |
| ) |
|
|
| with tempfile.TemporaryDirectory(prefix=f"{args.example_id}_2hz_frames_") as temp_dir_str: |
| temp_dir = Path(temp_dir_str) |
| frame_paths, video_stats = extract_sampled_frame_paths( |
| video_path, |
| sampled_indices_2hz, |
| temp_dir=temp_dir, |
| ) |
| request = InferRequest( |
| messages=[ |
| { |
| "role": "user", |
| "content": [ |
| {"type": "video", "video": frame_paths}, |
| {"type": "text", "text": prompt}, |
| ], |
| } |
| ] |
| ) |
| response = engine.infer( |
| [request], |
| RequestConfig(max_tokens=args.max_new_tokens, temperature=0.0, top_k=1, top_p=1.0), |
| )[0].choices[0].message.content |
|
|
| pred_point_times_sec, pred_point_progress = parse_point_blocks(response) |
| pred_dense_progress = align_curve_to_gt_dense( |
| point_times_sec=pred_point_times_sec, |
| point_values=pred_point_progress, |
| gt_times_sec=gt_times_sec, |
| ) |
| curve_metrics = compute_curve_metrics(gt_dense_progress, pred_dense_progress) |
| plot_path = maybe_write_curve_plot( |
| output_path=output_path, |
| gt_times_sec=gt_times_sec, |
| gt_dense_progress=gt_dense_progress, |
| pred_dense_progress=pred_dense_progress, |
| pred_point_times_sec=pred_point_times_sec, |
| pred_point_progress=pred_point_progress, |
| ) |
|
|
| output_row = { |
| "example_id": args.example_id, |
| "bucket": metadata.get("bucket"), |
| "global_episode_id": metadata.get("global_episode_id"), |
| "task_instruction": metadata.get("task_instruction"), |
| "task_description": metadata.get("task_description"), |
| "video_path": str(video_path.resolve()), |
| "prompt_variant": "chunk_all", |
| "input_sample_hz": CHUNK_ALL_SAMPLE_HZ, |
| "input_frame_indices_2hz": sampled_indices_2hz, |
| "input_timestamps_sec_2hz": sampled_timestamps_sec_2hz, |
| "input_frame_count_2hz": len(sampled_indices_2hz), |
| "decoded_video_stats": video_stats, |
| "prompt": prompt, |
| "response": response, |
| "benchmark_progress_type": metadata.get("benchmark_progress_type"), |
| "benchmark_progress_source": metadata.get("benchmark_progress_source"), |
| "benchmark_semantic_anchors": metadata.get("benchmark_semantic_anchors"), |
| "gt_dense_progress": gt_dense_progress, |
| "gt_dense_timestamps_sec": gt_times_sec, |
| "pred_curve_parse_ok": bool(pred_point_progress), |
| "pred_curve_point_times_sec": pred_point_times_sec, |
| "pred_curve_point_progress": pred_point_progress, |
| "pred_dense_progress_aligned_to_gt": pred_dense_progress, |
| "curve_compare_metrics": curve_metrics, |
| "curve_compare_plot": plot_path, |
| } |
| output_path.write_text(json.dumps(output_row, ensure_ascii=False) + "\n", encoding="utf-8") |
|
|
| print(f"example_id={args.example_id}") |
| print(f"video_path={video_path.resolve()}") |
| print(f"output_jsonl={output_path.resolve()}") |
| print(f"input_sample_hz={CHUNK_ALL_SAMPLE_HZ}") |
| print(f"input_frame_count_2hz={len(sampled_indices_2hz)}") |
| print(f"pred_keypoint_count={len(pred_point_progress)}") |
| print(f"gt_dense_point_count={len(gt_dense_progress)}") |
| print(f"curve_mae={format_metric(curve_metrics['mae'])}") |
| print(f"curve_rmse={format_metric(curve_metrics['rmse'])}") |
| print(f"curve_pearson={format_metric(curve_metrics['pearson'])}") |
| print(f"curve_spearman={format_metric(curve_metrics['spearman'])}") |
| if plot_path is not None: |
| print(f"curve_compare_plot={plot_path}") |
| print() |
| print(response) |
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
