Spaces:
Sleeping
Sleeping
| """Thin adapter wrapper around `RoboInterVQADataset` so it can be plumbed | |
| through the existing `build_dataset` → `TransformedAdapterDataset` pipeline. | |
| VQA repos do not have a LeRobot info.json. Detection in `create_adapter`: | |
| the repo root must contain a `meta/vqa_manifest.json` file describing which | |
| JSON files to load and which task family they belong to. | |
| `vqa_manifest.json` schema (minimal): | |
| { | |
| "task_family": "understanding" | "generation_short" | "generation_traj" | |
| | "task_planning", | |
| "image_root_subdir": "Understanding/image/train/droid", // relative to repo root | |
| "json_files": [ | |
| "Understanding/meta/train/droid/contact_decide.json", | |
| "Understanding/meta/train/droid/grounding_choice.json" | |
| ] | |
| } | |
| The image_root_subdir resolves relative paths in the JSON `images` field | |
| (LLaVA-style path). Image paths inside JSON typically include the same prefix | |
| already, so `image_root_subdir` is usually the repo root itself (""). | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import os | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from typing import Any | |
| from .base import BaseAdapter | |
| class _VQAAdapterMeta: | |
| """Mimic the ``DatasetMeta`` interface that build_dataset uses. | |
| Satisfies the full surface (``total_episodes``, ``total_frames``, | |
| ``camera_keys``, ``episodes``, ``require_schema()``) with minimal correct | |
| values: VQA has no robot trajectory (one "episode" per sample, no shard | |
| frame ranges, cameras from the schema's ``image_mapping``). | |
| """ | |
| schema: Any | |
| stats: dict | |
| fps: float = 1.0 | |
| robot_type: str = "vqa_no_robot" | |
| total_episodes: int = 0 | |
| total_frames: int = 0 | |
| video_keys: tuple = () | |
| camera_keys: tuple = () | |
| features: dict = None | |
| # MultiLeRobotDataset reads meta.episodes["dataset_from_index"/"to_index"]. | |
| # VQA samples aren't packed into shards, so expose empty parallel arrays. | |
| episodes: dict = None | |
| def __post_init__(self): | |
| if self.features is None: | |
| self.features = {} | |
| if self.episodes is None: | |
| self.episodes = { | |
| "dataset_from_index": [], | |
| "dataset_to_index": [], | |
| } | |
| def require_schema(self): | |
| """Return ``self.schema`` or raise if it is still ``None``. | |
| Mirrors ``DatasetMeta.require_schema`` so generic code can enforce the | |
| schema-present invariant uniformly across LeRobot and VQA adapters. | |
| """ | |
| if self.schema is None: | |
| from src.schema.errors import SchemaDiscoveryError | |
| raise SchemaDiscoveryError( | |
| "_VQAAdapterMeta.schema is None after adapter construction. " | |
| "VQAAdapter must resolve a schema (override_schema, or " | |
| "manifest schema_name) before handing meta to the pipeline." | |
| ) | |
| return self.schema | |
| class VQAAdapter(BaseAdapter): | |
| """Adapter wrapping RoboInterVQADataset, satisfying the LeRobot adapter | |
| surface that build_dataset uses (.meta with .schema/.stats, .num_episodes, | |
| .num_frames, __len__/__getitem__). A ``BaseAdapter`` subclass so generic | |
| factory consumers get the full documented contract. | |
| """ | |
| def __init__( | |
| self, | |
| repo_id: str, | |
| root: str, | |
| override_schema: Any = None, | |
| # Other adapter kwargs accepted for API compatibility (ignored by VQA): | |
| delta_timestamps: dict | None = None, | |
| image_transforms=None, | |
| external_stats: dict | None = None, | |
| use_external_stats: bool = False, | |
| video_backend: str | None = None, | |
| episode_filter=None, | |
| ): | |
| self.repo_id = repo_id | |
| self.root = Path(root) | |
| # VQA normalization stats are the built-in identity VQA_STATS | |
| # (state/action are zero-filled). Fail loud rather than silently ignore | |
| # an external-stats override so callers don't believe it took effect. | |
| if external_stats or use_external_stats: | |
| raise ValueError( | |
| "VQAAdapter does not support external_stats: VQA samples have " | |
| "zero-filled state/action and MUST use the built-in identity " | |
| f"VQA_STATS (got external_stats={external_stats!r}, " | |
| f"use_external_stats={use_external_stats!r}). Remove the " | |
| "external-stats override for VQA repos, or exclude this repo " | |
| "from external-stats resolution." | |
| ) | |
| manifest_path = self.root / "meta" / "vqa_manifest.json" | |
| if not manifest_path.is_file(): | |
| raise FileNotFoundError( | |
| f"VQAAdapter expects {manifest_path} (vqa_manifest.json missing). " | |
| f"See src/adapters/vqa_adapter.py docstring for schema." | |
| ) | |
| with open(manifest_path) as f: | |
| manifest = json.load(f) | |
| # Resolve schema: caller override > manifest `schema_name`. Require an | |
| # explicit schema_name (no silent default) so a corpus with different | |
| # semantics isn't bound to the wrong schema. | |
| if override_schema is not None: | |
| schema = override_schema | |
| else: | |
| from src.schema.registry import resolve as _resolve_schema | |
| schema_name = manifest.get("schema_name") | |
| if not schema_name: | |
| raise ValueError( | |
| f"VQA manifest {manifest_path} is missing required " | |
| "'schema_name'. Add an explicit schema_name (e.g. " | |
| "\"robointer_vqa\") to the manifest, or pass " | |
| "override_schema. Refusing to default silently so a " | |
| "non-robointer_vqa corpus is not bound to the wrong schema." | |
| ) | |
| schema = _resolve_schema(schema_name) | |
| from src.dataset.adapters.robointer_vqa_adapter import RoboInterVQADataset | |
| json_paths = [ | |
| os.path.join(root, jp) for jp in manifest["json_files"] | |
| ] | |
| image_root = os.path.join(root, manifest.get("image_root_subdir", "")) | |
| task_family = manifest["task_family"] | |
| self._dataset = RoboInterVQADataset( | |
| json_paths=json_paths, | |
| image_root=image_root, | |
| task_family=task_family, | |
| schema=schema, | |
| ) | |
| # One "episode" per VQA sample so PI0Mixture n^0.43 weighting and the | |
| # inherited num_episodes/num_frames properties (read from meta) work. | |
| n = len(self._dataset) | |
| # VQA has no on-disk video; the only image slot is the schema's | |
| # image_mapping target(s) (e.g. observation.images.primary). | |
| camera_keys = tuple( | |
| getattr(schema, "image_mapping", None) or {} | |
| ) | |
| # Identity stats: state/action are zero-filled so identity normalization | |
| # leaves them at zero. Populating observation.state/action/action_abs | |
| # also satisfies the action_mode='abs' guard. | |
| from schemas.robointer_vqa import VQA_STATS as _VQA_STATS | |
| self.meta = _VQAAdapterMeta( | |
| schema=schema, | |
| stats=dict(_VQA_STATS), | |
| total_episodes=n, | |
| total_frames=n, | |
| camera_keys=camera_keys, | |
| ) | |
| def __len__(self) -> int: | |
| return len(self._dataset) | |
| def __getitem__(self, idx: int) -> dict: | |
| return self._dataset[idx] | |