Spaces:
Running
Running
stouchinglhh
Claude
sync processor from GitLab c697b6e: 版本 1.19.4 / 前端 1.12.4 —— 服务端 i18n(首屏不再闪英文)、推理模块并入模型页、前端抽出 shared/、缓存键改用前端号
bb59211 | """视频 ↔ 数据 帧同步 —— 各数据流的帧数与 RGB 视频是否一致。 | |
| 为什么重要 | |
| ---------- | |
| UMI 的 action 按帧号对齐：第 i 行动作对应第 i 帧画面。任何一路数据与视频 | |
| 帧数不一致，都会让"看这一帧、预测下一步"的监督信号整体错位。 | |
| 与 ``video.frame_drop`` 的分工 | |
| ----------------------------- | |
| ``video.frame_drop`` 比的是 **视频帧数 vs parquet 总行数**（整体量）。 | |
| 本项比的是 **每一路数据流各自的帧数**（结构性）—— | |
| UMI 特有：Slam / gripper_state / force 是三条独立写入的列， | |
| 采集端丢帧时可能只有其中一路短了，总数却对得上。 | |
| 这也是用户提的"整个的数据有没有根据 RGB 视频进行帧同步"。 | |
| """ | |
| from __future__ import annotations | |
| from app.processing.cleaning.checks import ( | |
| CheckContext, CleaningCheck, register_check, | |
| Finding, PASS, WARN, FAIL, | |
| ) | |
| class UmiAvSyncCheck(CleaningCheck): | |
| slug = "umi.av_sync" | |
| label = "Video ↔ Data Frame Sync" | |
| version = "1.0" | |
| description = "Frame counts differ between data streams" | |
| # 需要视频和至少一路数据同时存在才有意义。 | |
| # ★ 不加 cross_modal —— 它比的是【同一个 UMI 设备内部】的视频与各数据列， | |
| # 不是两张设备卡片之间。requires_channels 已经把"必须有 rgb"表达清楚了， | |
| # 再加 cross_modal 会把它错误地挤进"跨设备" tab。 | |
| requires_channels = ("rgb", "time") | |
| # 属于哪些设备卡片 —— 前端设置面板按它分 tab | |
| device_cards = ("gripper_device",) | |
| default_params = {"tolerance": 2} | |
| default_severity = FAIL | |
| def run(self, ctx: CheckContext) -> list[Finding]: | |
| # 视频侧：取第一路 RGB 的帧数作为基准 | |
| rgb_streams = ctx.streams_of("rgb") | |
| if not rgb_streams: | |
| return [] | |
| reference = None | |
| reference_key = "" | |
| for stream in rgb_streams: | |
| frames = int((stream.video or {}).get("frame_count") or stream.rows or 0) | |
| if frames: | |
| reference, reference_key = frames, stream.key | |
| break | |
| if not reference: | |
| return [] | |
| tolerance = int(self.param(ctx, "tolerance", 2)) | |
| drifted: dict[str, int] = {} | |
| for stream in ctx.streams: | |
| if stream.channel in ("rgb", "depth") or not stream.rows: | |
| continue | |
| if stream.key == reference_key: | |
| continue | |
| diff = abs(int(stream.rows) - reference) | |
| if diff > tolerance: | |
| drifted[stream.key] = int(stream.rows) | |
| metrics = {"reference_stream": reference_key, | |
| "reference_frames": reference, | |
| "tolerance": tolerance, | |
| "drifted": drifted} | |
| if not drifted: | |
| return [self.finding(ctx, PASS, metrics=metrics)] | |
| detail = "，".join(f"{key}={rows}" for key, rows in sorted(drifted.items())) | |
| return [self.finding( | |
| ctx, | |
| FAIL, | |
| message=f"these streams do not match the video frame count ({reference}): {detail}", | |
| metrics=metrics, | |
| )] | |