#!/usr/bin/env python3 """Discover only the OraRL paper evaluation sources and write a private manifest.""" from __future__ import annotations import argparse import csv import json import os import sys import tempfile from collections.abc import Iterable, Mapping, Sequence from pathlib import Path from typing import Any PAPER_TASKS = ( "videomme", "videommev2", "mvbench", "mmvu", "videoholmes", "longvideobench", "mlvu", "vsi", "mmsi", "mindcube", "revsi", "spatial_grounding", "tracking", "stvg", "temporal_grounding", "segmentation", ) _SKIPPED_DIRECTORY_NAMES = frozenset( { ".git", "__pycache__", "checkpoint", "checkpoints", "logs", "lvbench", "onethinker-train-data", "output", "outputs", "train", "training", "videommmu", } ) _UPSTREAM_TERMS = "See the upstream benchmark terms" _MINDCUBE_EXPECTED_SAMPLES = 1050 _REVSI_EXPECTED_SAMPLES = 6808 _SEGMENTATION_EXPECTED_SAMPLES = { "mevis": 424, "reasonvos": 458, "refcoco": 3811, "refcocog": 2537, "refcocop": 3805, } _VIDEO_QA_SPECS: tuple[dict[str, Any], ...] = ( { "task": "videomme", "folder": "Video-MME", "split": "test", "annotation_tiers": ( ("videomme_preprocessed_384f_262k_total0.jsonl",), ("videomme.jsonl", "videomme.json"), ), "source_url": "https://github.com/BradyFU/Video-MME", "evaluation": { "prompt_profile": "videomme_default", "parser_profile": "multiple_choice", "metric_profile": "accuracy", }, "preprocessing": { "fps": 2, "max_frames": 384, "video_min_pixels": 4096, "video_max_pixels": 262144, "video_total_pixels": 0, }, "legacy": { "VIDEOMME_SETTING": ( "all-qwen3_vl-sub0-f384-fps2-min4096-max262144-total0-" "videomme_preprocessed_384f_262k_total0" ), "VIDEOMME_FPS": 2, "VIDEOMME_MAX_FRAMES": 384, "VIDEOMME_VIDEO_MIN_PIXELS": 4096, "VIDEOMME_VIDEO_MAX_PIXELS": 262144, "VIDEOMME_VIDEO_TOTAL_PIXELS": 0, }, }, { "task": "videommev2", "folder": "Video-MME-v2", "split": "test", "annotation_tiers": ( ("videommev2_preprocessed_384f_262k_total0.jsonl",), ("videommev2.jsonl", "videommev2.json"), ), "source_url": "https://github.com/BradyFU/Video-MME", "evaluation": { "prompt_profile": "videommev2_default", "parser_profile": "multiple_choice", "metric_profile": "accuracy", }, "preprocessing": { "fps": 2, "max_frames": 384, "video_min_pixels": 4096, "video_max_pixels": 262144, "video_total_pixels": 0, }, "legacy": { "VIDEOMMEV2_PROMPT_MODE": "default", "VIDEOMMEV2_ANSWER_FILTER": "", "VIDEOMMEV2_FPS": 2, "VIDEOMMEV2_MAX_FRAMES": 384, "VIDEOMMEV2_VIDEO_MIN_PIXELS": 4096, "VIDEOMMEV2_VIDEO_MAX_PIXELS": 262144, "VIDEOMMEV2_VIDEO_TOTAL_PIXELS": 0, }, }, { "task": "mvbench", "folder": "MVBench", "split": "test", "annotation_tiers": (("mvbench.json", "mvbench.jsonl"),), "source_url": "https://github.com/OpenGVLab/Ask-Anything", "evaluation": { "prompt_profile": "mvbench_default", "parser_profile": "multiple_choice", "metric_profile": "accuracy", }, }, { "task": "mmvu", "folder": "MMVU", "split": "test", "annotation_tiers": ( ("mmvu_mc.jsonl",), ("mmvu-mc.json", "mmvu_mc.json", "mmvu.jsonl"), ), "source_url": "https://github.com/yale-nlp/MMVU", "evaluation": { "prompt_profile": "mmvu_default", "parser_profile": "multiple_choice", "metric_profile": "accuracy", }, }, { "task": "videoholmes", "folder": "Video-Holmes", "split": "test", "annotation_tiers": (("videoholmes.jsonl", "videoholmes.json"),), "source_url": "https://github.com/TencentARC/Video-Holmes", "evaluation": { "prompt_profile": "videoholmes_default", "parser_profile": "multiple_choice", "metric_profile": "accuracy", }, }, { "task": "longvideobench", "folder": "LongVideoBench", "split": "val", "annotation_tiers": ( ("longvideobench_val.jsonl",), ("longvideobench.jsonl", "longvideobench.json"), ), "source_url": "https://github.com/longvideobench/LongVideoBench", "evaluation": { "prompt_profile": "longvideobench_default", "parser_profile": "multiple_choice", "metric_profile": "accuracy", }, "legacy": {"LONGVIDEOBENCH_USE_SUBTITLES": True}, }, { "task": "mlvu", "folder": "MLVU", "split": "dev", "annotation_tiers": ( ("mlvu_mc.jsonl",), ("mlvu-dev.json", "mlvu_mc.json", "mlvu.json"), ), "source_url": "https://github.com/JUNJIE99/MLVU", "evaluation": { "prompt_profile": "mlvu_default", "parser_profile": "multiple_choice", "metric_profile": "macro_accuracy", }, }, ) _COMMON_VIDEO_SETTINGS = { "batch_size": 1, "enable_thinking": False, "fps": 2, "gpu_memory_utilization": 0.9, "max_frames": 384, "max_model_len": 65536, "max_new_tokens": 128, "max_num_batched_tokens": 65536, "min_p": 0.0, "presence_penalty": 0.0, "prompt_mode": "default", "temperature": 0.0, "top_k": -1, "top_p": 1.0, "video_max_pixels": 262144, "video_min_pixels": 4096, "video_total_pixels": 0, } class DiscoveryError(ValueError): """Raised when a required paper source is missing or ambiguous.""" def _is_within(path: Path, directory: Path) -> bool: try: path.resolve(strict=False).relative_to(directory.resolve(strict=False)) except ValueError: return False return True def _required_directory(path: Path, label: str) -> Path: resolved = path.expanduser().resolve() if not resolved.is_dir(): raise DiscoveryError(f"{label} directory does not exist: {resolved}") return resolved def _walk_named_files(root: Path, names: Iterable[str]) -> list[Path]: wanted = set(names) matches: list[Path] = [] for current, directories, filenames in os.walk(root, followlinks=False): directories[:] = sorted( name for name in directories if name.casefold() not in _SKIPPED_DIRECTORY_NAMES ) current_path = Path(current) for filename in sorted(filenames): if filename in wanted: matches.append((current_path / filename).resolve()) return sorted(set(matches), key=str) def _walk_suffix_files(root: Path, suffixes: Iterable[str]) -> list[Path]: wanted = {suffix.casefold() for suffix in suffixes} matches: list[Path] = [] for current, directories, filenames in os.walk(root, followlinks=False): directories[:] = sorted( name for name in directories if name.casefold() not in _SKIPPED_DIRECTORY_NAMES ) current_path = Path(current) for filename in sorted(filenames): path = current_path / filename if path.suffix.casefold() in wanted: matches.append(path.resolve()) return sorted(set(matches), key=str) def _one_file(label: str, tiers: Sequence[Sequence[Path]]) -> Path: attempted: list[str] = [] for tier in tiers: candidates = sorted( {path.resolve() for path in tier if path.is_file()}, key=str, ) attempted.extend(str(path) for path in tier) if len(candidates) > 1: rendered = ", ".join(str(path) for path in candidates) raise DiscoveryError(f"ambiguous {label}; found: {rendered}") if candidates: return candidates[0] raise DiscoveryError(f"required {label} was not found; tried: {', '.join(attempted)}") def _runtime_annotation( runtime_root: Path, source_root: Path, label: str, name_tiers: Sequence[Sequence[str]], ) -> Path: valid_data = runtime_root / "eval" / "data" / "valid_data" tiers: list[list[Path]] = [] for names in name_tiers: tiers.append([valid_data / name for name in names]) for names in name_tiers: tiers.append(_walk_named_files(source_root, names)) return _one_file(label, tiers) def _named_annotation(root: Path, label: str, names: Sequence[str]) -> Path: return _one_file(label, [_walk_named_files(root, names)]) def _first_directory(label: str, tiers: Sequence[Sequence[Path]]) -> Path: attempted: list[str] = [] for tier in tiers: candidates = sorted( {path.resolve() for path in tier if path.is_dir()}, key=str, ) attempted.extend(str(path) for path in tier) if len(candidates) > 1: rendered = ", ".join(str(path) for path in candidates) raise DiscoveryError(f"ambiguous {label}; found: {rendered}") if candidates: return candidates[0] raise DiscoveryError(f"required {label} directory was not found; tried: {', '.join(attempted)}") def _cheap_expected_count(path: Path) -> int | None: suffix = path.suffix.casefold() if suffix in {".jsonl", ".ndjson"}: with path.open("r", encoding="utf-8") as handle: return sum(bool(line.strip()) for line in handle) if suffix == ".tsv": with path.open("r", encoding="utf-8", errors="replace", newline="") as handle: csv.field_size_limit(sys.maxsize) return sum(1 for _row in csv.DictReader(handle, delimiter="\t")) if suffix == ".parquet": try: import pyarrow.parquet as parquet except ImportError: return None try: return int(parquet.ParquetFile(path).metadata.num_rows) except (OSError, ValueError): return None return None def _mindcube_annotation(root: Path) -> Path: candidates = _walk_suffix_files(root, (".parquet",)) if not candidates: raise DiscoveryError( "official MindCube-Tiny parquet was not found under " f"{root}; expected {_MINDCUBE_EXPECTED_SAMPLES} rows" ) counted = [(path, _cheap_expected_count(path)) for path in candidates] official = [ path for path, count in counted if count == _MINDCUBE_EXPECTED_SAMPLES ] if len(official) == 1: return official[0] if len(official) > 1: rendered = ", ".join(str(path) for path in official) raise DiscoveryError( "ambiguous official MindCube-Tiny parquet; " f"found multiple {_MINDCUBE_EXPECTED_SAMPLES}-row files: {rendered}" ) canonical_unreadable = [ path for path, count in counted if count is None and path.name == "combined-00000-of-00001.parquet" ] if len(canonical_unreadable) == 1: return canonical_unreadable[0] rendered = ", ".join( f"{path} ({'unknown' if count is None else count} rows)" for path, count in counted ) raise DiscoveryError( f"official MindCube-Tiny requires {_MINDCUBE_EXPECTED_SAMPLES} rows; " f"found: {rendered}" ) def _revsi_annotation(root: Path) -> Path: annotation = root / "all_frame" / "test-00000-of-00001.parquet" if not annotation.is_file(): raise DiscoveryError( "official ReVSI all-frame annotation was not found; expected " f"{annotation}" ) count = _cheap_expected_count(annotation) if count is not None and count != _REVSI_EXPECTED_SAMPLES: raise DiscoveryError( f"official ReVSI all-frame split requires {_REVSI_EXPECTED_SAMPLES} rows; " f"found {count} in {annotation}" ) return annotation def _source_record( *, task: str, split: str, family: str, adapter: str, annotation: Path, source_url: str, evaluation: Mapping[str, Any], preprocessing: Mapping[str, Any] | None, legacy_environment: Mapping[str, Any] | None, media_roots: Mapping[str, Path] | None, preprocessed_roots: Mapping[str, Path] | None, authorized: bool, expected_count: int | None = None, ) -> dict[str, Any]: record: dict[str, Any] = { "benchmark": task, "eval_task": task, "split": split, "family": family, "adapter": adapter, "annotation_input": str(annotation.resolve()), "expected_count": ( expected_count if expected_count is not None else _cheap_expected_count(annotation) ), "license": _UPSTREAM_TERMS, "source_url": source_url, "redistribution_authorized": authorized, "evaluation": dict(evaluation), } if preprocessing: record["preprocessing"] = dict(preprocessing) if legacy_environment: record["legacy_environment"] = dict(legacy_environment) if media_roots: record["media_roots"] = { name: str(path.resolve()) for name, path in sorted(media_roots.items()) } if preprocessed_roots: record["preprocessed_roots"] = { name: str(path.resolve()) for name, path in sorted(preprocessed_roots.items()) } return record def _video_qa_records( data_root: Path, runtime_root: Path, *, authorized: bool, ) -> list[dict[str, Any]]: records: list[dict[str, Any]] = [] for spec in _VIDEO_QA_SPECS: task = str(spec["task"]) source_root = _required_directory(data_root / str(spec["folder"]), str(spec["folder"])) annotation = _runtime_annotation( runtime_root, source_root, f"{task} annotation", spec["annotation_tiers"], ) if task == "videomme": video_root = _first_directory( "videomme videos", ( (source_root / "data",), (source_root / "videos",), (source_root,), ), ) elif task == "mvbench": # MVBench annotations already include their dataset subdirectory # (for example, ``./star/...`` and ``./clevrer/...``). A separate # ``videos/`` directory may exist but is not the common parent of # those references. video_root = source_root else: video_root = source_root / "videos" if not video_root.is_dir(): video_root = source_root media_roots: dict[str, Path] = {"videos": video_root} if task == "longvideobench": subtitle_root = source_root / "subtitles" media_roots["subtitles"] = subtitle_root if subtitle_root.is_dir() else source_root preprocessed_roots: dict[str, Path] = {} if task in {"videomme", "videommev2"}: global_preprocessed = data_root / "preprocessed_videos" prepared_name = ( "videomme_preprocessed_384f_262k_total0" if task == "videomme" else "videommev2_preprocessed_384f_262k_total0" ) prepared = _first_directory( f"{task} preprocessed videos", ( (source_root / "preprocessed_videos_384f_262k_total0",), (source_root / "preprocessed_videos",), (global_preprocessed / prepared_name,), (global_preprocessed / task,), (global_preprocessed,), ), ) preprocessed_roots["default"] = prepared preprocessing = {**_COMMON_VIDEO_SETTINGS, **spec.get("preprocessing", {})} prefix = { "videomme": "VIDEOMME", "videommev2": "VIDEOMMEV2", "mvbench": "MVBENCH", "mmvu": "MMVU", "videoholmes": "VIDEOHOLMES", "longvideobench": "LONGVIDEOBENCH", "mlvu": "MLVU", }[task] legacy = {f"{prefix}_{key.upper()}": value for key, value in _COMMON_VIDEO_SETTINGS.items()} legacy.update(spec.get("legacy", {})) records.append( _source_record( task=task, split=str(spec["split"]), family="video_qa", adapter="generic", annotation=annotation, source_url=str(spec["source_url"]), evaluation=spec["evaluation"], preprocessing=preprocessing, legacy_environment=legacy, media_roots=media_roots, preprocessed_roots=preprocessed_roots, authorized=authorized, ) ) return records def _spatial_intelligence_records( data_root: Path, runtime_root: Path, *, authorized: bool, ) -> list[dict[str, Any]]: vsi_root = _required_directory(data_root / "VSI-Bench", "VSI-Bench") vsi_annotation = _runtime_annotation( runtime_root, vsi_root, "vsi annotation", ( ("vsibench_preprocessed_128f_16M.jsonl",), ("vsibench.jsonl", "vsibench.json"), ), ) global_preprocessed = data_root / "preprocessed_videos" vsi_preprocessed = _first_directory( "vsi preprocessed videos", ( ( vsi_root / "preprocessed_videos_128f_16M", vsi_root / "preprocessed_videos", ), ( global_preprocessed / "vsibench_preprocessed_128f_16M", global_preprocessed / "vsi", global_preprocessed / "VSI-Bench", ), (global_preprocessed,), ), ) records = [ _source_record( task="vsi", split="test", family="spatial_intelligence", adapter="generic", annotation=vsi_annotation, source_url="https://huggingface.co/datasets/nyu-visionx/VSI-Bench", evaluation={ "prompt_profile": "vsi_default", "parser_profile": "vsi_answer", "metric_profile": "vsi_official", }, preprocessing={ "fps": 2, "max_frames": 128, "video_min_pixels": 65536, "video_total_pixels": 16777216, }, legacy_environment={ "VSI_SETTING": ( "video128-16M-video-f128-fps2-min65536-maxnone-total16777216-" "vsibench_preprocessed_128f_16M" ), "VSI_BATCH_SIZE": 16, "VSI_MAX_MODEL_LEN": 32768, "VSI_MAX_NEW_TOKENS": 1024, "VSI_EXPECTED_SAMPLES": 5130, }, media_roots={"videos": vsi_root}, preprocessed_roots={"default": vsi_preprocessed}, authorized=authorized, ) ] mmsi_root = _required_directory(data_root / "MMSI-Bench", "MMSI-Bench") mmsi_annotation = _named_annotation( mmsi_root, "mmsi annotation", ("MMSI_bench.tsv",), ) records.append( _source_record( task="mmsi", split="test", family="spatial_intelligence", adapter="mmsi", annotation=mmsi_annotation, source_url="https://huggingface.co/datasets/RunsenXu/MMSI-Bench", evaluation={ "prompt_profile": "mmsi_default", "parser_profile": "multiple_choice", "metric_profile": "accuracy", }, preprocessing={"image_min_pixels": 4096, "image_max_pixels": 262144}, legacy_environment={ "MMSI_BACKEND": "transformers", "MMSI_MAX_IMAGES": 0, "MMSI_IMAGE_MIN_PIXELS": 4096, "MMSI_IMAGE_MAX_PIXELS": 262144, "MMSI_ENABLE_THINKING": False, }, media_roots=None, preprocessed_roots=None, authorized=authorized, ) ) mindcube_root = _required_directory(data_root / "MindCube-Tiny", "MindCube-Tiny") mindcube_annotation = _mindcube_annotation(mindcube_root) records.append( _source_record( task="mindcube", split="test", family="spatial_intelligence", adapter="mindcube", annotation=mindcube_annotation, source_url=( "https://huggingface.co/datasets/oscarqjh/MindCube_lmmseval/" "tree/7dd2725d9bd4149f2aad00a9843f72a3824da003" ), evaluation={ "prompt_profile": "mindcube_official", "parser_profile": "multiple_choice", "metric_profile": "micro_accuracy", "aggregation": "micro", "expected_group_counts": { "rotation": 200, "among": 600, "around": 250, }, }, preprocessing={"max_images": 4}, legacy_environment={ "MINDCUBE_MAX_IMAGES": 4, "MINDCUBE_BATCH_SIZE": 16, "MINDCUBE_MAX_MODEL_LEN": 32768, "MINDCUBE_MAX_NEW_TOKENS": 1024, }, media_roots={"images": mindcube_root}, preprocessed_roots=None, authorized=authorized, ) ) revsi_root = _required_directory(data_root / "ReVSI", "ReVSI") revsi_annotation = _revsi_annotation(revsi_root) revsi_frame_root = revsi_annotation.parent records.append( _source_record( task="revsi", split="test", family="spatial_intelligence", adapter="revsi", annotation=revsi_annotation, source_url="https://arxiv.org/abs/2605.25979", evaluation={ "prompt_profile": "revsi_official", "parser_profile": "revsi_answer", "metric_profile": "revsi_macro_average", "aggregation": "macro_by_question_type", "frame_protocol": "native_all_frame", }, preprocessing={ "fps": 2, "max_frames": 128, "exact_nframes": True, "video_min_pixels": 65536, "video_total_pixels": 16777216, }, legacy_environment={ "REVSI_SETTING": ( "native-all-f128-exacttrue-fps2-min65536-" "maxnone-total16777216" ), "REVSI_FRAME_BUDGET": "all", "REVSI_MAX_FRAMES": 128, "REVSI_EXACT_NFRAMES": True, "REVSI_FPS": 2, "REVSI_VIDEO_MIN_PIXELS": 65536, "REVSI_VIDEO_TOTAL_PIXELS": 16777216, "REVSI_BATCH_SIZE": 16, "REVSI_MAX_MODEL_LEN": 32768, "REVSI_MAX_NEW_TOKENS": 64, "REVSI_GPU_MEMORY_UTILIZATION": 0.9, "REVSI_EXPECTED_SAMPLES": _REVSI_EXPECTED_SAMPLES, "REVSI_ENABLE_THINKING": False, }, media_roots={"videos": revsi_frame_root}, preprocessed_roots=None, authorized=authorized, ) ) return records def _structured_records( data_root: Path, runtime_root: Path, *, authorized: bool, ) -> list[dict[str, Any]]: records: list[dict[str, Any]] = [] one_thinker_root = _required_directory( data_root / "OneThinker-eval", "OneThinker-eval", ) grounding_root = _required_directory( data_root / "Spatial-Grounding", "Spatial-Grounding", ) grounding_image_root = _first_directory( "Spatial Grounding images", ( ( one_thinker_root / "Refcoco", one_thinker_root / "RefCOCO", ), ( grounding_root / "train2014", grounding_root / "Refcoco", grounding_root / "RefCOCO", ), (grounding_root,), ), ) runtime_grounding_root = ( runtime_root / "eval" / "task" / "spatial_grounding" / "rec_jsons_processed" ) grounding_splits = ( ("refcoco_val", "refcoco_val.json", "refcoco-val"), ("refcoco_test_a", "refcoco_testA.json", "refcoco-testA"), ("refcoco_test_b", "refcoco_testB.json", "refcoco-testB"), ("refcocop_val", "refcocop_val.json", "refcoco+-val"), ("refcocop_test_a", "refcocop_testA.json", "refcoco+-testA"), ("refcocop_test_b", "refcocop_testB.json", "refcoco+-testB"), ("refcocog_val", "refcocog_val.json", "refcocog-val"), ("refcocog_test", "refcocog_test.json", "refcocog-test"), ) grounding_profile = { "prompt_profile": "qwen_native", "parser_profile": "norm1000_bbox", "metric_profile": "refcoco_iou", } grounding_legacy = { "SPATIAL_GROUNDING_DATASETS": ",".join( evaluator_name for _split, _filename, evaluator_name in grounding_splits ), "SPATIAL_GROUNDING_PROMPT_STYLE": "qwen_native", "SPATIAL_GROUNDING_COORD_SYSTEM": "norm1000", "SPATIAL_GROUNDING_BBOX_SELECT": "first", "SPATIAL_GROUNDING_MIN_TOKENS": 64, "SPATIAL_GROUNDING_TOTAL_TOKENS": 1024, "SPATIAL_GROUNDING_MAX_NEW_TOKENS": 1024, } for split, filename, _evaluator_name in grounding_splits: jsonl_filename = f"{Path(filename).stem}.jsonl" annotation = _one_file( f"spatial_grounding/{split} annotation", ( (runtime_grounding_root / filename,), (runtime_grounding_root / jsonl_filename,), _walk_named_files(grounding_root, (filename,)), _walk_named_files(grounding_root, (jsonl_filename,)), ), ) records.append( _source_record( task="spatial_grounding", split=split, family="spatial_grounding", adapter="spatial_grounding", annotation=annotation, source_url="https://huggingface.co/datasets/OneThink/OneThinker-eval", evaluation=grounding_profile, preprocessing={"coordinate_system": "norm1000"}, legacy_environment=grounding_legacy, media_roots={"images": grounding_image_root}, preprocessed_roots=None, authorized=authorized, ) ) one_thinker_specs = ( ( "tracking", "got10k", "eval_got10k.json", "tracking", { "prompt_profile": "tracking_default", "parser_profile": "tracking_boxes", "metric_profile": "got10k_ao", }, { "TRACKING_DATASETS": "eval_got10k", "TRACKING_MAX_FRAMES": 32, "TRACKING_FPS": 1, "TRACKING_VIDEO_MIN_PIXELS": 4096, "TRACKING_VIDEO_MAX_PIXELS": 786432, "TRACKING_VIDEO_TOTAL_PIXELS": 8388608, "TRACKING_MAX_NEW_TOKENS": 8192, "TRACKING_PROMPT_MODE": "default", }, ), ( "stvg", "stvg", "eval_stvg.json", "spatial_temporal_grounding", { "prompt_profile": "train_stvg", "parser_profile": "temporal_spatial_boxes", "metric_profile": "stvg_official", }, { "STVG_DATASETS": "eval_stvg", "STVG_MAX_FRAMES": 128, "STVG_FPS": 2, "STVG_VIDEO_MIN_PIXELS": 65536, "STVG_VIDEO_MAX_PIXELS": 393216, "STVG_VIDEO_TOTAL_PIXELS": 10485760, "STVG_MAX_NEW_TOKENS": 2048, "STVG_PROMPT_MODE": "train_stvg", }, ), ) for task, split, filename, family, evaluation, legacy in one_thinker_specs: annotation = _named_annotation( one_thinker_root, f"{task}/{split} annotation", (filename, f"{Path(filename).stem}.jsonl"), ) records.append( _source_record( task=task, split=split, family=family, adapter="one_thinker", annotation=annotation, source_url="https://huggingface.co/datasets/OneThink/OneThinker-eval", evaluation=evaluation, preprocessing={ key.casefold(): value for key, value in legacy.items() if key.endswith( ( "_FPS", "_MAX_FRAMES", "_VIDEO_MIN_PIXELS", "_VIDEO_MAX_PIXELS", "_VIDEO_TOTAL_PIXELS", ) ) }, legacy_environment=legacy, media_roots={"default": one_thinker_root}, preprocessed_roots=None, authorized=authorized, ) ) segmentation_splits = ( ("refcoco", "eval_seg_refcoco.json"), ("refcocop", "eval_seg_refcocop.json"), ("refcocog", "eval_seg_refcocog.json"), ("mevis", "eval_seg_mevis.json"), ("reasonvos", "eval_seg_reasonvos.json"), ) segmentation_legacy = { "SEGMENTATION_DATASETS": ",".join(split for split, _name in segmentation_splits), "SEGMENTATION_DATA_TYPE": "all", "SEGMENTATION_PROMPT_MODE": "train_seg", "SEGMENTATION_ENABLE_THINKING": False, "SEGMENTATION_MAX_FRAMES": 128, "SEGMENTATION_FPS": 2, "SEGMENTATION_VIDEO_READER": "decord", "SEGMENTATION_VIDEO_MIN_PIXELS": 4096, "SEGMENTATION_VIDEO_MAX_PIXELS": 262144, "SEGMENTATION_VIDEO_TOTAL_PIXELS": 16777216, "SEGMENTATION_MAX_PIXELS_IMAGE": 1048576, "SEGMENTATION_MIN_PIXELS_IMAGE": 4096, "SEGMENTATION_BATCH_SIZE": 16, "SEGMENTATION_MAX_MODEL_LEN": 32768, "SEGMENTATION_MAX_NEW_TOKENS": 1024, "SEGMENTATION_GPU_MEM_UTIL": 0.85, "SEGMENTATION_SEED": 42, "SEGMENTATION_RUN_SAM2": False, "SEGMENTATION_SETTING": ( "segmentation-refcoco_refcocop_refcocog_mevis_reasonvos-" "train_seg-f128-fps2-min4096-max262144-total16777216-" "readerdecord-new1024" ), } for split, filename in segmentation_splits: annotation = _named_annotation( one_thinker_root, f"segmentation/{split} annotation", (filename, f"{Path(filename).stem}.jsonl"), ) records.append( _source_record( task="segmentation", split=split, family="segmentation", adapter="one_thinker", annotation=annotation, source_url="https://huggingface.co/datasets/OneThink/OneThinker-eval", evaluation={ "prompt_profile": "train_seg", "parser_profile": "sam2_prompt", "metric_profile": "segmentation_official", "oracle_profile": "segmentation_rle", "oracle_location": "task_payload.segmentation_output", }, preprocessing={ "fps": 2, "max_frames": 128, "video_reader": "decord", "video_min_pixels": 4096, "video_max_pixels": 262144, "video_total_pixels": 16777216, }, legacy_environment=segmentation_legacy, media_roots={"default": one_thinker_root}, preprocessed_roots=None, authorized=authorized, expected_count=_SEGMENTATION_EXPECTED_SAMPLES[split], ) ) timelens_root = _required_directory(data_root / "TimeLens-Bench", "TimeLens-Bench") timelens_specs = ( ( "charades_timelens", "charades-timelens", 4, ((timelens_root / "video_shards" / "charades",),), ), ( "activitynet_timelens", "activitynet-timelens", 4, ((timelens_root / "video_shards" / "activitynet",),), ), ( "qvhighlights_timelens", "qvhighlights-timelens", 4, ( (timelens_root / "video_shards" / "qvhighlights",), (data_root / "qvhighlights-videos",), (timelens_root / "qvhighlights-videos",), ), ), ) timelens_datasets = ",".join(split for split, _stem, _fps, _roots in timelens_specs) common_timelens_environment = { "TIMELENS_DATASETS": timelens_datasets, "TIMELENS_ENABLE_THINKING": False, "TIMELENS_FPS": 4, "TIMELENS_MIN_TOKENS": 1, "TIMELENS_MAX_FRAMES": 2048, "TIMELENS_MAX_PIXELS": 409600, "TIMELENS_TOTAL_TOKENS": 128000, "TIMELENS_MAX_NEW_TOKENS": 128, "TIMELENS_PROMPT_MODE": "same", "TIMELENS_STOP_AFTER_ANSWER": True, "TIMELENS_NUM_WORKERS": 2, } for split, stem, fps, video_root_tiers in timelens_specs: annotation = _named_annotation( timelens_root, f"temporal_grounding/{split} annotation", (f"{stem}.json", f"{stem}.jsonl"), ) video_root = _first_directory(f"{split} videos", video_root_tiers) records.append( _source_record( task="temporal_grounding", split=split, family="temporal_grounding", adapter="timelens", annotation=annotation, source_url="https://huggingface.co/datasets/TencentARC/TimeLens-Bench", evaluation={ "prompt_profile": "timelens_same", "parser_profile": "temporal_spans", "metric_profile": "temporal_iou", }, preprocessing={ "fps": fps, "min_tokens": 1, "max_frames": 2048, "max_pixels": 409600, "total_tokens": 128000, }, legacy_environment=common_timelens_environment, media_roots={"videos": video_root}, preprocessed_roots=None, authorized=authorized, ) ) return records def discover_eval_sources( data_root: str | os.PathLike[str], runtime_root: str | os.PathLike[str], *, redistribution_authorized: bool = False, ) -> list[dict[str, Any]]: """Discover the fixed paper suite without scanning unrelated data trees.""" data = _required_directory(Path(data_root), "data root") runtime = _required_directory(Path(runtime_root), "runtime root") records = [ *_video_qa_records(data, runtime, authorized=redistribution_authorized), *_spatial_intelligence_records(data, runtime, authorized=redistribution_authorized), *_structured_records( data, runtime, authorized=redistribution_authorized, ), ] records.sort(key=lambda item: (str(item["benchmark"]), str(item["split"]))) observed_tasks = {str(record["eval_task"]) for record in records} if observed_tasks != set(PAPER_TASKS): missing = sorted(set(PAPER_TASKS) - observed_tasks) unexpected = sorted(observed_tasks - set(PAPER_TASKS)) raise DiscoveryError( f"paper task discovery mismatch; missing={missing}, unexpected={unexpected}" ) if any(record["benchmark"] != record["eval_task"] for record in records): raise DiscoveryError("canonical discovery requires benchmark equal to eval_task") return records def _write_jsonl(path: Path, records: Sequence[Mapping[str, Any]], *, overwrite: bool) -> None: if path.exists() and not overwrite: raise FileExistsError(f"refusing to overwrite source manifest: {path}") path.parent.mkdir(parents=True, exist_ok=True) descriptor, temporary_name = tempfile.mkstemp( prefix=f".{path.name}.", suffix=".tmp", dir=str(path.parent), ) try: with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as handle: for record in records: handle.write( json.dumps( record, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False, ) ) handle.write("\n") os.replace(temporary_name, path) except Exception: try: os.unlink(temporary_name) except FileNotFoundError: pass raise def create_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--data-root", required=True, help="Root containing licensed sources.") parser.add_argument( "--runtime-root", required=True, help="Compatible runtime checkout containing generated paper annotations.", ) parser.add_argument( "--output", required=True, help="Private JSONL source manifest outside the OraRL release tree.", ) parser.add_argument( "--confirm-redistribution-authorized", action="store_true", help="Record explicit authorization for every discovered source.", ) parser.add_argument("--overwrite", action="store_true") return parser def main(argv: Sequence[str] | None = None) -> int: namespace = create_parser().parse_args(argv) output = Path(namespace.output).expanduser().resolve(strict=False) release_tree = Path(namespace.runtime_root).expanduser().resolve() / "OraRL" if _is_within(output, release_tree): raise SystemExit("ERROR: --output must be private and outside the OraRL release tree") try: records = discover_eval_sources( namespace.data_root, namespace.runtime_root, redistribution_authorized=namespace.confirm_redistribution_authorized, ) _write_jsonl(output, records, overwrite=namespace.overwrite) except (DiscoveryError, FileExistsError, OSError, ValueError) as error: raise SystemExit(f"ERROR: {error}") from error unlocked = sum(record["expected_count"] is None for record in records) print( json.dumps( { "output": str(output), "records": len(records), "tasks": len({record["eval_task"] for record in records}), "unlocked_counts": unlocked, "redistribution_authorized": bool(namespace.confirm_redistribution_authorized), }, sort_keys=True, ) ) return 0 if __name__ == "__main__": raise SystemExit(main())