File size: 4,632 Bytes
53c10a4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 | #!/usr/bin/env python3
"""Finalize an annotation-only OraRL evaluation index for publication."""
from __future__ import annotations
import argparse
import json
import os
import tempfile
from pathlib import Path
from typing import Any
from orarl.evaluation.card import render_index_card
from orarl.evaluation.manifest import load_dataset_manifest
ALLOWED_ROOT_ENTRIES = {
".gitattributes",
"README.md",
"annotations",
"datasets.jsonl",
}
def _atomic_write(path: Path, content: str) -> None:
descriptor, temporary_name = tempfile.mkstemp(
prefix=f".{path.name}.",
suffix=".tmp",
dir=str(path.parent),
)
try:
with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as stream:
stream.write(content)
os.replace(temporary_name, path)
except Exception:
try:
os.unlink(temporary_name)
except FileNotFoundError:
pass
raise
def _set_segmentation_reader(record: dict[str, Any], reader: str) -> None:
if record.get("task") != "segmentation":
return
legacy = dict(record.get("legacy_environment", {}))
legacy["SEGMENTATION_VIDEO_READER"] = reader
setting = str(legacy.get("SEGMENTATION_SETTING", ""))
marker = f"reader{reader}"
if setting and marker not in setting:
new_marker = setting.rfind("-new")
setting = (
f"{setting[:new_marker]}-{marker}{setting[new_marker:]}"
if new_marker >= 0
else f"{setting}-{marker}"
)
legacy["SEGMENTATION_SETTING"] = setting
record["legacy_environment"] = legacy
preprocessing = dict(record.get("preprocessing", {}))
preprocessing["video_reader"] = reader
record["preprocessing"] = preprocessing
def _annotation_assets(root: Path, records: list[dict[str, Any]]) -> list[dict[str, Any]]:
assets: list[dict[str, Any]] = []
for record in records:
relative = str(record["annotation_path"])
annotation = root / relative
if annotation.is_symlink() or not annotation.is_file():
raise FileNotFoundError(f"annotation is missing: {annotation}")
with annotation.open("r", encoding="utf-8") as stream:
row_count = sum(1 for line in stream if line.strip())
expected = int(record["expected_count"])
if row_count != expected:
raise ValueError(
f"{relative}: expected {expected} rows, found {row_count}"
)
assets.append(
{
"benchmark": str(record["benchmark"]),
"bytes": annotation.stat().st_size,
"kind": "annotations",
"path": relative,
}
)
return assets
def finalize_index(
root_path: str | os.PathLike[str],
*,
repo_id: str,
segmentation_video_reader: str,
) -> dict[str, int]:
root = Path(root_path).expanduser().resolve()
unexpected = sorted(
path.name
for path in root.iterdir()
if path.name not in ALLOWED_ROOT_ENTRIES
)
if unexpected:
raise ValueError(
"metadata-only index contains unexpected root entries: "
+ ", ".join(unexpected)
)
records = [dict(record) for record in load_dataset_manifest(root)]
for record in records:
_set_segmentation_reader(record, segmentation_video_reader)
manifest = "".join(
json.dumps(record, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+ "\n"
for record in records
)
_atomic_write(root / "datasets.jsonl", manifest)
validated = [dict(record) for record in load_dataset_manifest(root)]
assets = _annotation_assets(root, validated)
_atomic_write(
root / "README.md",
render_index_card(validated, assets, repo_id=repo_id),
)
return {
"annotations": len(assets),
"bytes": sum(int(asset["bytes"]) for asset in assets),
"rows": sum(int(record["expected_count"]) for record in validated),
}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--root", required=True)
parser.add_argument("--repo-id", default="OraRL/OraRL-Data")
parser.add_argument("--segmentation-video-reader", default="decord")
args = parser.parse_args()
print(
json.dumps(
finalize_index(
args.root,
repo_id=args.repo_id,
segmentation_video_reader=args.segmentation_video_reader,
),
sort_keys=True,
)
)
if __name__ == "__main__":
main()
|