File size: 4,841 Bytes
27c7ee5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 | import hashlib
import json
import unittest
from decimal import Decimal, ROUND_HALF_UP
from pathlib import Path
class ReferenceTests(unittest.TestCase):
@staticmethod
def _load(name):
return json.loads(
(Path(__file__).parents[1] / "reference" / name).read_text(encoding="utf-8")
)
def test_overall_values_are_macro_averages(self):
data = self._load("paper_numbers.json")
for name, values in data["models"].items():
cell_with = [values[index] for index in (0, 2, 4, 6) if values[index] is not None]
cell_without = [values[index] for index in (1, 3, 5, 7) if values[index] is not None]
self.assertAlmostEqual(sum(cell_with) / len(cell_with), values[8], delta=0.00051, msg=name)
self.assertAlmostEqual(sum(cell_without) / len(cell_without), values[9], delta=0.00051, msg=name)
def test_historical_videoscore2_run_rounds_to_current_paper(self):
paper = self._load("paper_numbers.json")["models"]["VideoScore2"]
history = self._load("historical_provenance_2026-08-25.json")
metrics = history["historically_exact_current_paper_runs"]["video_score2"]["paper_metrics"]
recovered = [
metrics["t2v_quality"]["with_ties"],
metrics["t2v_quality"]["without_ties"],
metrics["t2v_alignment"]["with_ties"],
metrics["t2v_alignment"]["without_ties"],
metrics["i2v_quality"]["with_ties"],
metrics["i2v_quality"]["without_ties"],
metrics["i2v_alignment"]["with_ties"],
metrics["i2v_alignment"]["without_ties"],
metrics["overall_macro"]["with_ties"],
metrics["overall_macro"]["without_ties"],
]
def paper_round(value):
historical_aggregate = Decimal(str(value)).quantize(
Decimal("0.0001"), rounding=ROUND_HALF_UP
)
return float(
historical_aggregate.quantize(
Decimal("0.001"), rounding=ROUND_HALF_UP
)
)
self.assertEqual([paper_round(value) for value in recovered], paper)
def test_videoscore2_weight_manifest_is_complete(self):
weights = self._load("videoscore2_weights.json")
self.assertEqual(
weights["revision"],
"09a2732cb64fa566a1f332f978368292ce5c295c",
)
self.assertEqual(len(weights["files"]), 4)
self.assertEqual(
sum(item["bytes"] for item in weights["files"]),
weights["total_parameter_bytes"],
)
self.assertTrue(all(len(item["lfs_sha256"]) == 64 for item in weights["files"]))
def test_fresh_videoscore2_gpu_run_rounds_to_current_paper(self):
paper = self._load("paper_numbers.json")["models"]["VideoScore2"]
result = self._load("videoscore2_gpu_reproduction_2026-08-25.json")
metrics = result["validated_full_gpu_run"]["metrics"]
recovered = []
for cell in ("t2v/quality", "t2v/alignment", "i2v/quality", "i2v/alignment"):
recovered.extend((
metrics["cells"][cell]["acc_with_ties"],
metrics["cells"][cell]["acc_without_ties"],
))
recovered.extend((
metrics["overall_macro_with_ties"],
metrics["overall_macro_without_ties"],
))
def paper_round(value):
historical_aggregate = Decimal(str(value)).quantize(
Decimal("0.0001"), rounding=ROUND_HALF_UP
)
return float(
historical_aggregate.quantize(
Decimal("0.001"), rounding=ROUND_HALF_UP
)
)
self.assertEqual([paper_round(value) for value in recovered], paper)
self.assertEqual(result["validated_full_gpu_run"]["predictions"], 549)
self.assertEqual(result["validated_full_gpu_run"]["errors"], 0)
self.assertTrue(result["validated_full_gpu_run"]["paper_row_exact_at_3_decimals"])
def test_videoscore2_delivery_artifact_hashes(self):
root = Path(__file__).parents[1]
result = self._load("videoscore2_gpu_reproduction_2026-08-25.json")
expected = {
result["worker"]["path"]: result["worker"]["delivery_worker_sha256"],
result["weights"]["file_hash_manifest"]: result["weights"]["file_hash_manifest_sha256"],
result["validated_environment"]["exact_dependency_lock"]:
result["validated_environment"]["exact_dependency_lock_sha256"],
}
for relative_path, expected_sha256 in expected.items():
actual = hashlib.sha256((root / relative_path).read_bytes()).hexdigest()
self.assertEqual(actual, expected_sha256, msg=relative_path)
if __name__ == "__main__":
unittest.main()
|