tests/test_reference.py

import hashlib
import json
import unittest
from decimal import Decimal, ROUND_HALF_UP
from pathlib import Path


class ReferenceTests(unittest.TestCase):
    @staticmethod
    def _load(name):
        return json.loads(
            (Path(__file__).parents[1] / "reference" / name).read_text(encoding="utf-8")
        )

    def test_overall_values_are_macro_averages(self):
        data = self._load("paper_numbers.json")
        for name, values in data["models"].items():
            cell_with = [values[index] for index in (0, 2, 4, 6) if values[index] is not None]
            cell_without = [values[index] for index in (1, 3, 5, 7) if values[index] is not None]
            self.assertAlmostEqual(sum(cell_with) / len(cell_with), values[8], delta=0.00051, msg=name)
            self.assertAlmostEqual(sum(cell_without) / len(cell_without), values[9], delta=0.00051, msg=name)

    def test_historical_videoscore2_run_rounds_to_current_paper(self):
        paper = self._load("paper_numbers.json")["models"]["VideoScore2"]
        history = self._load("historical_provenance_2026-08-25.json")
        metrics = history["historically_exact_current_paper_runs"]["video_score2"]["paper_metrics"]
        recovered = [
            metrics["t2v_quality"]["with_ties"],
            metrics["t2v_quality"]["without_ties"],
            metrics["t2v_alignment"]["with_ties"],
            metrics["t2v_alignment"]["without_ties"],
            metrics["i2v_quality"]["with_ties"],
            metrics["i2v_quality"]["without_ties"],
            metrics["i2v_alignment"]["with_ties"],
            metrics["i2v_alignment"]["without_ties"],
            metrics["overall_macro"]["with_ties"],
            metrics["overall_macro"]["without_ties"],
        ]
        def paper_round(value):
            historical_aggregate = Decimal(str(value)).quantize(
                Decimal("0.0001"), rounding=ROUND_HALF_UP
            )
            return float(
                historical_aggregate.quantize(
                    Decimal("0.001"), rounding=ROUND_HALF_UP
                )
            )
        self.assertEqual([paper_round(value) for value in recovered], paper)

    def test_videoscore2_weight_manifest_is_complete(self):
        weights = self._load("videoscore2_weights.json")
        self.assertEqual(
            weights["revision"],
            "09a2732cb64fa566a1f332f978368292ce5c295c",
        )
        self.assertEqual(len(weights["files"]), 4)
        self.assertEqual(
            sum(item["bytes"] for item in weights["files"]),
            weights["total_parameter_bytes"],
        )
        self.assertTrue(all(len(item["lfs_sha256"]) == 64 for item in weights["files"]))

    def test_fresh_videoscore2_gpu_run_rounds_to_current_paper(self):
        paper = self._load("paper_numbers.json")["models"]["VideoScore2"]
        result = self._load("videoscore2_gpu_reproduction_2026-08-25.json")
        metrics = result["validated_full_gpu_run"]["metrics"]
        recovered = []
        for cell in ("t2v/quality", "t2v/alignment", "i2v/quality", "i2v/alignment"):
            recovered.extend((
                metrics["cells"][cell]["acc_with_ties"],
                metrics["cells"][cell]["acc_without_ties"],
            ))
        recovered.extend((
            metrics["overall_macro_with_ties"],
            metrics["overall_macro_without_ties"],
        ))
        def paper_round(value):
            historical_aggregate = Decimal(str(value)).quantize(
                Decimal("0.0001"), rounding=ROUND_HALF_UP
            )
            return float(
                historical_aggregate.quantize(
                    Decimal("0.001"), rounding=ROUND_HALF_UP
                )
            )
        self.assertEqual([paper_round(value) for value in recovered], paper)
        self.assertEqual(result["validated_full_gpu_run"]["predictions"], 549)
        self.assertEqual(result["validated_full_gpu_run"]["errors"], 0)
        self.assertTrue(result["validated_full_gpu_run"]["paper_row_exact_at_3_decimals"])

    def test_videoscore2_delivery_artifact_hashes(self):
        root = Path(__file__).parents[1]
        result = self._load("videoscore2_gpu_reproduction_2026-08-25.json")
        expected = {
            result["worker"]["path"]: result["worker"]["delivery_worker_sha256"],
            result["weights"]["file_hash_manifest"]: result["weights"]["file_hash_manifest_sha256"],
            result["validated_environment"]["exact_dependency_lock"]:
                result["validated_environment"]["exact_dependency_lock_sha256"],
        }
        for relative_path, expected_sha256 in expected.items():
            actual = hashlib.sha256((root / relative_path).read_bytes()).hexdigest()
            self.assertEqual(actual, expected_sha256, msg=relative_path)


if __name__ == "__main__":
    unittest.main()