training/validate_release.py

#!/usr/bin/env python3
"""Fail closed unless every local release claim has matching evidence."""

from __future__ import annotations

import argparse
import hashlib
import json
import math
from pathlib import Path

import numpy as np


EXPECTED_MODELS = {
    "GoktugD/goktugtr-retrieval-270m-v1",
    "microsoft/harrier-oss-v1-270m",
    "newmindai/Mursit-Base-TR-Retrieval",
    "intfloat/multilingual-e5-base",
}
EXPECTED_DATASET_HASHES = {
    "train-00000-of-00001.parquet": "01aa846d9dee0cccf916b8f8337e617e95d9e0be5f81ecf25bab01ae92c38d19",
    "validation-00000-of-00001.parquet": "41d2d7476f53099b9ab29d6b4f30cd6e13b8a9f8a73a9309238832f975544b48",
}


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--release-dir", type=Path, default=Path("release/goktugtr-retrieval-270m-v1")
    )
    parser.add_argument("--space-dir", type=Path, default=Path("space"))
    parser.add_argument("--dataset-dir", type=Path, default=Path("dataset"))
    return parser.parse_args()


def sha256(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def main() -> None:
    args = parse_args()
    release = args.release_dir.resolve()
    manifest = json.loads((release / "release-manifest.json").read_text(encoding="utf-8"))
    for item in manifest["files"]:
        path = release / item["path"]
        assert path.is_file(), f"Manifest file missing: {item['path']}"
        assert path.stat().st_size == item["bytes"], f"Size mismatch: {item['path']}"
        assert sha256(path) == item["sha256"], f"SHA-256 mismatch: {item['path']}"

    card = (release / "README.md").read_text(encoding="utf-8")
    assert "TBD" not in card, "Model card still contains placeholder results"
    assert "Release status:" not in card, "Model card still contains local draft notice"

    results = json.loads(
        (release / "results/turhistquad-results.json").read_text(encoding="utf-8")
    )
    assert len(results) == 4
    assert {item["model"] for item in results} == EXPECTED_MODELS
    for item in results:
        assert item["queries"] == 1024
        assert item["corpus_documents"] == 1213
        for metric in ("mrr_at_10", "ndcg_at_10", "recall_at_10", "recall_at_100"):
            assert math.isfinite(item[metric]) and 0 <= item[metric] <= 1

    overlap = json.loads(
        (release / "results/turhistquad-overlap-audit.json").read_text(encoding="utf-8")
    )
    assert overlap["query_to_query_exact_overlaps"] == 0
    assert overlap["passage_to_document_exact_overlaps"] == 0
    assert overlap["any_training_text_to_benchmark_text_exact_overlaps"] == 0

    q8 = json.loads(
        (release / "results/onnx-browser-validation.json").read_text(encoding="utf-8")
    )
    assert q8["embedding_dimension"] == 640
    assert q8["mean_pytorch_to_q8_cosine"] >= 0.99
    assert q8["minimum_pytorch_to_q8_cosine"] >= 0.97
    assert q8["example_query_top1_agreement"] >= 0.75

    documents = json.loads((args.space_dir / "corpus.json").read_text(encoding="utf-8"))
    embeddings = np.asarray(
        json.loads((args.space_dir / "embeddings.json").read_text(encoding="utf-8")),
        dtype="float32",
    )
    assert len(documents) == 24
    assert embeddings.shape == (24, 640)
    assert np.allclose(np.linalg.norm(embeddings, axis=1), 1.0, atol=2e-4)
    assert "sdk: static" in (args.space_dir / "README.md").read_text(encoding="utf-8")
    assert not (args.space_dir / "requirements.txt").exists()
    assert not (args.space_dir / "app.py").exists()

    for name, expected in EXPECTED_DATASET_HASHES.items():
        assert sha256(args.dataset_dir / "data" / name) == expected

    report = {
        "release_files_verified": len(manifest["files"]),
        "benchmark_models_verified": len(results),
        "space_embeddings": list(embeddings.shape),
        "dataset_files_verified": len(EXPECTED_DATASET_HASHES),
        "status": "pass",
    }
    print(json.dumps(report, indent=2))


if __name__ == "__main__":
    main()