#!/usr/bin/env python3 """Fail closed unless every local release claim has matching evidence.""" from __future__ import annotations import argparse import hashlib import json import math from pathlib import Path import numpy as np EXPECTED_MODELS = { "GoktugD/goktugtr-retrieval-270m-v1", "microsoft/harrier-oss-v1-270m", "newmindai/Mursit-Base-TR-Retrieval", "intfloat/multilingual-e5-base", } EXPECTED_DATASET_HASHES = { "train-00000-of-00001.parquet": "01aa846d9dee0cccf916b8f8337e617e95d9e0be5f81ecf25bab01ae92c38d19", "validation-00000-of-00001.parquet": "41d2d7476f53099b9ab29d6b4f30cd6e13b8a9f8a73a9309238832f975544b48", } def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument( "--release-dir", type=Path, default=Path("release/goktugtr-retrieval-270m-v1") ) parser.add_argument("--space-dir", type=Path, default=Path("space")) parser.add_argument("--dataset-dir", type=Path, default=Path("dataset")) return parser.parse_args() def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def main() -> None: args = parse_args() release = args.release_dir.resolve() manifest = json.loads((release / "release-manifest.json").read_text(encoding="utf-8")) for item in manifest["files"]: path = release / item["path"] assert path.is_file(), f"Manifest file missing: {item['path']}" assert path.stat().st_size == item["bytes"], f"Size mismatch: {item['path']}" assert sha256(path) == item["sha256"], f"SHA-256 mismatch: {item['path']}" card = (release / "README.md").read_text(encoding="utf-8") assert "TBD" not in card, "Model card still contains placeholder results" assert "Release status:" not in card, "Model card still contains local draft notice" results = json.loads( (release / "results/turhistquad-results.json").read_text(encoding="utf-8") ) assert len(results) == 4 assert {item["model"] for item in results} == EXPECTED_MODELS for item in results: assert item["queries"] == 1024 assert item["corpus_documents"] == 1213 for metric in ("mrr_at_10", "ndcg_at_10", "recall_at_10", "recall_at_100"): assert math.isfinite(item[metric]) and 0 <= item[metric] <= 1 overlap = json.loads( (release / "results/turhistquad-overlap-audit.json").read_text(encoding="utf-8") ) assert overlap["query_to_query_exact_overlaps"] == 0 assert overlap["passage_to_document_exact_overlaps"] == 0 assert overlap["any_training_text_to_benchmark_text_exact_overlaps"] == 0 q8 = json.loads( (release / "results/onnx-browser-validation.json").read_text(encoding="utf-8") ) assert q8["embedding_dimension"] == 640 assert q8["mean_pytorch_to_q8_cosine"] >= 0.99 assert q8["minimum_pytorch_to_q8_cosine"] >= 0.97 assert q8["example_query_top1_agreement"] >= 0.75 documents = json.loads((args.space_dir / "corpus.json").read_text(encoding="utf-8")) embeddings = np.asarray( json.loads((args.space_dir / "embeddings.json").read_text(encoding="utf-8")), dtype="float32", ) assert len(documents) == 24 assert embeddings.shape == (24, 640) assert np.allclose(np.linalg.norm(embeddings, axis=1), 1.0, atol=2e-4) assert "sdk: static" in (args.space_dir / "README.md").read_text(encoding="utf-8") assert not (args.space_dir / "requirements.txt").exists() assert not (args.space_dir / "app.py").exists() for name, expected in EXPECTED_DATASET_HASHES.items(): assert sha256(args.dataset_dir / "data" / name) == expected report = { "release_files_verified": len(manifest["files"]), "benchmark_models_verified": len(results), "space_embeddings": list(embeddings.shape), "dataset_files_verified": len(EXPECTED_DATASET_HASHES), "status": "pass", } print(json.dumps(report, indent=2)) if __name__ == "__main__": main()