| |
| """Offline integrity and boundary verifier for the static smoke-test package. |
| |
| It validates only this wrapper's recorded metadata and files. It does not make |
| network requests, import the released implementation, or execute any probe. |
| """ |
|
|
| from __future__ import annotations |
|
|
| import hashlib |
| import json |
| import re |
| import stat |
| import sys |
| from pathlib import Path |
|
|
|
|
| ROOT = Path(__file__).resolve().parent |
| FEED_REVISION = "7b5b56aebf3abe590eab9f2c241a796125cab928" |
| FEED_ETAG = "f8b74517cd33712f80399eea913efac0bb41078f" |
| FEED_URL = ( |
| "https://huggingface.co/spaces/ICML-2026-agent-repro/challenge/resolve/" |
| f"{FEED_REVISION}/claims.json" |
| ) |
| SOURCE_COMMIT = "564652bbdfa9ca6b905a3891a52b878a2af3a18e" |
| SOURCE_TREE = "1da7572f290b2b8c9be1492bc00e587a47e4a61c" |
| REQUIRED_TAGS = ( |
| "icml2026-repro", |
| "paper-42cOdUJOhH", |
| "released-implementation", |
| "bounded-smoke-test", |
| ) |
| CLAIMS = ( |
| ( |
| "C1", |
| "Benchmarking framework enables accurate finite-difference estimates of curvature, reach, and volume for data manifolds", |
| "pages/claim-c1/page.md", |
| "limited released-implementation compatibility evidence only", |
| ), |
| ( |
| "C2", |
| "Framework assesses generalization bounds and analyzes geometry evolution across network layers in variational autoencoders", |
| "pages/claim-c2/page.md", |
| "not assessed", |
| ), |
| ) |
| REQUIRED_FILES = { |
| "README.md", |
| "index.html", |
| "PROVENANCE.md", |
| "LICENSE", |
| "NOTICE", |
| "MANIFEST.md", |
| "manifest.json", |
| "verify.py", |
| "MANIFEST.sha256", |
| "evidence/claims-snapshot.json", |
| "evidence/run-record.md", |
| "evidence/toy-finite-difference-result.json", |
| "evidence/unit-tests.txt", |
| "evidence/runtime.txt", |
| "evidence/environment-lock.txt", |
| *(page for _, _, page, _ in CLAIMS), |
| "pages/execution/page.md", |
| } |
| MARKDOWN_LINK = re.compile(r"\[[^\]]+\]\(([^)]+)\)") |
| HTML_HREF = re.compile(r'\bhref="([^"]+)"') |
| EVENT_HANDLER = re.compile(r"\son[a-z]+\s*=", re.IGNORECASE) |
| SECRET_PATTERNS = ( |
| re.compile(r"hf_" + r"[A-Za-z0-9]{20,}"), |
| re.compile(r"sk-" + r"[A-Za-z0-9]{20,}"), |
| re.compile(r"AKIA" + r"[A-Z0-9]{16}"), |
| ) |
| PRIVATE_PREFIXES = ( |
| "/" + "Users" + "/", |
| "/" + "home" + "/", |
| "C:" + "\\" + "Users" + "\\", |
| ) |
|
|
|
|
| class VerificationError(RuntimeError): |
| """Raised when the static package contract is not met.""" |
|
|
|
|
| def require(condition: bool, message: str) -> None: |
| if not condition: |
| raise VerificationError(message) |
|
|
|
|
| def read_text(relative_path: str) -> str: |
| return (ROOT / relative_path).read_text(encoding="utf-8") |
|
|
|
|
| def read_json(relative_path: str) -> dict: |
| try: |
| value = json.loads(read_text(relative_path)) |
| except json.JSONDecodeError as error: |
| raise VerificationError(f"invalid JSON in {relative_path}: {error}") from error |
| require(isinstance(value, dict), f"{relative_path} must contain a JSON object") |
| return value |
|
|
|
|
| def all_non_git_paths() -> list[Path]: |
| return sorted( |
| (path for path in ROOT.rglob("*") if ".git" not in path.parts), |
| key=lambda path: path.relative_to(ROOT).as_posix(), |
| ) |
|
|
|
|
| def package_files() -> list[Path]: |
| return [ |
| path |
| for path in all_non_git_paths() |
| if path.is_file() |
| and "__pycache__" not in path.parts |
| and path.name != "MANIFEST.sha256" |
| ] |
|
|
|
|
| def require_local_target(source: Path, target: str, context: str) -> None: |
| destination = target.split("#", 1)[0] |
| if not destination: |
| return |
| resolved = (source.parent / destination).resolve() |
| require(resolved.is_relative_to(ROOT.resolve()), f"local link escapes package in {context}: {target}") |
| require(resolved.is_file(), f"broken local link in {context}: {target}") |
|
|
|
|
| def check_physical_tree() -> None: |
| for path in all_non_git_paths(): |
| relative_path = path.relative_to(ROOT).as_posix() |
| require("__pycache__" not in path.parts, f"cached path is not allowed: {relative_path}") |
| require(not path.is_symlink(), f"symlink is not allowed: {relative_path}") |
| if path.is_file(): |
| require(not stat.S_ISREG(path.stat().st_mode) or not (path.stat().st_mode & 0o111), f"executable file is not allowed: {relative_path}") |
|
|
|
|
| def check_required_files() -> None: |
| actual = {path.relative_to(ROOT).as_posix() for path in package_files()} | {"MANIFEST.sha256"} |
| require(REQUIRED_FILES <= actual, f"missing required files: {sorted(REQUIRED_FILES - actual)}") |
| forbidden_suffixes = {".pdf", ".tex", ".tar", ".gz", ".zip", ".ipynb", ".pt", ".ckpt", ".whl"} |
| for path in package_files(): |
| relative_path = path.relative_to(ROOT).as_posix() |
| require(path.suffix.lower() not in forbidden_suffixes, f"forbidden artefact: {relative_path}") |
|
|
|
|
| def check_readme_and_manifest() -> tuple[dict, dict]: |
| readme = read_text("README.md") |
| require(readme.startswith("---\n"), "README.md must start with Space YAML front matter") |
| front_matter, separator, _ = readme[4:].partition("\n---\n") |
| require(bool(separator), "README.md front matter is not closed") |
| require("sdk: static" in front_matter, "README.md must set sdk: static") |
| require("app_file: index.html" in front_matter, "README.md must set app_file: index.html") |
| require("pinned: false" in front_matter, "README.md must set pinned: false") |
| for tag in REQUIRED_TAGS: |
| require(f" - {tag}" in front_matter, f"README.md missing tag: {tag}") |
| require("does not verify either official claim" in readme, "README.md must retain the claim boundary") |
|
|
| manifest = read_json("manifest.json") |
| package = manifest.get("package", {}) |
| space = manifest.get("space", {}) |
| require(package.get("type") == "static-space-bounded-released-implementation-smoke-test", "package type is wrong") |
| require(package.get("non_interactive") is True, "package must remain non-interactive") |
| require(space.get("sdk") == "static", "manifest Space SDK is wrong") |
| require(space.get("app_file") == "index.html", "manifest app file is wrong") |
| require(set(REQUIRED_TAGS) == set(space.get("required_tags", [])), "manifest tags do not match the README") |
|
|
| feed = manifest.get("claims_feed", {}) |
| require(feed.get("url") == FEED_URL, "manifest claims-feed URL is not immutable") |
| require(feed.get("space_revision") == FEED_REVISION, "manifest feed revision is wrong") |
| require(feed.get("asset_etag") == FEED_ETAG, "manifest feed ETag is wrong") |
| source = manifest.get("source", {}) |
| require(source.get("commit") == SOURCE_COMMIT, "source commit is wrong") |
| require(source.get("tree") == SOURCE_TREE, "source tree is wrong") |
| require(source.get("licence") == "BSD-3-Clause", "source licence is wrong") |
| return manifest, read_json("evidence/claims-snapshot.json") |
|
|
|
|
| def check_claims(manifest: dict, snapshot: dict) -> None: |
| manifest_claims = manifest.get("claims") |
| snapshot_claims = snapshot.get("claims") |
| require(isinstance(manifest_claims, list) and len(manifest_claims) == len(CLAIMS), "manifest claim count is wrong") |
| require(isinstance(snapshot_claims, list) and len(snapshot_claims) == len(CLAIMS), "snapshot claim count is wrong") |
| snapshot_feed = snapshot.get("claims_feed", {}) |
| require(snapshot_feed.get("url") == FEED_URL, "snapshot feed URL is not immutable") |
| require(snapshot_feed.get("space_revision") == FEED_REVISION, "snapshot feed revision is wrong") |
| require(snapshot_feed.get("asset_etag") == FEED_ETAG, "snapshot feed ETag is wrong") |
| for expected, manifest_claim, snapshot_claim in zip(CLAIMS, manifest_claims, snapshot_claims): |
| claim_id, claim_text, page, outcome = expected |
| require(manifest_claim.get("id") == claim_id, f"manifest claim id mismatch: {claim_id}") |
| require(snapshot_claim.get("id") == claim_id, f"snapshot claim id mismatch: {claim_id}") |
| require(manifest_claim.get("exact_registry_claim") == claim_text, f"manifest claim text mismatch: {claim_id}") |
| require(snapshot_claim.get("text") == claim_text, f"snapshot claim text mismatch: {claim_id}") |
| require(manifest_claim.get("registry_status") == "unverified", f"manifest registry status mismatch: {claim_id}") |
| require(snapshot_claim.get("registry_status") == "unverified", f"snapshot registry status mismatch: {claim_id}") |
| require(manifest_claim.get("page") == page, f"manifest page mismatch: {claim_id}") |
| require(snapshot_claim.get("page") == page, f"snapshot page mismatch: {claim_id}") |
| require(manifest_claim.get("fixed_outcome") == outcome, f"manifest outcome mismatch: {claim_id}") |
| require(snapshot_claim.get("fixed_outcome") == outcome, f"snapshot outcome mismatch: {claim_id}") |
| content = read_text(page) |
| require(content.count(claim_text) == 1, f"claim must appear exactly once in {page}") |
| require("## Fixed outcome" in content, f"missing fixed outcome in {page}") |
| require(outcome in content, f"wrong outcome in {page}") |
|
|
|
|
| def check_execution(manifest: dict) -> None: |
| result = read_json("evidence/toy-finite-difference-result.json") |
| circle = result.get("circle", {}) |
| torus = result.get("torus", {}) |
| require(circle.get("grid_shape") == [257, 2], "circle grid shape is wrong") |
| require(torus.get("grid_shape") == [64, 64, 3], "torus grid shape is wrong") |
| expected_metrics = { |
| "circle_volume_relative_error": circle.get("relative_error"), |
| "torus_area_relative_error": torus.get("relative_area_error"), |
| "torus_scalar_curvature_max_abs_error": torus.get("scalar_curvature_max_abs_error"), |
| } |
| for key, value in expected_metrics.items(): |
| require(manifest.get("execution", {}).get(key) == value, f"execution metric mismatch: {key}") |
| require(circle["relative_error"] < 1e-3, "circle probe threshold fails") |
| require(torus["relative_area_error"] < 5e-3, "torus area probe threshold fails") |
| require(torus["scalar_curvature_max_abs_error"] < 0.1, "torus curvature probe threshold fails") |
| tests = read_text("evidence/unit-tests.txt") |
| require(tests.count("3 passed") == 2, "selected tests must show two passing runs") |
| runtime = read_text("evidence/runtime.txt") |
| for expected in ("python=3.13.14", "torch=2.13.0", "torch_cuda_available=False"): |
| require(expected in runtime, f"runtime record missing {expected}") |
| record = read_text("evidence/run-record.md") |
| for expected in (SOURCE_COMMIT, "No data was downloaded", "not a performance evaluation"): |
| require(expected in record, f"run record missing boundary: {expected}") |
|
|
|
|
| def check_links_and_static_html() -> None: |
| for path in package_files(): |
| if path.suffix.lower() != ".md": |
| continue |
| relative_path = path.relative_to(ROOT).as_posix() |
| for target in MARKDOWN_LINK.findall(path.read_text(encoding="utf-8")): |
| if target.startswith(("https://", "http://", "mailto:")): |
| continue |
| require_local_target(path, target, relative_path) |
|
|
| html = read_text("index.html") |
| lowered = html.lower() |
| for forbidden in ("<script", "<form", "<iframe", "<object", "<embed", "javascript:", "data:", "@import", "url("): |
| require(forbidden not in lowered, f"active or remote HTML construct is forbidden: {forbidden}") |
| require(not EVENT_HANDLER.search(html), "HTML event handler is forbidden") |
| require(" src=" not in lowered, "HTML remote/local asset loading is forbidden") |
| for target in HTML_HREF.findall(html): |
| require(not target.startswith(("https://", "http://", "javascript:", "data:")), f"HTML remote target is forbidden: {target}") |
| require_local_target(ROOT / "index.html", target, "index.html") |
| require("No author code, backend, evaluation, model training" in html, "HTML evidence boundary is missing") |
|
|
|
|
| def check_content_boundary() -> None: |
| for path in package_files(): |
| relative_path = path.relative_to(ROOT).as_posix() |
| content = path.read_text(encoding="utf-8") |
| require(not any(prefix in content for prefix in PRIVATE_PREFIXES), f"private path in {relative_path}") |
| require(not any(pattern.search(content) for pattern in SECRET_PATTERNS), f"token-shaped string in {relative_path}") |
| notice = read_text("NOTICE") |
| require("No author source" in notice, "NOTICE must state source exclusion") |
| provenance = read_text("PROVENANCE.md") |
| for expected in (SOURCE_COMMIT, SOURCE_TREE, "BSD-3-Clause", "not establish a paper result"): |
| require(expected in provenance, f"provenance missing {expected}") |
|
|
|
|
| def check_hash_manifest() -> None: |
| entries: dict[str, str] = {} |
| for line in read_text("MANIFEST.sha256").splitlines(): |
| digest, separator, relative_path = line.partition(" ") |
| require(separator == " " and len(digest) == 64 and relative_path, f"invalid hash entry: {line}") |
| require(relative_path not in entries, f"duplicate hash entry: {relative_path}") |
| entries[relative_path] = digest |
| actual_paths = {path.relative_to(ROOT).as_posix() for path in package_files()} |
| require(set(entries) == actual_paths, "hash-manifest file set does not match package") |
| for relative_path in sorted(actual_paths): |
| digest = hashlib.sha256((ROOT / relative_path).read_bytes()).hexdigest() |
| require(entries[relative_path] == digest, f"SHA-256 mismatch: {relative_path}") |
|
|
|
|
| def main() -> int: |
| try: |
| check_physical_tree() |
| check_required_files() |
| manifest, snapshot = check_readme_and_manifest() |
| check_claims(manifest, snapshot) |
| check_execution(manifest) |
| check_links_and_static_html() |
| check_content_boundary() |
| check_hash_manifest() |
| except VerificationError as error: |
| print(f"FAIL: {error}", file=sys.stderr) |
| return 1 |
| print("PASS: static bounded released-implementation smoke-test package is internally consistent") |
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
