File size: 2,503 Bytes
0be8f22 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | from __future__ import annotations
import os
import subprocess
import sys
from pathlib import Path
import pytest
from autocad_bench.bundle import (
BUNDLE_FILENAME,
BUNDLE_VERSION,
BundleError,
export_bundle,
validate_bundle,
)
from autocad_bench.common.paths import BENCHMARK_ROOT
from autocad_bench.common.runtime_contract import DEFAULT_EVALUATOR_VERSION
from autocad_bench.tasks.manifest import load_manifest
def test_run_ready_bundle_is_portable_allowlisted_and_integrity_bound(
tmp_path: Path,
) -> None:
output = tmp_path / "bundle"
exported = export_bundle(
output,
evaluator_versions=(DEFAULT_EVALUATOR_VERSION,),
)
assert exported.bundle_version == BUNDLE_VERSION
assert exported.task_count == 50
assert exported.evaluator_versions == (DEFAULT_EVALUATOR_VERSION,)
assert len(exported.files) == 204
assert (output / BUNDLE_FILENAME).is_file()
assert len(load_manifest(root=output)) == 50
assert all(not entry.path.startswith("configs/") for entry in exported.files)
assert all(".env" not in entry.path for entry in exported.files)
validated = validate_bundle(output)
assert validated == exported
subprocess_result = subprocess.run(
[
sys.executable,
"-c",
(
"from autocad_bench.tasks.manifest import "
"BENCHMARK_ROOT, load_manifest; "
"print(BENCHMARK_ROOT); print(len(load_manifest()))"
),
],
cwd=tmp_path,
env={**os.environ, "AUTOCAD_BENCH_ROOT": str(output)},
check=True,
capture_output=True,
text=True,
)
assert subprocess_result.stdout.splitlines() == [str(output), "50"]
manifest_path = output / "tasks" / "manifest.jsonl"
original = manifest_path.read_bytes()
manifest_path.write_bytes(original + b"\n")
with pytest.raises(BundleError, match="size changed"):
validate_bundle(output)
manifest_path.write_bytes(original)
unexpected = output / "credentials.env"
unexpected.write_text("SECRET=not-allowed\n")
with pytest.raises(BundleError, match="unexpected=.*credentials.env"):
validate_bundle(output)
def test_bundle_export_refuses_to_overlay_an_existing_directory(
tmp_path: Path,
) -> None:
output = tmp_path / "existing"
output.mkdir()
with pytest.raises(BundleError, match="already exists"):
export_bundle(output, source_root=BENCHMARK_ROOT)
|