Spaces:
Paused
Paused
File size: 2,835 Bytes
2f59706 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 | """Tests for the CLI commands and offline audit isolation."""
from __future__ import annotations
import json
import os
import subprocess
import sys
from pathlib import Path
import pytest
from rbench_repro.evidence import AuditInputs, build_evidence, validate_evidence
from rbench_repro.model import canonical_json, sha256_bytes
def project_root() -> Path:
return Path(__file__).resolve().parents[1]
def offline_environment() -> dict[str, str]:
env = {key: value for key, value in os.environ.items()}
env["ALL_PROXY"] = "http://127.0.0.1:9"
env["HTTPS_PROXY"] = "http://127.0.0.1:9"
env["HTTP_PROXY"] = "http://127.0.0.1:9"
env["NO_PROXY"] = ""
env["UV_CACHE_DIR"] = str(project_root() / ".cache" / "uv")
return env
def test_audit_is_offline_schema_valid_and_byte_identical(
complete_audit_inputs,
):
"""Evidence builds are deterministic and schema-valid when run offline."""
schema_path = project_root() / "schema" / "evidence-v1.schema.json"
generated_at = "2026-07-27T00:00:00+00:00"
tool_revision = "a" * 40
outputs = []
for _index in range(2):
evidence = build_evidence(complete_audit_inputs, generated_at, tool_revision)
validate_evidence(evidence, schema_path)
outputs.append(canonical_json(evidence))
assert outputs[0] == outputs[1]
def test_invalid_input_preserves_existing_output(tmp_path):
"""Audit with missing manifest exits nonzero and preserves existing output."""
output = tmp_path / "results.json"
output.write_bytes(b"preserve\n")
result = subprocess.run(
[
sys.executable, "-m", "rbench_repro.cli",
"audit",
"--manifest", str(tmp_path / "missing.json"),
"--cache-dir", str(tmp_path / "cache"),
"--schema", str(project_root() / "schema" / "evidence-v1.schema.json"),
"--output", str(output),
"--generated-at", "2026-07-27T00:00:00+00:00",
"--tool-revision", "a" * 40,
],
capture_output=True,
env=offline_environment(),
text=True,
)
assert result.returncode != 0
assert output.read_bytes() == b"preserve\n"
assert "Traceback" not in result.stderr
def test_cli_validate_rejects_invalid_evidence(tmp_path):
"""The validate subcommand exits nonzero for invalid evidence."""
invalid = tmp_path / "invalid.json"
invalid.write_text("{}")
result = subprocess.run(
[
sys.executable, "-m", "rbench_repro.cli",
"validate",
str(invalid),
"--schema", str(project_root() / "schema" / "evidence-v1.schema.json"),
],
capture_output=True,
env=offline_environment(),
text=True,
)
assert result.returncode != 0
assert "Traceback" not in result.stderr
|