File size: 1,936 Bytes
e3584eb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | import os
import shutil
import tempfile
import zipfile
from benchmark.artifacts import create_benchmark_package
from benchmark import config
def test_benchmark_package_generation():
# Setup mock variables
run_id = "test-pkg-run-456"
source_metadata = {
"filename": "sample.txt",
"original_text": "Sample text",
"extracted_text": "Sample text",
"normalized_text": "sample text",
"style": "custom"
}
questions = [{
"question_id": "Q0001",
"question": "Sample?",
"expected_answer": "Yes",
"supporting_fact_ids": ["F0001"],
"source_position": 0.5
}]
results_summary = {
"run_label": "CUSTOM EVALUATOR RUN",
"per_question": [],
"overall": {},
"config": {},
"execution_order": [],
"hardware_comparison": {"plain_device": "cpu", "rif_device": "cpu", "different_hardware": False}
}
try:
zip_path = create_benchmark_package(
run_id=run_id,
source_metadata=source_metadata,
questions=questions,
model_pairs_meta={},
plain_models_list={},
rif_models_list={},
raw_logs=[],
results_summary=results_summary
)
assert os.path.exists(zip_path)
assert zip_path.endswith(".zip")
# Verify ZIP contains expected directories/files
with zipfile.ZipFile(zip_path, "r") as z:
namelist = z.namelist()
assert any("integrity/sha256-checksums.txt" in x for x in namelist)
assert any("source/extracted-source.txt" in x for x in namelist)
finally:
# Cleanup
run_dir = os.path.join(config.RUNS_DIR, f"kalpana-benchmark-{run_id}")
if os.path.exists(run_dir):
shutil.rmtree(run_dir)
if os.path.exists(zip_path):
os.remove(zip_path)
|