Spaces:
Running
Running
File size: 1,304 Bytes
c1060df | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | #!/usr/bin/env python3
"""Run all AdaptShield tasks with the local rule baseline."""
from __future__ import annotations
from baseline import TASKS, run_task
def status_for(result: dict) -> str:
score = result["score"]
passed = (
result["done"] and
result["normalized_score_present"] and
0.01 <= score <= 0.99
)
return "PASS" if passed else "FAIL"
def main() -> int:
results = [run_task(task, emit_logs=False) for task in TASKS]
print("AdaptShield Evaluation")
print()
print(f"{'Task':<24} {'Score':>7} {'Steps':>5} {'normalized_score':>18} {'Status':>8}")
print("-" * 68)
for result in results:
normalized = "yes" if result["normalized_score_present"] else "no"
print(
f"{result['task']:<24} "
f"{result['score']:>7.3f} "
f"{result['steps']:>5} "
f"{normalized:>18} "
f"{status_for(result):>8}"
)
scores = [result["score"] for result in results]
staircase = all(left > right for left, right in zip(scores, scores[1:]))
print()
print(f"Difficulty staircase: {'PASS' if staircase else 'FAIL'}")
return 0 if all(status_for(result) == "PASS" for result in results) else 1
if __name__ == "__main__":
raise SystemExit(main())
|