Spaces:
Sleeping
Sleeping
File size: 2,405 Bytes
c282634 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | """
Benchmark reference policies over many seeds (in-process, no network).
python scripts/evaluate_baselines.py --seeds 100
Prints a Markdown table (mean ± std of grader scores per task) used in the README.
The oracle reads ground truth and is an upper bound, not a real agent.
"""
from __future__ import annotations
import argparse
import itertools
import os
import sys
import time
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import numpy as np # noqa: E402
from pll_cyber.baselines import ConstantAgent, PhysicsHeuristicAgent, RandomAgent # noqa: E402
from pll_cyber.environment import PLLCyberEnvironment # noqa: E402
from pll_cyber.evaluation import run_episode # noqa: E402
class Oracle:
def __init__(self, env):
self.env = env
def act(self, obs):
sc = self.env._sc
lab = sc.onset_step is not None and obs["step"] >= sc.onset_step
return {"attack_detected": lab, "attack_type": sc.attack.type_id if lab else 0, "confidence": 1.0,
"protective_action": 2 if (lab and sc.task_id == 2) else 0}
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--seeds", type=int, default=100)
args = ap.parse_args()
seeds = range(1000, 1000 + args.seeds) # disjoint from inference.py's evaluation seeds
rnd = itertools.count()
policies = {
"Always alarm": lambda env, t: ConstantAgent(True, [1, 1, 4][t]),
"Never alarm": lambda env, t: ConstantAgent(False),
"Random (p=0.5)": lambda env, t: RandomAgent(seed=next(rnd)),
"Physics heuristic": lambda env, t: PhysicsHeuristicAgent(),
"Oracle (ground truth)": lambda env, t: Oracle(env),
}
env = PLLCyberEnvironment()
t0 = time.time()
print("| Policy | Task 0 (easy) | Task 1 (medium) | Task 2 (hard) | Mean |")
print("|---|---|---|---|---|")
for name, make in policies.items():
cells, means = [], []
for t in range(3):
s = np.array([run_episode(make(env, t), t, seed, env)["score"] for seed in seeds])
cells.append(f"{s.mean():.3f} ± {s.std():.3f}")
means.append(s.mean())
print(f"| {name} | " + " | ".join(cells) + f" | {np.mean(means):.3f} |", flush=True)
print(f"\n{args.seeds} seeds per task, {time.time() - t0:.0f}s", file=sys.stderr)
if __name__ == "__main__":
main()
|