Spaces:
Sleeping
Sleeping
Download scripts/evaluate_baselines.py from krishuggingface/CyberAttack-PLL: direct link, hf CLI and curl.
- Browser
- Download file 2.41 kB
-
https://huggingface.co/spaces/krishuggingface/CyberAttack-PLL/resolve/main/scripts/evaluate_baselines.py
- Command line
-
hf download hf://spaces/krishuggingface/CyberAttack-PLL/scripts/evaluate_baselines.py
-
curl -L -o evaluate_baselines.py https://huggingface.co/spaces/krishuggingface/CyberAttack-PLL/resolve/main/scripts/evaluate_baselines.py
2.41 kB
| """ | |
| Benchmark reference policies over many seeds (in-process, no network). | |
| python scripts/evaluate_baselines.py --seeds 100 | |
| Prints a Markdown table (mean ± std of grader scores per task) used in the README. | |
| The oracle reads ground truth and is an upper bound, not a real agent. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import itertools | |
| import os | |
| import sys | |
| import time | |
| sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) | |
| import numpy as np # noqa: E402 | |
| from pll_cyber.baselines import ConstantAgent, PhysicsHeuristicAgent, RandomAgent # noqa: E402 | |
| from pll_cyber.environment import PLLCyberEnvironment # noqa: E402 | |
| from pll_cyber.evaluation import run_episode # noqa: E402 | |
| class Oracle: | |
| def __init__(self, env): | |
| self.env = env | |
| def act(self, obs): | |
| sc = self.env._sc | |
| lab = sc.onset_step is not None and obs["step"] >= sc.onset_step | |
| return {"attack_detected": lab, "attack_type": sc.attack.type_id if lab else 0, "confidence": 1.0, | |
| "protective_action": 2 if (lab and sc.task_id == 2) else 0} | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--seeds", type=int, default=100) | |
| args = ap.parse_args() | |
| seeds = range(1000, 1000 + args.seeds) # disjoint from inference.py's evaluation seeds | |
| rnd = itertools.count() | |
| policies = { | |
| "Always alarm": lambda env, t: ConstantAgent(True, [1, 1, 4][t]), | |
| "Never alarm": lambda env, t: ConstantAgent(False), | |
| "Random (p=0.5)": lambda env, t: RandomAgent(seed=next(rnd)), | |
| "Physics heuristic": lambda env, t: PhysicsHeuristicAgent(), | |
| "Oracle (ground truth)": lambda env, t: Oracle(env), | |
| } | |
| env = PLLCyberEnvironment() | |
| t0 = time.time() | |
| print("| Policy | Task 0 (easy) | Task 1 (medium) | Task 2 (hard) | Mean |") | |
| print("|---|---|---|---|---|") | |
| for name, make in policies.items(): | |
| cells, means = [], [] | |
| for t in range(3): | |
| s = np.array([run_episode(make(env, t), t, seed, env)["score"] for seed in seeds]) | |
| cells.append(f"{s.mean():.3f} ± {s.std():.3f}") | |
| means.append(s.mean()) | |
| print(f"| {name} | " + " | ".join(cells) + f" | {np.mean(means):.3f} |", flush=True) | |
| print(f"\n{args.seeds} seeds per task, {time.time() - t0:.0f}s", file=sys.stderr) | |
| if __name__ == "__main__": | |
| main() | |