File size: 2,405 Bytes
c282634
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
"""
Benchmark reference policies over many seeds (in-process, no network).

    python scripts/evaluate_baselines.py --seeds 100

Prints a Markdown table (mean ± std of grader scores per task) used in the README.
The oracle reads ground truth and is an upper bound, not a real agent.
"""

from __future__ import annotations

import argparse
import itertools
import os
import sys
import time

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import numpy as np  # noqa: E402

from pll_cyber.baselines import ConstantAgent, PhysicsHeuristicAgent, RandomAgent  # noqa: E402
from pll_cyber.environment import PLLCyberEnvironment  # noqa: E402
from pll_cyber.evaluation import run_episode  # noqa: E402


class Oracle:
    def __init__(self, env):
        self.env = env

    def act(self, obs):
        sc = self.env._sc
        lab = sc.onset_step is not None and obs["step"] >= sc.onset_step
        return {"attack_detected": lab, "attack_type": sc.attack.type_id if lab else 0, "confidence": 1.0,
                "protective_action": 2 if (lab and sc.task_id == 2) else 0}


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--seeds", type=int, default=100)
    args = ap.parse_args()
    seeds = range(1000, 1000 + args.seeds)          # disjoint from inference.py's evaluation seeds
    rnd = itertools.count()
    policies = {
        "Always alarm": lambda env, t: ConstantAgent(True, [1, 1, 4][t]),
        "Never alarm": lambda env, t: ConstantAgent(False),
        "Random (p=0.5)": lambda env, t: RandomAgent(seed=next(rnd)),
        "Physics heuristic": lambda env, t: PhysicsHeuristicAgent(),
        "Oracle (ground truth)": lambda env, t: Oracle(env),
    }
    env = PLLCyberEnvironment()
    t0 = time.time()
    print("| Policy | Task 0 (easy) | Task 1 (medium) | Task 2 (hard) | Mean |")
    print("|---|---|---|---|---|")
    for name, make in policies.items():
        cells, means = [], []
        for t in range(3):
            s = np.array([run_episode(make(env, t), t, seed, env)["score"] for seed in seeds])
            cells.append(f"{s.mean():.3f} ± {s.std():.3f}")
            means.append(s.mean())
        print(f"| {name} | " + " | ".join(cells) + f" | {np.mean(means):.3f} |", flush=True)
    print(f"\n{args.seeds} seeds per task, {time.time() - t0:.0f}s", file=sys.stderr)


if __name__ == "__main__":
    main()