File size: 3,472 Bytes
c282634
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
"""Baseline inference script: parsing, LLM loop with a fake client, log format, failure fallback."""

import re
import types

import pytest

import inference
from inference import LLMAgent, LocalEnv, parse_action, run_episode


def test_parse_action_variants():
    assert parse_action('{"attack_detected": true, "attack_type": 2, "confidence": 0.7, "protective_action": 1}') == \
        {"attack_detected": True, "attack_type": 2, "confidence": 0.7, "protective_action": 1}
    fenced = 'Sure!\n```json\n{"attack_detected": "true", "attack_type": 9, "confidence": 3}\n```'
    assert parse_action(fenced) == {"attack_detected": True, "attack_type": 4, "confidence": 1.0, "protective_action": 0}
    assert parse_action("no json here") is None
    assert parse_action('{"attack_type": "x"}') is None


class _FakeClient:
    """Mimics openai.OpenAI().chat.completions.create."""

    def __init__(self, reply=None, exc=None):
        self.calls = 0

        def create(**kw):
            self.calls += 1
            assert kw["messages"][0]["role"] == "system"
            if exc:
                raise exc
            msg = types.SimpleNamespace(content=reply)
            return types.SimpleNamespace(choices=[types.SimpleNamespace(message=msg)])

        self.chat = types.SimpleNamespace(completions=types.SimpleNamespace(create=create))


@pytest.fixture(autouse=True)
def _reset_breaker():
    LLMAgent._disabled = False
    LLMAgent._consecutive_failures = 0
    yield
    LLMAgent._disabled = False


LOG = re.compile(r"^\[(START|STEP|END)\] ")


def test_llm_episode_and_log_format(capsys):
    fake = _FakeClient('{"attack_detected": false, "attack_type": 0, "confidence": 0.9, "protective_action": 0}')
    res = run_episode(0, 1, lambda sid: LocalEnv(), lambda: LLMAgent(fake), "fake-model")
    out = capsys.readouterr().out.strip().splitlines()
    assert fake.calls == 40 and res["success"] and res["stats"]["llm"] == 40
    assert out[0].startswith("[START] task=sinusoidal_fdi_detection env=pll-cyberattack-detection model=fake-model")
    assert all(LOG.match(line) for line in out)
    assert sum(line.startswith("[STEP]") for line in out) == 40
    end = out[-1]
    assert re.match(r"^\[END\] success=true steps=40 score=\d\.\d{3} rewards=(-?\d+\.\d\d,){39}-?\d+\.\d\d$", end)


def test_auth_failure_falls_back_to_heuristic(capsys):
    class E(Exception):
        status_code = 402
    fake = _FakeClient(exc=E("no credits"))
    res = run_episode(2, 1, lambda sid: LocalEnv(), lambda: LLMAgent(fake), "fake-model")
    assert res["success"] and fake.calls == 1          # disabled after the first permanent error
    assert res["stats"]["fallback"] == 40 and res["score"] > 0.01
    assert "llm_disabled_fallback" in capsys.readouterr().out


def test_unparseable_output_does_not_crash(capsys):
    fake = _FakeClient("I think there might be an attack")
    res = run_episode(1, 3, lambda sid: LocalEnv(), lambda: LLMAgent(fake), "fake-model")
    assert res["success"] and res["stats"]["fallback"] == 40 and not LLMAgent._disabled
    assert "unparseable_llm_output" in capsys.readouterr().out


def test_eval_seeds_cover_scenarios():
    from pll_cyber.scenarios import sample_scenario
    kinds = {t: {sample_scenario(t, s).attack.kind for s in inference.EVAL_SEEDS[t][:3]} for t in range(3)}
    assert kinds[0] == {"sinusoidal", "none"}
    assert kinds[1] == {"sinusoidal", "pulse", "ramp"}
    assert kinds[2] == {"phase_spoofing", "none"}