Spaces:
Running
Running
File size: 2,911 Bytes
94db88e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 | """Adversarial Kubernetes-style controller.
Every tick (after the agent's action) the controller scans the topology and
takes "well-meaning" automated remediations:
* any node with mem_pct >= 0.95 gets restarted (status -> rolling_back
for 2 ticks, then -> healthy). Restarting destroys whatever evidence
the agent was capturing — this is the "K8s killed my heap dump" trap.
* any node failing health checks (error_rate >= 0.5) for two consecutive
ticks is restarted similarly.
Pause-health-checks counterplay: if `node.health_checks_paused` is True the
controller skips that node, letting the agent debug in peace.
The controller is intentionally aggressive: it can make incidents WORSE if
the agent doesn't pause checks first.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from .topology import (Topology, ServiceNode, HEALTHY, ROLLING_BACK,
MEMORY_LEAK, CPU_THROTTLED, DEAD)
@dataclass
class K8sController:
enabled: bool = True
# name -> consecutive ticks above the unhealthy threshold.
_strikes: dict[str, int] = field(default_factory=dict)
# restart_log records every controller-driven restart so the env can
# detect when the agent's evidence got nuked.
restart_log: list[dict] = field(default_factory=list)
def step(self, topology: Topology, tick: int,
pending_action_targets: set[str]) -> list[str]:
"""Run one controller tick. Returns names of nodes the controller
kicked. `pending_action_targets` lets us avoid double-acting on a
node the agent is already remediating."""
if not self.enabled:
return []
kicked: list[str] = []
for name, node in topology.nodes.items():
if node.health_checks_paused:
self._strikes[name] = 0
continue
if name in pending_action_targets:
self._strikes[name] = 0
continue
unhealthy = (node.mem_pct >= 0.95
or (node.error_rate >= 0.5 and node.status != ROLLING_BACK)
or node.status == MEMORY_LEAK and node.mem_pct >= 0.90)
if unhealthy:
self._strikes[name] = self._strikes.get(name, 0) + 1
else:
self._strikes[name] = 0
if self._strikes.get(name, 0) >= 2:
# Kick the pod.
node.status = ROLLING_BACK
node.mem_pct = 0.30
node.cpu_pct = 0.20
node.error_rate = 0.20
node.leak_rate_per_tick = 0.0 # restart resets the leak
self.restart_log.append({
"tick": tick, "node": name, "reason": "health_check_failure",
})
kicked.append(name)
self._strikes[name] = 0
return kicked
|