Spaces:
Running
Running
r1cksync
feat(phase8-10): saboteur+slack+replay, 381 scenarios, Colab Unsloth+Qwen72B critic, push helpers
94db88e | """Adversarial Kubernetes-style controller. | |
| Every tick (after the agent's action) the controller scans the topology and | |
| takes "well-meaning" automated remediations: | |
| * any node with mem_pct >= 0.95 gets restarted (status -> rolling_back | |
| for 2 ticks, then -> healthy). Restarting destroys whatever evidence | |
| the agent was capturing — this is the "K8s killed my heap dump" trap. | |
| * any node failing health checks (error_rate >= 0.5) for two consecutive | |
| ticks is restarted similarly. | |
| Pause-health-checks counterplay: if `node.health_checks_paused` is True the | |
| controller skips that node, letting the agent debug in peace. | |
| The controller is intentionally aggressive: it can make incidents WORSE if | |
| the agent doesn't pause checks first. | |
| """ | |
| from __future__ import annotations | |
| from dataclasses import dataclass, field | |
| from .topology import (Topology, ServiceNode, HEALTHY, ROLLING_BACK, | |
| MEMORY_LEAK, CPU_THROTTLED, DEAD) | |
| class K8sController: | |
| enabled: bool = True | |
| # name -> consecutive ticks above the unhealthy threshold. | |
| _strikes: dict[str, int] = field(default_factory=dict) | |
| # restart_log records every controller-driven restart so the env can | |
| # detect when the agent's evidence got nuked. | |
| restart_log: list[dict] = field(default_factory=list) | |
| def step(self, topology: Topology, tick: int, | |
| pending_action_targets: set[str]) -> list[str]: | |
| """Run one controller tick. Returns names of nodes the controller | |
| kicked. `pending_action_targets` lets us avoid double-acting on a | |
| node the agent is already remediating.""" | |
| if not self.enabled: | |
| return [] | |
| kicked: list[str] = [] | |
| for name, node in topology.nodes.items(): | |
| if node.health_checks_paused: | |
| self._strikes[name] = 0 | |
| continue | |
| if name in pending_action_targets: | |
| self._strikes[name] = 0 | |
| continue | |
| unhealthy = (node.mem_pct >= 0.95 | |
| or (node.error_rate >= 0.5 and node.status != ROLLING_BACK) | |
| or node.status == MEMORY_LEAK and node.mem_pct >= 0.90) | |
| if unhealthy: | |
| self._strikes[name] = self._strikes.get(name, 0) + 1 | |
| else: | |
| self._strikes[name] = 0 | |
| if self._strikes.get(name, 0) >= 2: | |
| # Kick the pod. | |
| node.status = ROLLING_BACK | |
| node.mem_pct = 0.30 | |
| node.cpu_pct = 0.20 | |
| node.error_rate = 0.20 | |
| node.leak_rate_per_tick = 0.0 # restart resets the leak | |
| self.restart_log.append({ | |
| "tick": tick, "node": name, "reason": "health_check_failure", | |
| }) | |
| kicked.append(name) | |
| self._strikes[name] = 0 | |
| return kicked | |