PranavKK1201 commited on
Commit ·
77ede9e
1
Parent(s): 4b5c463
basic of grafan, kuber, and prometheus added
Browse files- client.py +5 -0
- control/__init__.py +2 -0
- control/kubernetes_executor.py +108 -0
- control/validation.py +25 -0
- models.py +14 -0
- pyproject.toml +5 -2
- server/AntiAtropos_environment.py +91 -41
- server/Dockerfile +18 -0
- server/requirements.txt +3 -3
- simulator.py +50 -0
- telemetry/__init__.py +2 -0
- telemetry/mapping.py +27 -0
- telemetry/prometheus_client.py +134 -0
client.py
CHANGED
|
@@ -94,6 +94,7 @@ class AntiAtroposEnv(
|
|
| 94 |
observation = ClusterObservation(
|
| 95 |
cluster_id=obs_data.get("cluster_id", ""),
|
| 96 |
task_id=obs_data.get("task_id", "task-1"),
|
|
|
|
| 97 |
active_nodes=obs_data.get("active_nodes", 0),
|
| 98 |
average_latency_ms=obs_data.get("average_latency_ms", 0.0),
|
| 99 |
error_rate=obs_data.get("error_rate", 0.0),
|
|
@@ -106,6 +107,10 @@ class AntiAtroposEnv(
|
|
| 106 |
sla_violations=obs_data.get("sla_violations", 0),
|
| 107 |
invalid_action_count=obs_data.get("invalid_action_count", 0),
|
| 108 |
vip_failure_count=obs_data.get("vip_failure_count", 0),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 109 |
done=payload.get("done", False),
|
| 110 |
reward=payload.get("reward", 0.0),
|
| 111 |
)
|
|
|
|
| 94 |
observation = ClusterObservation(
|
| 95 |
cluster_id=obs_data.get("cluster_id", ""),
|
| 96 |
task_id=obs_data.get("task_id", "task-1"),
|
| 97 |
+
mode=obs_data.get("mode", "simulated"),
|
| 98 |
active_nodes=obs_data.get("active_nodes", 0),
|
| 99 |
average_latency_ms=obs_data.get("average_latency_ms", 0.0),
|
| 100 |
error_rate=obs_data.get("error_rate", 0.0),
|
|
|
|
| 107 |
sla_violations=obs_data.get("sla_violations", 0),
|
| 108 |
invalid_action_count=obs_data.get("invalid_action_count", 0),
|
| 109 |
vip_failure_count=obs_data.get("vip_failure_count", 0),
|
| 110 |
+
metric_timestamp=obs_data.get("metric_timestamp", 0.0),
|
| 111 |
+
data_freshness_ms=obs_data.get("data_freshness_ms", 0),
|
| 112 |
+
action_ack_status=obs_data.get("action_ack_status", "success"),
|
| 113 |
+
choke_level=obs_data.get("choke_level", 0.0),
|
| 114 |
done=payload.get("done", False),
|
| 115 |
reward=payload.get("reward", 0.0),
|
| 116 |
)
|
control/__init__.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .kubernetes_executor import KubernetesExecutor
|
| 2 |
+
from .validation import ActionValidator
|
control/kubernetes_executor.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import json
|
| 3 |
+
from typing import Optional
|
| 4 |
+
|
| 5 |
+
class KubernetesExecutor:
|
| 6 |
+
"""
|
| 7 |
+
Executes high-level SRE actions on a Kubernetes cluster.
|
| 8 |
+
Provides a safe layer between SREAgent and actual infrastructure.
|
| 9 |
+
"""
|
| 10 |
+
def __init__(self, kubeconfig: Optional[str] = None):
|
| 11 |
+
# Use provided path or env var, defaulting to mock if neither is found
|
| 12 |
+
self.kubeconfig = kubeconfig or os.getenv("KUBECONFIG")
|
| 13 |
+
self.is_mock = not self.kubeconfig or self.kubeconfig.lower() == "mock"
|
| 14 |
+
self.namespace = os.getenv("ANTIATROPOS_K8S_NAMESPACE", "default")
|
| 15 |
+
self.deployment_prefix = os.getenv("ANTIATROPOS_DEPLOYMENT_PREFIX", "")
|
| 16 |
+
self.min_replicas = int(os.getenv("ANTIATROPOS_MIN_REPLICAS", "1"))
|
| 17 |
+
self.max_replicas = int(os.getenv("ANTIATROPOS_MAX_REPLICAS", "20"))
|
| 18 |
+
self.scale_step = int(os.getenv("ANTIATROPOS_SCALE_STEP", "3"))
|
| 19 |
+
self._apps_v1_api = None
|
| 20 |
+
self._node_deployment_map = self._load_node_deployment_map()
|
| 21 |
+
|
| 22 |
+
def execute(self, action_type: str, target: str, parameter: float) -> str:
|
| 23 |
+
"""
|
| 24 |
+
Translates SRE actions to Kube requests (ScaleDeployment, PatchIngress, etc.)
|
| 25 |
+
"""
|
| 26 |
+
if self.is_mock:
|
| 27 |
+
return self._mock_execution(action_type, target, parameter)
|
| 28 |
+
|
| 29 |
+
try:
|
| 30 |
+
return self._real_execution(action_type, target, parameter)
|
| 31 |
+
except Exception as e:
|
| 32 |
+
return f"Error: Failed to execute {action_type} on {target}: {str(e)}"
|
| 33 |
+
|
| 34 |
+
def _real_execution(self, action_type: str, target: str, parameter: float) -> str:
|
| 35 |
+
"""Execute bounded actions on a Kubernetes cluster."""
|
| 36 |
+
if action_type == "NO_OP":
|
| 37 |
+
return "Ack: NO_OP - no cluster mutation"
|
| 38 |
+
|
| 39 |
+
if action_type in ("SCALE_UP", "SCALE_DOWN"):
|
| 40 |
+
return self._scale_deployment(action_type, target, parameter)
|
| 41 |
+
|
| 42 |
+
return f"Rejected: {action_type} is not enabled for live Kubernetes execution"
|
| 43 |
+
|
| 44 |
+
def _mock_execution(self, action_type: str, target: str, parameter: float) -> str:
|
| 45 |
+
"""Returns mock acknowledgement for actions."""
|
| 46 |
+
# TODO: Add realistic latency simulation for K8s control plane
|
| 47 |
+
return f"Ack: {action_type} for {target} with value {parameter} - Status: Applied"
|
| 48 |
+
|
| 49 |
+
def _scale_deployment(self, action_type: str, target: str, parameter: float) -> str:
|
| 50 |
+
deployment_name = self._resolve_deployment_name(target)
|
| 51 |
+
apps_v1 = self._get_apps_v1_api()
|
| 52 |
+
|
| 53 |
+
scale_obj = apps_v1.read_namespaced_deployment_scale(
|
| 54 |
+
name=deployment_name,
|
| 55 |
+
namespace=self.namespace,
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
current = int(scale_obj.spec.replicas or self.min_replicas)
|
| 59 |
+
delta = max(1, int(float(parameter) * self.scale_step))
|
| 60 |
+
if action_type == "SCALE_UP":
|
| 61 |
+
desired = min(self.max_replicas, current + delta)
|
| 62 |
+
else:
|
| 63 |
+
desired = max(self.min_replicas, current - delta)
|
| 64 |
+
|
| 65 |
+
if desired == current:
|
| 66 |
+
return (
|
| 67 |
+
f"Ack: {action_type} for {target} - replicas unchanged at {current} "
|
| 68 |
+
f"(bounds {self.min_replicas}-{self.max_replicas})"
|
| 69 |
+
)
|
| 70 |
+
|
| 71 |
+
apps_v1.patch_namespaced_deployment_scale(
|
| 72 |
+
name=deployment_name,
|
| 73 |
+
namespace=self.namespace,
|
| 74 |
+
body={"spec": {"replicas": desired}},
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
return f"Ack: {action_type} for {target} - deployment {deployment_name} scaled {current}->{desired}"
|
| 78 |
+
|
| 79 |
+
def _get_apps_v1_api(self):
|
| 80 |
+
if self._apps_v1_api is not None:
|
| 81 |
+
return self._apps_v1_api
|
| 82 |
+
|
| 83 |
+
from kubernetes import client, config
|
| 84 |
+
|
| 85 |
+
if self.kubeconfig and self.kubeconfig.lower() not in ("mock", ""):
|
| 86 |
+
config.load_kube_config(config_file=self.kubeconfig)
|
| 87 |
+
else:
|
| 88 |
+
config.load_incluster_config()
|
| 89 |
+
|
| 90 |
+
self._apps_v1_api = client.AppsV1Api()
|
| 91 |
+
return self._apps_v1_api
|
| 92 |
+
|
| 93 |
+
def _load_node_deployment_map(self) -> dict[str, str]:
|
| 94 |
+
raw = os.getenv("ANTIATROPOS_NODE_DEPLOYMENT_MAP", "")
|
| 95 |
+
if not raw:
|
| 96 |
+
return {}
|
| 97 |
+
try:
|
| 98 |
+
data = json.loads(raw)
|
| 99 |
+
if isinstance(data, dict):
|
| 100 |
+
return {str(k): str(v) for k, v in data.items()}
|
| 101 |
+
except json.JSONDecodeError:
|
| 102 |
+
return {}
|
| 103 |
+
return {}
|
| 104 |
+
|
| 105 |
+
def _resolve_deployment_name(self, target: str) -> str:
|
| 106 |
+
if target in self._node_deployment_map:
|
| 107 |
+
return self._node_deployment_map[target]
|
| 108 |
+
return f"{self.deployment_prefix}{target}"
|
control/validation.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import List, Optional
|
| 2 |
+
from pydantic import BaseModel
|
| 3 |
+
|
| 4 |
+
class ActionValidator:
|
| 5 |
+
"""
|
| 6 |
+
Validates SRE actions to ensure they stay within safety boundaries.
|
| 7 |
+
Prevents destructive operations like 100% shedding on critical nodes.
|
| 8 |
+
"""
|
| 9 |
+
def __init__(self, critical_nodes: Optional[List[str]] = None):
|
| 10 |
+
self.critical_nodes = critical_nodes or ["node-0", "node-1", "node-2"]
|
| 11 |
+
|
| 12 |
+
def validate(self, action_type: str, target: str, parameter: float, valid_targets: Optional[List[str]] = None) -> (bool, str):
|
| 13 |
+
"""
|
| 14 |
+
Returns (is_valid, error_message).
|
| 15 |
+
"""
|
| 16 |
+
if valid_targets is not None and target not in valid_targets:
|
| 17 |
+
return False, f"Unknown target node: {target}"
|
| 18 |
+
|
| 19 |
+
if action_type == "SHED_LOAD" and target in self.critical_nodes:
|
| 20 |
+
return False, f"Forbidden: Load shedding on critical node {target}."
|
| 21 |
+
|
| 22 |
+
if action_type in ["SCALE_UP", "SCALE_DOWN"] and parameter < 0.0:
|
| 23 |
+
return False, "Negative scaling parameters are not allowed."
|
| 24 |
+
|
| 25 |
+
return True, "Success"
|
models.py
CHANGED
|
@@ -2,6 +2,11 @@ from enum import Enum
|
|
| 2 |
from typing import Annotated, Literal, Optional
|
| 3 |
from pydantic import BaseModel, Field
|
| 4 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
# ---------------------------------------------------------------------------
|
| 6 |
# SRE Action Schema (Control Plane)
|
| 7 |
# ---------------------------------------------------------------------------
|
|
@@ -90,6 +95,8 @@ class ClusterObservation(BaseModel):
|
|
| 90 |
step: int
|
| 91 |
max_steps: int
|
| 92 |
|
|
|
|
|
|
|
| 93 |
active_nodes: int = Field(ge=0, le=5)
|
| 94 |
|
| 95 |
average_latency_ms: float = Field(
|
|
@@ -141,8 +148,15 @@ class ClusterObservation(BaseModel):
|
|
| 141 |
description="Number of failed VIP nodes in the current observation.",
|
| 142 |
)
|
| 143 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
nodes: list[NodeObservation]
|
| 145 |
|
| 146 |
# Episode interaction fields (handled by framework)
|
| 147 |
done: bool = False
|
| 148 |
reward: float = 0.0
|
|
|
|
|
|
| 2 |
from typing import Annotated, Literal, Optional
|
| 3 |
from pydantic import BaseModel, Field
|
| 4 |
|
| 5 |
+
class EnvironmentMode(str, Enum):
|
| 6 |
+
SIMULATED = "simulated"
|
| 7 |
+
HYBRID = "hybrid"
|
| 8 |
+
LIVE = "live"
|
| 9 |
+
|
| 10 |
# ---------------------------------------------------------------------------
|
| 11 |
# SRE Action Schema (Control Plane)
|
| 12 |
# ---------------------------------------------------------------------------
|
|
|
|
| 95 |
step: int
|
| 96 |
max_steps: int
|
| 97 |
|
| 98 |
+
mode: EnvironmentMode = EnvironmentMode.SIMULATED
|
| 99 |
+
|
| 100 |
active_nodes: int = Field(ge=0, le=5)
|
| 101 |
|
| 102 |
average_latency_ms: float = Field(
|
|
|
|
| 148 |
description="Number of failed VIP nodes in the current observation.",
|
| 149 |
)
|
| 150 |
|
| 151 |
+
# New fields for Prometheus/Kubernetes integration
|
| 152 |
+
metric_timestamp: float = 0.0
|
| 153 |
+
data_freshness_ms: int = 0
|
| 154 |
+
action_ack_status: str = "success"
|
| 155 |
+
choke_level: float = 0.0
|
| 156 |
+
|
| 157 |
nodes: list[NodeObservation]
|
| 158 |
|
| 159 |
# Episode interaction fields (handled by framework)
|
| 160 |
done: bool = False
|
| 161 |
reward: float = 0.0
|
| 162 |
+
|
pyproject.toml
CHANGED
|
@@ -26,6 +26,9 @@ dependencies = [
|
|
| 26 |
# "gymnasium>=0.29.0",
|
| 27 |
# "openspiel>=1.0.0",
|
| 28 |
# "smolagents>=1.22.0,<2",
|
|
|
|
|
|
|
|
|
|
| 29 |
]
|
| 30 |
|
| 31 |
[project.optional-dependencies]
|
|
@@ -41,5 +44,5 @@ server = "AntiAtropos.server.app:main"
|
|
| 41 |
|
| 42 |
[tool.setuptools]
|
| 43 |
include-package-data = true
|
| 44 |
-
packages = ["AntiAtropos", "AntiAtropos.server"]
|
| 45 |
-
package-dir = { "AntiAtropos" = ".", "AntiAtropos.server" = "server" }
|
|
|
|
| 26 |
# "gymnasium>=0.29.0",
|
| 27 |
# "openspiel>=1.0.0",
|
| 28 |
# "smolagents>=1.22.0,<2",
|
| 29 |
+
"kubernetes>=28.0.0",
|
| 30 |
+
"prometheus-api-client>=0.5.0",
|
| 31 |
+
"requests>=2.31.0",
|
| 32 |
]
|
| 33 |
|
| 34 |
[project.optional-dependencies]
|
|
|
|
| 44 |
|
| 45 |
[tool.setuptools]
|
| 46 |
include-package-data = true
|
| 47 |
+
packages = ["AntiAtropos", "AntiAtropos.server", "AntiAtropos.control", "AntiAtropos.telemetry"]
|
| 48 |
+
package-dir = { "AntiAtropos" = ".", "AntiAtropos.server" = "server", "AntiAtropos.control" = "control", "AntiAtropos.telemetry" = "telemetry" }
|
server/AntiAtropos_environment.py
CHANGED
|
@@ -1,35 +1,21 @@
|
|
| 1 |
-
|
| 2 |
-
# All rights reserved.
|
| 3 |
-
#
|
| 4 |
-
# This source code is licensed under the BSD-style license found in the
|
| 5 |
-
# LICENSE file in the root directory of this source tree.
|
| 6 |
-
|
| 7 |
-
"""
|
| 8 |
-
AntiAtropos Environment — Server-Side Implementation.
|
| 9 |
-
|
| 10 |
-
This module is the central orchestration layer. It:
|
| 11 |
-
1. Receives a typed SREAction from the OpenEnv framework.
|
| 12 |
-
2. Feeds it into the simulator (simulator.py) to advance time by one tick.
|
| 13 |
-
3. Queries stability.py to compute Lyapunov energy and the scalar reward.
|
| 14 |
-
4. Packages the resulting cluster state into a ClusterObservation and returns it.
|
| 15 |
-
|
| 16 |
-
The environment implements a discrete-time fluid-queue model where stability
|
| 17 |
-
is governed by Lyapunov Drift-Plus-Penalty theory.
|
| 18 |
-
"""
|
| 19 |
-
|
| 20 |
from uuid import uuid4
|
| 21 |
|
| 22 |
from openenv.core.env_server.interfaces import Environment
|
| 23 |
from openenv.core.env_server.types import State
|
| 24 |
|
| 25 |
try:
|
| 26 |
-
from ..models import SREAction, ClusterObservation, NodeObservation, NodeStatus
|
| 27 |
from ..simulator import ClusterSimulator, COST_PER_CAPACITY_UNIT_PER_HOUR
|
| 28 |
from ..stability import compute_lyapunov, compute_reward
|
|
|
|
|
|
|
| 29 |
except ImportError:
|
| 30 |
-
from models import SREAction, ClusterObservation, NodeObservation, NodeStatus # type: ignore[no-redef]
|
| 31 |
from simulator import ClusterSimulator, COST_PER_CAPACITY_UNIT_PER_HOUR # type: ignore[no-redef]
|
| 32 |
from stability import compute_lyapunov, compute_reward # type: ignore[no-redef]
|
|
|
|
|
|
|
| 33 |
|
| 34 |
|
| 35 |
# ---------------------------------------------------------------------------
|
|
@@ -64,22 +50,55 @@ class AntiAtroposEnvironment(Environment):
|
|
| 64 |
"""Initialise environment metadata and the simulation core."""
|
| 65 |
self._state = State(episode_id=str(uuid4()), step_count=0)
|
| 66 |
self._task_id: str = "task-1"
|
|
|
|
|
|
|
|
|
|
| 67 |
self._sim: ClusterSimulator = ClusterSimulator(n_nodes=N_NODES, task_id="task-1")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 68 |
self._nodes_true: list[dict] = []
|
| 69 |
self._nodes_obs: list[dict] = []
|
| 70 |
self._prev_lyapunov: float = 0.0
|
| 71 |
self._sla_violations: int = 0
|
|
|
|
|
|
|
| 72 |
|
| 73 |
-
def reset(self, task_id: str = "task-1") -> ClusterObservation:
|
| 74 |
"""
|
| 75 |
-
Start a fresh episode with a specific task profile.
|
| 76 |
"""
|
| 77 |
self._state = State(episode_id=str(uuid4()), step_count=0)
|
| 78 |
self._task_id = task_id
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 79 |
self._sla_violations = 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
|
| 81 |
-
# Initialize the production simulator
|
| 82 |
self._sim.reset(task_id=task_id)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 83 |
self._nodes_true = self._sim.state(for_agent=False)
|
| 84 |
self._nodes_obs = self._sim.state(for_agent=True)
|
| 85 |
self._prev_lyapunov = compute_lyapunov(self._nodes_true)
|
|
@@ -91,26 +110,56 @@ class AntiAtroposEnvironment(Environment):
|
|
| 91 |
Advance the simulation by one discrete time tick.
|
| 92 |
"""
|
| 93 |
self._state.step_count += 1
|
| 94 |
-
|
| 95 |
-
# 1.
|
| 96 |
-
self.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
self._sim.tick()
|
| 98 |
|
| 99 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 100 |
self._nodes_true = self._sim.state(for_agent=False)
|
| 101 |
self._nodes_obs = self._sim.state(for_agent=True)
|
| 102 |
|
| 103 |
-
#
|
| 104 |
avg_latency = self._avg_latency(self._nodes_true)
|
| 105 |
error_rate = self._error_rate(self._nodes_true)
|
| 106 |
sla_violation_step = 1 if (avg_latency > 200.0 or error_rate > 0.05) else 0
|
| 107 |
if sla_violation_step:
|
| 108 |
self._sla_violations += 1
|
| 109 |
|
| 110 |
-
#
|
| 111 |
current_lyapunov = compute_lyapunov(self._nodes_true)
|
| 112 |
|
| 113 |
-
#
|
| 114 |
cost = self._compute_cost(self._nodes_true)
|
| 115 |
reward = compute_reward(
|
| 116 |
v_prev=self._prev_lyapunov,
|
|
@@ -124,13 +173,13 @@ class AntiAtroposEnvironment(Environment):
|
|
| 124 |
|
| 125 |
self._prev_lyapunov = current_lyapunov
|
| 126 |
|
| 127 |
-
#
|
| 128 |
done = (
|
| 129 |
self._state.step_count >= MAX_STEPS
|
| 130 |
or all(n["status"] == NodeStatus.FAILED for n in self._nodes_true)
|
| 131 |
)
|
| 132 |
|
| 133 |
-
#
|
| 134 |
obs = self._build_observation()
|
| 135 |
obs.done = done
|
| 136 |
obs.reward = reward
|
|
@@ -150,7 +199,6 @@ class AntiAtroposEnvironment(Environment):
|
|
| 150 |
for node in nodes_true:
|
| 151 |
if node["status"] == NodeStatus.FAILED:
|
| 152 |
continue
|
| 153 |
-
# Both live and pending units are billed as infrastructure is provisioned.
|
| 154 |
total_capacity_units += int(node.get("capacity_units", 0))
|
| 155 |
total_capacity_units += int(node.get("pending_capacity_units", 0))
|
| 156 |
return total_capacity_units * COST_PER_CAPACITY_UNIT_PER_HOUR
|
|
@@ -177,10 +225,6 @@ class AntiAtroposEnvironment(Environment):
|
|
| 177 |
total_incoming = sum(float(n.get("incoming_request_rate", 0.0)) * float(n.get("importance_weight", 1.0)) for n in nodes)
|
| 178 |
if total_incoming <= 0:
|
| 179 |
return 0.0
|
| 180 |
-
|
| 181 |
-
# dropped_requests already includes both:
|
| 182 |
-
# - explicit shedding (SHED_LOAD)
|
| 183 |
-
# - traffic sent to FAILED nodes in simulator._update_queues
|
| 184 |
total_drops = sum(float(n.get("dropped_requests", 0.0)) * float(n.get("importance_weight", 1.0)) for n in nodes)
|
| 185 |
return min(1.0, total_drops / total_incoming)
|
| 186 |
|
|
@@ -206,12 +250,12 @@ class AntiAtroposEnvironment(Environment):
|
|
| 206 |
for n in self._nodes_obs
|
| 207 |
]
|
| 208 |
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
# so that sensor dropout (-1.0 latency) doesn't fake a 'good' score.
|
| 212 |
return ClusterObservation(
|
| 213 |
cluster_id=self._state.episode_id,
|
| 214 |
task_id=self._task_id,
|
|
|
|
| 215 |
active_nodes=sum(1 for n in self._nodes_true if n["status"] != NodeStatus.FAILED),
|
| 216 |
average_latency_ms=min(1.0, max(0.0, self._avg_latency(self._nodes_true) / MAX_LATENCY_NORM)),
|
| 217 |
error_rate=self._error_rate(self._nodes_true),
|
|
@@ -224,6 +268,12 @@ class AntiAtroposEnvironment(Environment):
|
|
| 224 |
sla_violations=self._sla_violations,
|
| 225 |
invalid_action_count=self._sim.invalid_action_count,
|
| 226 |
vip_failure_count=self._vip_failure_count(self._nodes_true),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 227 |
done=False,
|
| 228 |
reward=0.0,
|
| 229 |
)
|
|
|
|
|
|
|
|
|
| 1 |
+
import time
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
from uuid import uuid4
|
| 3 |
|
| 4 |
from openenv.core.env_server.interfaces import Environment
|
| 5 |
from openenv.core.env_server.types import State
|
| 6 |
|
| 7 |
try:
|
| 8 |
+
from ..models import SREAction, ClusterObservation, NodeObservation, NodeStatus, EnvironmentMode
|
| 9 |
from ..simulator import ClusterSimulator, COST_PER_CAPACITY_UNIT_PER_HOUR
|
| 10 |
from ..stability import compute_lyapunov, compute_reward
|
| 11 |
+
from ..telemetry import PrometheusClient
|
| 12 |
+
from ..control import KubernetesExecutor, ActionValidator
|
| 13 |
except ImportError:
|
| 14 |
+
from models import SREAction, ClusterObservation, NodeObservation, NodeStatus, EnvironmentMode # type: ignore[no-redef]
|
| 15 |
from simulator import ClusterSimulator, COST_PER_CAPACITY_UNIT_PER_HOUR # type: ignore[no-redef]
|
| 16 |
from stability import compute_lyapunov, compute_reward # type: ignore[no-redef]
|
| 17 |
+
from telemetry import PrometheusClient # type: ignore[no-redef]
|
| 18 |
+
from control import KubernetesExecutor, ActionValidator # type: ignore[no-redef]
|
| 19 |
|
| 20 |
|
| 21 |
# ---------------------------------------------------------------------------
|
|
|
|
| 50 |
"""Initialise environment metadata and the simulation core."""
|
| 51 |
self._state = State(episode_id=str(uuid4()), step_count=0)
|
| 52 |
self._task_id: str = "task-1"
|
| 53 |
+
self._mode: EnvironmentMode = EnvironmentMode.SIMULATED
|
| 54 |
+
|
| 55 |
+
# Core components
|
| 56 |
self._sim: ClusterSimulator = ClusterSimulator(n_nodes=N_NODES, task_id="task-1")
|
| 57 |
+
self._telemetry = PrometheusClient()
|
| 58 |
+
self._executor = KubernetesExecutor()
|
| 59 |
+
self._validator = ActionValidator()
|
| 60 |
+
|
| 61 |
self._nodes_true: list[dict] = []
|
| 62 |
self._nodes_obs: list[dict] = []
|
| 63 |
self._prev_lyapunov: float = 0.0
|
| 64 |
self._sla_violations: int = 0
|
| 65 |
+
self._action_ack_status: str = "success"
|
| 66 |
+
self._last_metric_time: float = 0.0
|
| 67 |
|
| 68 |
+
def reset(self, task_id: str = "task-1", mode: str = "simulated") -> ClusterObservation:
|
| 69 |
"""
|
| 70 |
+
Start a fresh episode with a specific task profile and mode.
|
| 71 |
"""
|
| 72 |
self._state = State(episode_id=str(uuid4()), step_count=0)
|
| 73 |
self._task_id = task_id
|
| 74 |
+
try:
|
| 75 |
+
self._mode = EnvironmentMode(mode)
|
| 76 |
+
except ValueError:
|
| 77 |
+
self._mode = EnvironmentMode.SIMULATED
|
| 78 |
+
|
| 79 |
self._sla_violations = 0
|
| 80 |
+
self._action_ack_status = "success"
|
| 81 |
+
|
| 82 |
+
# Only set baseline metric time for hybrid/live to prevent misleading freshness in SIM
|
| 83 |
+
if self._mode in [EnvironmentMode.HYBRID, EnvironmentMode.LIVE]:
|
| 84 |
+
self._last_metric_time = time.time()
|
| 85 |
+
else:
|
| 86 |
+
self._last_metric_time = 0.0
|
| 87 |
+
|
| 88 |
+
# Initialize core components based on mode
|
| 89 |
+
if self._mode != EnvironmentMode.SIMULATED:
|
| 90 |
+
# In Hybrid/Live mode, we might want to connect to real endpoints
|
| 91 |
+
# self._telemetry = PrometheusClient(url=os.getenv("PROMETHEUS_URL"))
|
| 92 |
+
pass
|
| 93 |
|
|
|
|
| 94 |
self._sim.reset(task_id=task_id)
|
| 95 |
+
|
| 96 |
+
# If in hybrid mode, immediately pull a baseline
|
| 97 |
+
if self._mode in [EnvironmentMode.HYBRID, EnvironmentMode.LIVE]:
|
| 98 |
+
node_ids = [n["node_id"] for n in self._sim.state(for_agent=False)]
|
| 99 |
+
metrics = self._telemetry.fetch_latest_metrics(node_ids)
|
| 100 |
+
self._sim.reconcile_state(metrics)
|
| 101 |
+
|
| 102 |
self._nodes_true = self._sim.state(for_agent=False)
|
| 103 |
self._nodes_obs = self._sim.state(for_agent=True)
|
| 104 |
self._prev_lyapunov = compute_lyapunov(self._nodes_true)
|
|
|
|
| 110 |
Advance the simulation by one discrete time tick.
|
| 111 |
"""
|
| 112 |
self._state.step_count += 1
|
| 113 |
+
|
| 114 |
+
# 1. Action Validation & Execution
|
| 115 |
+
valid_targets = [n["node_id"] for n in self._nodes_true]
|
| 116 |
+
is_valid, error = self._validator.validate(
|
| 117 |
+
action.action_type,
|
| 118 |
+
action.target_node_id,
|
| 119 |
+
action.parameter,
|
| 120 |
+
valid_targets=valid_targets
|
| 121 |
+
)
|
| 122 |
+
|
| 123 |
+
if not is_valid:
|
| 124 |
+
self._action_ack_status = f"Rejected: {error}"
|
| 125 |
+
# Increment invalid action count on the simulator so it's consistent
|
| 126 |
+
self._sim.invalid_action_count += 1
|
| 127 |
+
# Still advance time but the action didn't happen
|
| 128 |
+
else:
|
| 129 |
+
if self._mode == EnvironmentMode.LIVE:
|
| 130 |
+
# In LIVE mode, we actually hit the cluster
|
| 131 |
+
self._action_ack_status = self._executor.execute(action.action_type, action.target_node_id, action.parameter)
|
| 132 |
+
else:
|
| 133 |
+
self._action_ack_status = "success (simulated)"
|
| 134 |
+
|
| 135 |
+
# Always update the physics engine to keep tracking expectations
|
| 136 |
+
self._sim.apply_action(action)
|
| 137 |
+
|
| 138 |
+
# 2. Advance Physics
|
| 139 |
self._sim.tick()
|
| 140 |
|
| 141 |
+
# 3. Telemetry Ingestion (Hybrid / Live)
|
| 142 |
+
if self._mode in [EnvironmentMode.HYBRID, EnvironmentMode.LIVE]:
|
| 143 |
+
node_ids = [n["node_id"] for n in self._nodes_true]
|
| 144 |
+
metrics = self._telemetry.fetch_latest_metrics(node_ids)
|
| 145 |
+
self._sim.reconcile_state(metrics)
|
| 146 |
+
self._last_metric_time = time.time()
|
| 147 |
+
|
| 148 |
+
# 4. Extract states (Ground Truth for reward; Observation for agent)
|
| 149 |
self._nodes_true = self._sim.state(for_agent=False)
|
| 150 |
self._nodes_obs = self._sim.state(for_agent=True)
|
| 151 |
|
| 152 |
+
# 5. SLA Check
|
| 153 |
avg_latency = self._avg_latency(self._nodes_true)
|
| 154 |
error_rate = self._error_rate(self._nodes_true)
|
| 155 |
sla_violation_step = 1 if (avg_latency > 200.0 or error_rate > 0.05) else 0
|
| 156 |
if sla_violation_step:
|
| 157 |
self._sla_violations += 1
|
| 158 |
|
| 159 |
+
# 6. Compute Lyapunov stability metrics from Ground Truth
|
| 160 |
current_lyapunov = compute_lyapunov(self._nodes_true)
|
| 161 |
|
| 162 |
+
# 7. Compute scalar reward
|
| 163 |
cost = self._compute_cost(self._nodes_true)
|
| 164 |
reward = compute_reward(
|
| 165 |
v_prev=self._prev_lyapunov,
|
|
|
|
| 173 |
|
| 174 |
self._prev_lyapunov = current_lyapunov
|
| 175 |
|
| 176 |
+
# 8. Termination check
|
| 177 |
done = (
|
| 178 |
self._state.step_count >= MAX_STEPS
|
| 179 |
or all(n["status"] == NodeStatus.FAILED for n in self._nodes_true)
|
| 180 |
)
|
| 181 |
|
| 182 |
+
# 9. Package Observation
|
| 183 |
obs = self._build_observation()
|
| 184 |
obs.done = done
|
| 185 |
obs.reward = reward
|
|
|
|
| 199 |
for node in nodes_true:
|
| 200 |
if node["status"] == NodeStatus.FAILED:
|
| 201 |
continue
|
|
|
|
| 202 |
total_capacity_units += int(node.get("capacity_units", 0))
|
| 203 |
total_capacity_units += int(node.get("pending_capacity_units", 0))
|
| 204 |
return total_capacity_units * COST_PER_CAPACITY_UNIT_PER_HOUR
|
|
|
|
| 225 |
total_incoming = sum(float(n.get("incoming_request_rate", 0.0)) * float(n.get("importance_weight", 1.0)) for n in nodes)
|
| 226 |
if total_incoming <= 0:
|
| 227 |
return 0.0
|
|
|
|
|
|
|
|
|
|
|
|
|
| 228 |
total_drops = sum(float(n.get("dropped_requests", 0.0)) * float(n.get("importance_weight", 1.0)) for n in nodes)
|
| 229 |
return min(1.0, total_drops / total_incoming)
|
| 230 |
|
|
|
|
| 250 |
for n in self._nodes_obs
|
| 251 |
]
|
| 252 |
|
| 253 |
+
freshness = int((time.time() - self._last_metric_time) * 1000) if self._last_metric_time > 0 else 0
|
| 254 |
+
|
|
|
|
| 255 |
return ClusterObservation(
|
| 256 |
cluster_id=self._state.episode_id,
|
| 257 |
task_id=self._task_id,
|
| 258 |
+
mode=self._mode,
|
| 259 |
active_nodes=sum(1 for n in self._nodes_true if n["status"] != NodeStatus.FAILED),
|
| 260 |
average_latency_ms=min(1.0, max(0.0, self._avg_latency(self._nodes_true) / MAX_LATENCY_NORM)),
|
| 261 |
error_rate=self._error_rate(self._nodes_true),
|
|
|
|
| 268 |
sla_violations=self._sla_violations,
|
| 269 |
invalid_action_count=self._sim.invalid_action_count,
|
| 270 |
vip_failure_count=self._vip_failure_count(self._nodes_true),
|
| 271 |
+
metric_timestamp=self._last_metric_time,
|
| 272 |
+
data_freshness_ms=freshness,
|
| 273 |
+
action_ack_status=self._action_ack_status,
|
| 274 |
+
choke_level=0.0,
|
| 275 |
done=False,
|
| 276 |
reward=0.0,
|
| 277 |
)
|
| 278 |
+
|
| 279 |
+
|
server/Dockerfile
CHANGED
|
@@ -59,6 +59,11 @@ FROM ${BASE_IMAGE}
|
|
| 59 |
|
| 60 |
WORKDIR /app
|
| 61 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
# Copy the virtual environment from builder
|
| 63 |
COPY --from=builder /app/env/.venv /app/.venv
|
| 64 |
|
|
@@ -71,6 +76,19 @@ ENV PATH="/app/.venv/bin:$PATH"
|
|
| 71 |
# Set PYTHONPATH so imports work correctly
|
| 72 |
ENV PYTHONPATH="/app/env:$PYTHONPATH"
|
| 73 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
# Health check
|
| 75 |
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
| 76 |
CMD curl -f http://localhost:8000/health || exit 1
|
|
|
|
| 59 |
|
| 60 |
WORKDIR /app
|
| 61 |
|
| 62 |
+
# Ensure runtime has curl for health checks and diagnostics
|
| 63 |
+
RUN apt-get update && \
|
| 64 |
+
apt-get install -y --no-install-recommends curl && \
|
| 65 |
+
rm -rf /var/lib/apt/lists/*
|
| 66 |
+
|
| 67 |
# Copy the virtual environment from builder
|
| 68 |
COPY --from=builder /app/env/.venv /app/.venv
|
| 69 |
|
|
|
|
| 76 |
# Set PYTHONPATH so imports work correctly
|
| 77 |
ENV PYTHONPATH="/app/env:$PYTHONPATH"
|
| 78 |
|
| 79 |
+
# Default integration config (safe local fallback)
|
| 80 |
+
# Override these at runtime for real Hybrid/Live deployments.
|
| 81 |
+
ENV ANTIATROPOS_ENV_MODE="simulated"
|
| 82 |
+
ENV ANTIATROPOS_STRICT_REAL="false"
|
| 83 |
+
ENV PROMETHEUS_URL="mock"
|
| 84 |
+
ENV KUBECONFIG="mock"
|
| 85 |
+
ENV ANTIATROPOS_K8S_NAMESPACE="default"
|
| 86 |
+
ENV ANTIATROPOS_DEPLOYMENT_PREFIX=""
|
| 87 |
+
ENV ANTIATROPOS_MIN_REPLICAS="1"
|
| 88 |
+
ENV ANTIATROPOS_MAX_REPLICAS="20"
|
| 89 |
+
ENV ANTIATROPOS_SCALE_STEP="3"
|
| 90 |
+
ENV ANTIATROPOS_NODE_DEPLOYMENT_MAP="{}"
|
| 91 |
+
|
| 92 |
# Health check
|
| 93 |
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
| 94 |
CMD curl -f http://localhost:8000/health || exit 1
|
server/requirements.txt
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
openenv[core]>=0.2.0
|
| 2 |
fastapi>=0.115.0
|
| 3 |
uvicorn>=0.24.0
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
|
|
|
| 1 |
openenv[core]>=0.2.0
|
| 2 |
fastapi>=0.115.0
|
| 3 |
uvicorn>=0.24.0
|
| 4 |
+
kubernetes>=28.0.0
|
| 5 |
+
prometheus-api-client>=0.5.0
|
| 6 |
+
requests>=2.31.0
|
simulator.py
CHANGED
|
@@ -433,3 +433,53 @@ class ClusterSimulator:
|
|
| 433 |
n.status = NodeStatus.DEGRADED
|
| 434 |
elif n.status == NodeStatus.DEGRADED and n.queue_depth < (OVERLOAD_THRESHOLD / 2):
|
| 435 |
n.status = NodeStatus.HEALTHY
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 433 |
n.status = NodeStatus.DEGRADED
|
| 434 |
elif n.status == NodeStatus.DEGRADED and n.queue_depth < (OVERLOAD_THRESHOLD / 2):
|
| 435 |
n.status = NodeStatus.HEALTHY
|
| 436 |
+
|
| 437 |
+
def reconcile_state(self, telemetry_map: dict) -> None:
|
| 438 |
+
"""
|
| 439 |
+
Reconcile internal simulator state with external telemetry signals.
|
| 440 |
+
Used in 'hybrid' or 'live' modes to align the physics engine with reality.
|
| 441 |
+
|
| 442 |
+
telemetry_map: node_id -> TelemetryRecord (or dict)
|
| 443 |
+
"""
|
| 444 |
+
for node in self._nodes:
|
| 445 |
+
if node.node_id in telemetry_map:
|
| 446 |
+
record = telemetry_map[node.node_id]
|
| 447 |
+
# If record is an object (TelemetryRecord), access fields; otherwise treat as dict
|
| 448 |
+
if hasattr(record, "queue_depth"):
|
| 449 |
+
q_ext = float(record.queue_depth)
|
| 450 |
+
r_ext = float(record.request_rate)
|
| 451 |
+
c_ext = float(record.cpu_utilization)
|
| 452 |
+
e_ext = float(record.error_rate)
|
| 453 |
+
l_ext = float(record.latency_ms)
|
| 454 |
+
else:
|
| 455 |
+
q_ext = float(record.get("queue_depth", node.queue_depth))
|
| 456 |
+
r_ext = float(record.get("request_rate", node.incoming_request_rate))
|
| 457 |
+
c_ext = float(record.get("cpu_utilization", node.cpu_utilization))
|
| 458 |
+
e_ext = float(record.get("error_rate", 0.0))
|
| 459 |
+
l_ext = float(record.get("latency_ms", node.latency_ms))
|
| 460 |
+
|
| 461 |
+
# Smoothly blend the external state into physics to prevent step jumps
|
| 462 |
+
# trust: 0.7 (reality) / 0.3 (simulation prediction)
|
| 463 |
+
node.queue_depth = (node.queue_depth * 0.3) + (q_ext * 0.7)
|
| 464 |
+
node.incoming_request_rate = (node.incoming_request_rate * 0.3) + (r_ext * 0.7)
|
| 465 |
+
node.cpu_utilization = (node.cpu_utilization * 0.3) + (c_ext * 0.7)
|
| 466 |
+
node.latency_ms = (node.latency_ms * 0.3) + (l_ext * 0.7)
|
| 467 |
+
|
| 468 |
+
# Status reconciliation: Fail the node if error rate is high
|
| 469 |
+
if e_ext > 0.5:
|
| 470 |
+
node.status = NodeStatus.FAILED
|
| 471 |
+
elif e_ext > 0.1 and node.status == NodeStatus.HEALTHY:
|
| 472 |
+
node.status = NodeStatus.DEGRADED
|
| 473 |
+
elif e_ext <= 0.05 and node.status == NodeStatus.DEGRADED:
|
| 474 |
+
# Allow recovery if telemetry says it's clean (physics will still check queue)
|
| 475 |
+
node.status = NodeStatus.HEALTHY
|
| 476 |
+
|
| 477 |
+
# Crucial: re-derive metrics so latency/cpu are consistent (except for the blended values)
|
| 478 |
+
# Note: We blend latency/cpu above, but _update_derived_metrics might overwrite them.
|
| 479 |
+
# So we update them after blending if we want physics to win, or before if telemetry wins.
|
| 480 |
+
# Usually, for SRE dashboard, we want the blended 'reality'.
|
| 481 |
+
# However, _update_derived_metrics is used to compute 'current' state in pure sim.
|
| 482 |
+
# We'll skip it if we just reconciled to keep the blended values, OR refine it.
|
| 483 |
+
# For now, let's just make sure statuses are updated based on new queue depths.
|
| 484 |
+
self._update_statuses()
|
| 485 |
+
|
telemetry/__init__.py
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from .prometheus_client import PrometheusClient, TelemetryRecord
|
| 2 |
+
from .mapping import MetricMapper
|
telemetry/mapping.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from typing import Dict, List, Any
|
| 2 |
+
import os
|
| 3 |
+
|
| 4 |
+
class MetricMapper:
|
| 5 |
+
"""
|
| 6 |
+
Utility for mapping Prometheus label sets into internal node IDs.
|
| 7 |
+
In environments with many pods, we need to decide which pods to
|
| 8 |
+
aggregate for a given node_id.
|
| 9 |
+
"""
|
| 10 |
+
def __init__(self, mapping_strategy: str = "sum"):
|
| 11 |
+
self.strategy = mapping_strategy
|
| 12 |
+
self.node_mapping = self._load_node_mapping()
|
| 13 |
+
|
| 14 |
+
def _load_node_mapping(self) -> Dict[str, str]:
|
| 15 |
+
"""Loads pod-to-node mapping from config/labels."""
|
| 16 |
+
# Simple default mapping for demonstration
|
| 17 |
+
return {
|
| 18 |
+
"web-node-1": "node-0",
|
| 19 |
+
"api-node-2": "node-1",
|
| 20 |
+
"db-node-3": "node-2"
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
def aggregate_node_metrics(self, raw_metrics: List[Dict[str, Any]]) -> Dict[str, Dict[str, float]]:
|
| 24 |
+
"""Aggregates pod-level metrics into node-level telemetry."""
|
| 25 |
+
# TODO: Implement aggregation based on strategy (sum, mean, max)
|
| 26 |
+
# For now, we return a structural placeholder
|
| 27 |
+
return {}
|
telemetry/prometheus_client.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import random
|
| 3 |
+
from typing import Dict, List, Optional
|
| 4 |
+
import requests
|
| 5 |
+
from pydantic import BaseModel
|
| 6 |
+
|
| 7 |
+
class TelemetryRecord(BaseModel):
|
| 8 |
+
node_id: str
|
| 9 |
+
latency_ms: float
|
| 10 |
+
request_rate: float
|
| 11 |
+
error_rate: float
|
| 12 |
+
cpu_utilization: float
|
| 13 |
+
queue_depth: float
|
| 14 |
+
|
| 15 |
+
class PrometheusClient:
|
| 16 |
+
"""
|
| 17 |
+
Adapter to fetch and normalize metrics from Prometheus.
|
| 18 |
+
Supports a mock mode for local development without a live cluster.
|
| 19 |
+
"""
|
| 20 |
+
def __init__(self, prometheus_url: Optional[str] = None):
|
| 21 |
+
# Use provided URL or env var, defaulting to mock if neither is found
|
| 22 |
+
self.url = prometheus_url or os.getenv("PROMETHEUS_URL")
|
| 23 |
+
self.is_mock = not self.url or self.url.lower() == "mock"
|
| 24 |
+
self.timeout_s = float(os.getenv("ANTIATROPOS_PROM_TIMEOUT_S", "2.5"))
|
| 25 |
+
self.strict_real = os.getenv("ANTIATROPOS_STRICT_REAL", "false").lower() == "true"
|
| 26 |
+
|
| 27 |
+
self.request_rate_query = os.getenv(
|
| 28 |
+
"ANTIATROPOS_PROM_QUERY_REQUEST_RATE",
|
| 29 |
+
'sum(rate(http_requests_total{node_id="{node_id}"}[1m]))'
|
| 30 |
+
)
|
| 31 |
+
self.latency_ms_query = os.getenv(
|
| 32 |
+
"ANTIATROPOS_PROM_QUERY_LATENCY_MS",
|
| 33 |
+
'histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{node_id="{node_id}"}[5m])) by (le)) * 1000'
|
| 34 |
+
)
|
| 35 |
+
self.error_rate_query = os.getenv(
|
| 36 |
+
"ANTIATROPOS_PROM_QUERY_ERROR_RATE",
|
| 37 |
+
'sum(rate(http_requests_total{node_id="{node_id}",status=~"5.."}[1m])) / clamp_min(sum(rate(http_requests_total{node_id="{node_id}"}[1m])), 1)'
|
| 38 |
+
)
|
| 39 |
+
self.cpu_query = os.getenv(
|
| 40 |
+
"ANTIATROPOS_PROM_QUERY_CPU",
|
| 41 |
+
'avg(rate(container_cpu_usage_seconds_total{pod=~".*{node_id}.*"}[1m]))'
|
| 42 |
+
)
|
| 43 |
+
self.queue_depth_query = os.getenv(
|
| 44 |
+
"ANTIATROPOS_PROM_QUERY_QUEUE_DEPTH",
|
| 45 |
+
'sum(queue_depth{node_id="{node_id}"})'
|
| 46 |
+
)
|
| 47 |
+
|
| 48 |
+
def fetch_latest_metrics(self, node_ids: List[str]) -> Dict[str, TelemetryRecord]:
|
| 49 |
+
"""
|
| 50 |
+
Query Prometheus for the latest metrics for the given nodes.
|
| 51 |
+
Returns a mapping from node_id to TelemetryRecord.
|
| 52 |
+
"""
|
| 53 |
+
if self.is_mock:
|
| 54 |
+
return self._generate_mock_metrics(node_ids)
|
| 55 |
+
|
| 56 |
+
# Real implementation using queries
|
| 57 |
+
try:
|
| 58 |
+
return self._fetch_real_metrics(node_ids)
|
| 59 |
+
except Exception:
|
| 60 |
+
if self.strict_real:
|
| 61 |
+
raise
|
| 62 |
+
return self._generate_mock_metrics(node_ids)
|
| 63 |
+
|
| 64 |
+
def _fetch_real_metrics(self, node_ids: List[str]) -> Dict[str, TelemetryRecord]:
|
| 65 |
+
"""Fetches node telemetry from Prometheus instant queries."""
|
| 66 |
+
metrics: Dict[str, TelemetryRecord] = {}
|
| 67 |
+
saw_any_real_signal = False
|
| 68 |
+
|
| 69 |
+
for node_id in node_ids:
|
| 70 |
+
req_rate = self._query_scalar(self.request_rate_query.format(node_id=node_id))
|
| 71 |
+
lat_ms = self._query_scalar(self.latency_ms_query.format(node_id=node_id))
|
| 72 |
+
err_rate = self._query_scalar(self.error_rate_query.format(node_id=node_id))
|
| 73 |
+
cpu = self._query_scalar(self.cpu_query.format(node_id=node_id))
|
| 74 |
+
q_depth = self._query_scalar(self.queue_depth_query.format(node_id=node_id))
|
| 75 |
+
|
| 76 |
+
if any(v is not None for v in [req_rate, lat_ms, err_rate, cpu, q_depth]):
|
| 77 |
+
saw_any_real_signal = True
|
| 78 |
+
|
| 79 |
+
metrics[node_id] = TelemetryRecord(
|
| 80 |
+
node_id=node_id,
|
| 81 |
+
latency_ms=float(lat_ms if lat_ms is not None else 20.0),
|
| 82 |
+
request_rate=float(req_rate if req_rate is not None else 0.0),
|
| 83 |
+
error_rate=max(0.0, min(1.0, float(err_rate if err_rate is not None else 0.0))),
|
| 84 |
+
cpu_utilization=max(0.0, min(1.0, float(cpu if cpu is not None else 0.0))),
|
| 85 |
+
queue_depth=max(0.0, float(q_depth if q_depth is not None else 0.0)),
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
if self.strict_real and not saw_any_real_signal:
|
| 89 |
+
raise RuntimeError("Prometheus returned no usable real telemetry for requested node IDs.")
|
| 90 |
+
|
| 91 |
+
return metrics
|
| 92 |
+
|
| 93 |
+
def _query_scalar(self, promql: str) -> Optional[float]:
|
| 94 |
+
"""Runs a scalar/vector Prometheus instant query and returns the first value."""
|
| 95 |
+
if not self.url:
|
| 96 |
+
return None
|
| 97 |
+
|
| 98 |
+
response = requests.get(
|
| 99 |
+
f"{self.url.rstrip('/')}/api/v1/query",
|
| 100 |
+
params={"query": promql},
|
| 101 |
+
timeout=self.timeout_s,
|
| 102 |
+
)
|
| 103 |
+
response.raise_for_status()
|
| 104 |
+
payload = response.json()
|
| 105 |
+
|
| 106 |
+
if payload.get("status") != "success":
|
| 107 |
+
return None
|
| 108 |
+
|
| 109 |
+
result = payload.get("data", {}).get("result", [])
|
| 110 |
+
if not result:
|
| 111 |
+
return None
|
| 112 |
+
|
| 113 |
+
value = result[0].get("value")
|
| 114 |
+
if not value or len(value) < 2:
|
| 115 |
+
return None
|
| 116 |
+
|
| 117 |
+
try:
|
| 118 |
+
return float(value[1])
|
| 119 |
+
except (TypeError, ValueError):
|
| 120 |
+
return None
|
| 121 |
+
|
| 122 |
+
def _generate_mock_metrics(self, node_ids: List[str]) -> Dict[str, TelemetryRecord]:
|
| 123 |
+
"""Generates realistic-looking mock telemetry."""
|
| 124 |
+
metrics = {}
|
| 125 |
+
for nid in node_ids:
|
| 126 |
+
metrics[nid] = TelemetryRecord(
|
| 127 |
+
node_id=nid,
|
| 128 |
+
latency_ms=random.uniform(20.0, 150.0),
|
| 129 |
+
request_rate=random.uniform(10.0, 50.0),
|
| 130 |
+
error_rate=random.uniform(0.0, 0.05),
|
| 131 |
+
cpu_utilization=random.uniform(0.1, 0.8),
|
| 132 |
+
queue_depth=random.uniform(0.0, 50.0)
|
| 133 |
+
)
|
| 134 |
+
return metrics
|