Enhance Kubernetes executor and deployment configurations

- Introduced remote execution capabilities in KubernetesExecutor to delegate actions to a FastAPI control plane.
- Updated .env.example to clarify Prometheus URL usage for local and VM telemetry.
- Modified Prometheus Helm values to expose the service on NodePort 30090 and added a scrape job for annotated pods in the prod-sre namespace.
- Updated deployment scripts and README to reflect new Prometheus configurations and provide clearer instructions for local setup.

.env.example

@@ -6,7 +6,8 @@ ANTIATROPOS_ENV_MODE=live
 # Reward output to agent
 ANTIATROPOS_REWARD_OUTPUT_MODE=normalized
 
-# Local Prometheus endpoint (use kubectl port-forward to localhost:9090)
+# Prometheus endpoint used by local simulator FastAPI.
+# For VM telemetry, set to droplet Prometheus NodePort, e.g. http://206.189.136.21:30090
 PROMETHEUS_URL=http://localhost:9090
 ANTIATROPOS_PROM_TIMEOUT_S=5.0
 ANTIATROPOS_STRICT_REAL=false

control/kubernetes_executor.py

@@ -2,6 +2,7 @@ import os
 import json
 import time
 import logging
+import requests
 from uuid import uuid4
 from typing import Optional
 
@@ -15,7 +16,12 @@ class KubernetesExecutor:
     def __init__(self, kubeconfig: Optional[str] = None):
         # Use provided path or env var, defaulting to mock if neither is found
         self.kubeconfig = kubeconfig or os.getenv("KUBECONFIG")
-        self.is_mock = not self.kubeconfig or self.kubeconfig.lower() == "mock"
+        self.remote_control_url = os.getenv("ANTIATROPOS_CONTROL_PLANE_URL", "").strip().rstrip("/")
+        self.remote_timeout_s = float(os.getenv("ANTIATROPOS_CONTROL_TIMEOUT_S", "5.0"))
+        self.is_mock = (
+            not self.remote_control_url
+            and (not self.kubeconfig or self.kubeconfig.lower() == "mock")
+        )
         self.namespace = os.getenv("ANTIATROPOS_K8S_NAMESPACE", "default")
         self.min_replicas = int(os.getenv("ANTIATROPOS_MIN_REPLICAS", "1"))
         self.max_replicas = self._parse_max_replicas(os.getenv("ANTIATROPOS_MAX_REPLICAS"))
@@ -112,6 +118,9 @@ class KubernetesExecutor:
         """Execute bounded actions on a Kubernetes cluster."""
         action = self._normalize_action_type(action_type)
 
+        if self.remote_control_url:
+            return self._remote_execution(action, target, parameter)
+
         if action == "NO_OP":
             return "Ack: NO_OP - no cluster mutation"
 
@@ -163,6 +172,70 @@ class KubernetesExecutor:
             f"in namespace {namespace} scaled {current}->{desired}"
         )
 
+    def _remote_execution(self, action: str, target: str, parameter: float) -> str:
+        """
+        Delegate action execution to a remote FastAPI control plane.
+
+        Expected remote endpoint contract:
+          - POST /step
+          - Request: {action_type, target_node_id, parameter}
+          - Success response includes ack_status and starts with "Ack:"
+        """
+        if not self.remote_control_url:
+            raise ValueError("ANTIATROPOS_CONTROL_PLANE_URL is not configured")
+
+        endpoint = f"{self.remote_control_url}/step"
+        action_payload = {
+            "action_type": action,
+            "target_node_id": target,
+            "parameter": float(parameter),
+        }
+        payload = action_payload
+
+        try:
+            response = requests.post(endpoint, json=payload, timeout=self.remote_timeout_s)
+        except requests.RequestException as exc:
+            raise RuntimeError(f"Remote control-plane request failed: {exc}") from exc
+
+        if response.status_code == 422:
+            # OpenEnv server.app expects {"action": {...}} shape on /step.
+            try:
+                body = response.json()
+                detail = str(body.get("detail", body))
+            except Exception:
+                detail = response.text.strip()
+            if "body" in detail and "action" in detail:
+                try:
+                    response = requests.post(
+                        endpoint,
+                        json={"action": action_payload},
+                        timeout=self.remote_timeout_s,
+                    )
+                except requests.RequestException as exc:
+                    raise RuntimeError(f"Remote control-plane retry failed: {exc}") from exc
+
+        if response.status_code >= 400:
+            detail = ""
+            try:
+                body = response.json()
+                detail = str(body.get("detail", body))
+            except Exception:
+                detail = response.text.strip()
+            raise RuntimeError(
+                f"Remote control-plane rejected action ({response.status_code}): {detail}"
+            )
+
+        try:
+            data = response.json()
+        except Exception as exc:
+            raise RuntimeError("Remote control-plane returned non-JSON response") from exc
+
+        ack = str(data.get("ack_status", "")).strip()
+        if not ack:
+            action_id = str(data.get("action_id", "")).strip() or "remote"
+            return f"Ack: {action} for {target} via remote control-plane ({action_id})"
+        return ack
+
     def _get_apps_v1_api(self):
         if self._apps_v1_api is not None:
             return self._apps_v1_api

deploy/do/README.md

@@ -23,6 +23,8 @@ sudo REPO_DIR=/opt/AntiAtropos FASTAPI_PORT=8010 MAX_REPLICAS=200 bash deploy/do
 ## What the script configures
 
 - k3s kubelet with `max-pods=250`
+- Prometheus service exposed on NodePort `30090`
+- Prometheus scrape job for annotated pods in namespace `prod-sre`
 - Env file at `.env.droplet` with:
   - `ANTIATROPOS_ENV_MODE=live`
   - `KUBECONFIG=/etc/rancher/k3s/k3s.yaml`
@@ -38,9 +40,16 @@ systemctl status antiatropos-fastapi --no-pager
 curl http://127.0.0.1:8000/config/runtime
 kubectl get deploy -n prod-sre
 kubectl get pods -n monitoring
+curl http://127.0.0.1:30090/api/v1/targets
 kubectl -n monitoring port-forward svc/grafana 3000:80
 ```
 
+If your local simulator FastAPI should use VM telemetry, set local `.env`:
+
+```env
+PROMETHEUS_URL=http://<droplet-ip>:30090
+```
+
 ## Agent call example
 
 ```bash

deploy/do/deploy-droplet-one-shot.sh

@@ -139,11 +139,18 @@ for _ in {1..30}; do
   sleep 2
 done
 
+PUBLIC_IP="$(curl -fsS https://api.ipify.org 2>/dev/null || true)"
+if [[ -z "${PUBLIC_IP}" ]]; then
+  PUBLIC_IP="$(hostname -I 2>/dev/null | awk '{print $1}')"
+fi
+PROM_URL_DISPLAY="http://${PUBLIC_IP:-<droplet-ip>}:30090"
+
 echo ""
 echo "=== Deploy Complete ==="
 echo "FastAPI runtime: http://127.0.0.1:${FASTAPI_PORT}/config/runtime"
 echo "FastAPI health:  http://127.0.0.1:${FASTAPI_PORT}/state"
 echo "Prometheus svc:  kubectl -n ${MONITORING_NAMESPACE} get svc prometheus-server"
+echo "Prometheus URL:  ${PROM_URL_DISPLAY}"
 echo "Grafana access:  kubectl -n ${MONITORING_NAMESPACE} port-forward svc/grafana 3000:80"
 echo ""
 echo "Service status command:"

deploy/prometheus-helm-values.yaml

@@ -2,12 +2,28 @@ server:
   global:
     scrape_interval: 15s
     evaluation_interval: 15s
+  service:
+    type: NodePort
+    nodePort: 30090
 
 extraScrapeConfigs: |
-  - job_name: antiatropos-fastapi
-    scrape_protocols:
-      - PrometheusText1.0.0
-      - PrometheusText0.0.4
-    metrics_path: /metrics
-    static_configs:
-      - targets: ['host.docker.internal:8000']
+  # Scrape all annotated workload pods in prod-sre.
+  - job_name: antiatropos-prod-sre-pods
+    kubernetes_sd_configs:
+      - role: pod
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_namespace]
+        action: keep
+        regex: prod-sre
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+        action: keep
+        regex: "true"
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+        action: replace
+        target_label: __metrics_path__
+        regex: (.+)
+      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+        action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        target_label: __address__