consolidate

.env.example

@@ -7,14 +7,16 @@ ANTIATROPOS_ENV_MODE=live
 ANTIATROPOS_REWARD_OUTPUT_MODE=normalized
 
 # Prometheus endpoint used by local simulator FastAPI.
-# For VM telemetry, set to droplet Prometheus NodePort, e.g. http://206.189.136.21:30090
-PROMETHEUS_URL=http://localhost:9090
+# Consolidated path: droplet Prometheus NodePort.
+PROMETHEUS_URL=http://<droplet-ip>:30090
 ANTIATROPOS_PROM_TIMEOUT_S=5.0
 ANTIATROPOS_STRICT_REAL=false
 ANTIATROPOS_METRIC_AGGREGATION=sum
 
 # Kubernetes execution settings
 KUBECONFIG=C:/Users/your-user/.kube/config
+ANTIATROPOS_CONTROL_PLANE_URL=http://<droplet-ip>:8010
+ANTIATROPOS_CONTROL_TIMEOUT_S=8.0
 ANTIATROPOS_K8S_NAMESPACE=prod-sre
 ANTIATROPOS_MIN_REPLICAS=1
 ANTIATROPOS_MAX_REPLICAS=
@@ -31,6 +33,7 @@ ANTIATROPOS_GRAFANA_MODE=local
 # If GROQ_API_KEY is set and API_BASE_URL is not set, inference.py auto-uses Groq.
 GROQ_API_KEY=
 MODEL_NAME=llama-3.1-8b-instant
+# Local OpenEnv runtime remains authoritative.
 ENV_URL=http://localhost:8000
 ANTIATROPOS_MODE=live
 ANTIATROPOS_LABEL_NODE_MAP={"payments":"node-0","checkout":"node-1","catalog":"node-2","cart":"node-3","auth":"node-4"}

control/kubernetes_executor.py

@@ -18,6 +18,8 @@ class KubernetesExecutor:
         self.kubeconfig = kubeconfig or os.getenv("KUBECONFIG")
         self.remote_control_url = os.getenv("ANTIATROPOS_CONTROL_PLANE_URL", "").strip().rstrip("/")
         self.remote_timeout_s = float(os.getenv("ANTIATROPOS_CONTROL_TIMEOUT_S", "5.0"))
+        self.remote_retry_count = int(os.getenv("ANTIATROPOS_CONTROL_RETRY_COUNT", "2"))
+        self.remote_retry_backoff_s = float(os.getenv("ANTIATROPOS_CONTROL_RETRY_BACKOFF_S", "0.25"))
         self.is_mock = (
             not self.remote_control_url
             and (not self.kubeconfig or self.kubeconfig.lower() == "mock")
@@ -29,6 +31,8 @@ class KubernetesExecutor:
         self._apps_v1_api = None
         self._node_workload_map = self._load_node_workload_map()
         self._live_supported_actions = {"NO_OP", "SCALE_UP", "SCALE_DOWN"}
+        self.k8s_retry_count = int(os.getenv("ANTIATROPOS_K8S_RETRY_COUNT", "2"))
+        self.k8s_retry_backoff_s = float(os.getenv("ANTIATROPOS_K8S_RETRY_BACKOFF_S", "0.2"))
 
     @staticmethod
     def _parse_max_replicas(raw: Optional[str]) -> Optional[int]:
@@ -161,10 +165,11 @@ class KubernetesExecutor:
                 f"(bounds {self.min_replicas}-{upper})"
             )
 
-        apps_v1.patch_namespaced_deployment_scale(
-            name=deployment_name,
+        self._patch_deployment_scale_with_retry(
+            apps_v1=apps_v1,
+            deployment_name=deployment_name,
             namespace=namespace,
-            body={"spec": {"replicas": desired}},
+            desired=desired,
         )
 
         return (
@@ -172,6 +177,34 @@ class KubernetesExecutor:
             f"in namespace {namespace} scaled {current}->{desired}"
         )
 
+    def _patch_deployment_scale_with_retry(self, apps_v1, deployment_name: str, namespace: str, desired: int) -> None:
+        """
+        Patch deployment replicas with retries for transient API server errors.
+        """
+        from kubernetes.client.rest import ApiException
+
+        max_attempts = max(1, self.k8s_retry_count + 1)
+        for attempt in range(1, max_attempts + 1):
+            try:
+                apps_v1.patch_namespaced_deployment_scale(
+                    name=deployment_name,
+                    namespace=namespace,
+                    body={"spec": {"replicas": desired}},
+                )
+                return
+            except ApiException as exc:
+                retryable = exc.status in (409, 429, 500, 502, 503, 504)
+                if (not retryable) or attempt >= max_attempts:
+                    raise
+                sleep_s = self.k8s_retry_backoff_s * (2 ** (attempt - 1))
+                logger.warning(
+                    "Retrying deployment scale patch after ApiException status=%s attempt=%s/%s",
+                    exc.status,
+                    attempt,
+                    max_attempts,
+                )
+                time.sleep(sleep_s)
+
     def _remote_execution(self, action: str, target: str, parameter: float) -> str:
         """
         Delegate action execution to a remote FastAPI control plane.
@@ -180,6 +213,9 @@ class KubernetesExecutor:
           - POST /step
           - Request: {action_type, target_node_id, parameter}
           - Success response includes ack_status and starts with "Ack:"
+
+        This contract matches server.local_laptop_control and is the only
+        supported remote control-plane format.
         """
         if not self.remote_control_url:
             raise ValueError("ANTIATROPOS_CONTROL_PLANE_URL is not configured")
@@ -192,27 +228,7 @@ class KubernetesExecutor:
         }
         payload = action_payload
 
-        try:
-            response = requests.post(endpoint, json=payload, timeout=self.remote_timeout_s)
-        except requests.RequestException as exc:
-            raise RuntimeError(f"Remote control-plane request failed: {exc}") from exc
-
-        if response.status_code == 422:
-            # OpenEnv server.app expects {"action": {...}} shape on /step.
-            try:
-                body = response.json()
-                detail = str(body.get("detail", body))
-            except Exception:
-                detail = response.text.strip()
-            if "body" in detail and "action" in detail:
-                try:
-                    response = requests.post(
-                        endpoint,
-                        json={"action": action_payload},
-                        timeout=self.remote_timeout_s,
-                    )
-                except requests.RequestException as exc:
-                    raise RuntimeError(f"Remote control-plane retry failed: {exc}") from exc
+        response = self._post_with_retry(endpoint=endpoint, payload=payload)
 
         if response.status_code >= 400:
             detail = ""
@@ -221,6 +237,12 @@ class KubernetesExecutor:
                 detail = str(body.get("detail", body))
             except Exception:
                 detail = response.text.strip()
+            if response.status_code == 422 and "action" in detail:
+                detail = (
+                    f"{detail}. Expected lightweight control-plane contract at "
+                    f"{endpoint}: "
+                    '{"action_type":"SCALE_UP","target_node_id":"node-0","parameter":1.0}'
+                )
             raise RuntimeError(
                 f"Remote control-plane rejected action ({response.status_code}): {detail}"
             )
@@ -236,6 +258,47 @@ class KubernetesExecutor:
             return f"Ack: {action} for {target} via remote control-plane ({action_id})"
         return ack
 
+    def _post_with_retry(self, endpoint: str, payload: dict) -> requests.Response:
+        """
+        POST helper with retries for transient HTTP/network failures.
+        """
+        max_attempts = max(1, self.remote_retry_count + 1)
+        last_exc: Optional[Exception] = None
+
+        for attempt in range(1, max_attempts + 1):
+            try:
+                response = requests.post(endpoint, json=payload, timeout=self.remote_timeout_s)
+            except requests.RequestException as exc:
+                last_exc = exc
+                if attempt >= max_attempts:
+                    break
+                sleep_s = self.remote_retry_backoff_s * (2 ** (attempt - 1))
+                logger.warning(
+                    "Retrying remote control-plane POST after network error attempt=%s/%s: %s",
+                    attempt,
+                    max_attempts,
+                    exc,
+                )
+                time.sleep(sleep_s)
+                continue
+
+            if response.status_code >= 500 and attempt < max_attempts:
+                sleep_s = self.remote_retry_backoff_s * (2 ** (attempt - 1))
+                logger.warning(
+                    "Retrying remote control-plane POST after HTTP %s attempt=%s/%s",
+                    response.status_code,
+                    attempt,
+                    max_attempts,
+                )
+                time.sleep(sleep_s)
+                continue
+
+            return response
+
+        if last_exc is not None:
+            raise RuntimeError(f"Remote control-plane request failed: {last_exc}") from last_exc
+        raise RuntimeError("Remote control-plane request failed after retries")
+
     def _get_apps_v1_api(self):
         if self._apps_v1_api is not None:
             return self._apps_v1_api

deploy/do/README.md

@@ -4,7 +4,12 @@ This deploy flow is for a single Ubuntu Droplet running:
 - k3s (single-node Kubernetes)
 - AntiAtropos sample workloads (`prod-sre`)
 - Prometheus + Grafana (`monitoring`)
-- FastAPI control server (`antiatropos-fastapi` systemd service)
+- lightweight control-plane API (`antiatropos-control` on port `8010`)
+
+The OpenEnv runtime (`server.app`) is intentionally **not** run on the droplet.
+The only supported split is:
+- local machine: OpenEnv server + inference loop
+- droplet: Kubernetes executor API + observability stack
 
 ## Run
 
@@ -17,7 +22,7 @@ sudo bash deploy/do/deploy-droplet-one-shot.sh
 Optional overrides:
 
 ```bash
-sudo REPO_DIR=/opt/AntiAtropos FASTAPI_PORT=8010 MAX_REPLICAS=200 bash deploy/do/deploy-droplet-one-shot.sh
+sudo REPO_DIR=/opt/AntiAtropos CONTROL_PORT=8010 MAX_REPLICAS=200 bash deploy/do/deploy-droplet-one-shot.sh
 ```
 
 ## What the script configures
@@ -26,34 +31,62 @@ sudo REPO_DIR=/opt/AntiAtropos FASTAPI_PORT=8010 MAX_REPLICAS=200 bash deploy/do
 - Prometheus service exposed on NodePort `30090`
 - Prometheus scrape job for annotated pods in namespace `prod-sre`
 - Env file at `.env.droplet` with:
-  - `ANTIATROPOS_ENV_MODE=live`
   - `KUBECONFIG=/etc/rancher/k3s/k3s.yaml`
   - `ANTIATROPOS_WORKLOAD_MAP` for `node-0`..`node-4`
 - Systemd service:
-  - Name: `antiatropos-fastapi`
-  - Exec: `uvicorn server.app:app --host 0.0.0.0 --port 8000`
+  - Name: `antiatropos-control`
+  - Exec: `uvicorn server.local_laptop_control:app --host 0.0.0.0 --port 8010`
+- Legacy cleanup:
+  - `antiatropos-fastapi` (VM OpenEnv service) is disabled/removed by default deploy path
 
 ## Verify
 
 ```bash
-systemctl status antiatropos-fastapi --no-pager
-curl http://127.0.0.1:8000/config/runtime
+systemctl status antiatropos-control --no-pager
+curl http://127.0.0.1:8010/health
 kubectl get deploy -n prod-sre
 kubectl get pods -n monitoring
 curl http://127.0.0.1:30090/api/v1/targets
 kubectl -n monitoring port-forward svc/grafana 3000:80
 ```
 
-If your local simulator FastAPI should use VM telemetry, set local `.env`:
+Set local `.env` to use this consolidated path:
 
 ```env
+ENV_URL=http://localhost:8000
+ANTIATROPOS_CONTROL_PLANE_URL=http://<droplet-ip>:8010
 PROMETHEUS_URL=http://<droplet-ip>:30090
 ```
 
-## Agent call example
+## Deterministic remote-scaling proof
+
+On droplet, watch desired replicas:
 
 ```bash
-curl -X POST http://127.0.0.1:8000/step \
+watch -n 1 'kubectl -n prod-sre get deploy -o custom-columns=NAME:.metadata.name,DESIRED:.spec.replicas,READY:.status.readyReplicas,AVAILABLE:.status.availableReplicas'
+```
+
+From local machine, send one control action:
+
+```bash
+curl -X POST http://<droplet-ip>:8010/step \
   -H "Content-Type: application/json" \
-  -d '{"action_type":"SCALE_UP","target_node_id":"node-3","parameter":0.6}'
+  -d '{"action_type":"SCALE_UP","target_node_id":"node-0","parameter":1.0}'
 ```
+
+If `payments` desired replicas increase, scaling is happening on droplet.
+
+## Troubleshooting
+
+- **Pods do not move during inference**
+  - Verify local env points to droplet control API:
+    - `ANTIATROPOS_CONTROL_PLANE_URL=http://<droplet-ip>:8010`
+  - Check droplet control health:
+    - `curl http://127.0.0.1:8010/health`
+  - Check service status:
+    - `systemctl status antiatropos-control --no-pager`
+- **Connection refused from local to droplet:8010**
+  - Service not running or firewall closed.
+  - Start service and open firewall if needed.
+- **Need to remove legacy VM OpenEnv service**
+  - `sudo bash deploy/do/uninstall-legacy-openenv.sh`

deploy/do/antiatropos-control.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=AntiAtropos Droplet Control API
+After=network-online.target k3s.service
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=root
+WorkingDirectory=/root/Anti-Atropos
+EnvironmentFile=/root/Anti-Atropos/.env.droplet
+ExecStart=/root/Anti-Atropos/.venv-droplet/bin/uvicorn server.local_laptop_control:app --host 0.0.0.0 --port 8010
+Restart=always
+RestartSec=3
+
+[Install]
+WantedBy=multi-user.target

deploy/do/deploy-droplet-one-shot.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 # - Installs k3s with kubelet max-pods=250
 # - Deploys workloads + Prometheus + Grafana
 # - Creates env file for live Kubernetes scaling
-# - Starts FastAPI server via systemd (antiatropos-fastapi)
+# - Starts lightweight control-plane API via systemd (antiatropos-control)
 
 if [[ "${EUID}" -ne 0 ]]; then
   echo "Run as root: sudo bash deploy/do/deploy-droplet-one-shot.sh"
@@ -14,8 +14,8 @@ fi
 
 REPO_DIR="${REPO_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}"
 KUBECONFIG_PATH="${KUBECONFIG_PATH:-/etc/rancher/k3s/k3s.yaml}"
-FASTAPI_PORT="${FASTAPI_PORT:-8000}"
-FASTAPI_HOST="${FASTAPI_HOST:-0.0.0.0}"
+CONTROL_PORT="${CONTROL_PORT:-8010}"
+CONTROL_HOST="${CONTROL_HOST:-0.0.0.0}"
 K8S_NAMESPACE="${K8S_NAMESPACE:-prod-sre}"
 MONITORING_NAMESPACE="${MONITORING_NAMESPACE:-monitoring}"
 PY_VENV_DIR="${PY_VENV_DIR:-${REPO_DIR}/.venv-droplet}"
@@ -28,7 +28,7 @@ WORKLOAD_MAP="${WORKLOAD_MAP:-{\"node-0\":{\"deployment\":\"payments\",\"namespa
 echo "=== AntiAtropos Droplet One-Shot Deploy ==="
 echo "Repo:        ${REPO_DIR}"
 echo "Kubeconfig:  ${KUBECONFIG_PATH}"
-echo "FastAPI:     ${FASTAPI_HOST}:${FASTAPI_PORT}"
+echo "Control API: ${CONTROL_HOST}:${CONTROL_PORT}"
 echo ""
 
 if [[ ! -f "${REPO_DIR}/deploy/local-laptop.yaml" ]]; then
@@ -85,7 +85,6 @@ helm upgrade --install grafana grafana/grafana \
 
 if [[ ! -f "${ENV_FILE}" ]]; then
   cat > "${ENV_FILE}" <<EOF
-ANTIATROPOS_ENV_MODE=live
 KUBECONFIG=/etc/rancher/k3s/k3s.yaml
 ANTIATROPOS_K8S_NAMESPACE=prod-sre
 ANTIATROPOS_MIN_REPLICAS=${MIN_REPLICAS}
@@ -108,9 +107,16 @@ else
   "${PY_VENV_DIR}/bin/pip" install -r "${REPO_DIR}/server/requirements.txt"
 fi
 
-cat > /etc/systemd/system/antiatropos-fastapi.service <<EOF
+# Hard cleanup: remove legacy VM OpenEnv service if it exists.
+if systemctl list-unit-files | grep -q '^antiatropos-fastapi\.service'; then
+  echo "Disabling legacy service antiatropos-fastapi..."
+  systemctl disable --now antiatropos-fastapi >/dev/null 2>&1 || true
+  rm -f /etc/systemd/system/antiatropos-fastapi.service
+fi
+
+cat > /etc/systemd/system/antiatropos-control.service <<EOF
 [Unit]
-Description=AntiAtropos FastAPI Server
+Description=AntiAtropos Droplet Control API
 After=network-online.target k3s.service
 Wants=network-online.target
 
@@ -119,7 +125,7 @@ Type=simple
 User=root
 WorkingDirectory=${REPO_DIR}
 EnvironmentFile=${ENV_FILE}
-ExecStart=${PY_VENV_DIR}/bin/uvicorn server.app:app --host ${FASTAPI_HOST} --port ${FASTAPI_PORT}
+ExecStart=${PY_VENV_DIR}/bin/uvicorn server.local_laptop_control:app --host ${CONTROL_HOST} --port ${CONTROL_PORT}
 Restart=always
 RestartSec=3
 
@@ -128,12 +134,12 @@ WantedBy=multi-user.target
 EOF
 
 systemctl daemon-reload
-systemctl enable --now antiatropos-fastapi
+systemctl enable --now antiatropos-control
 
 echo ""
-echo "Waiting for app readiness..."
+echo "Waiting for control API readiness..."
 for _ in {1..30}; do
-  if curl -fsS "http://127.0.0.1:${FASTAPI_PORT}/config/runtime" >/dev/null 2>&1; then
+  if curl -fsS "http://127.0.0.1:${CONTROL_PORT}/health" >/dev/null 2>&1; then
     break
   fi
   sleep 2
@@ -147,15 +153,18 @@ PROM_URL_DISPLAY="http://${PUBLIC_IP:-<droplet-ip>}:30090"
 
 echo ""
 echo "=== Deploy Complete ==="
-echo "FastAPI runtime: http://127.0.0.1:${FASTAPI_PORT}/config/runtime"
-echo "FastAPI health:  http://127.0.0.1:${FASTAPI_PORT}/state"
+echo "Control health:  http://127.0.0.1:${CONTROL_PORT}/health"
+echo "Control step:    http://127.0.0.1:${CONTROL_PORT}/step"
 echo "Prometheus svc:  kubectl -n ${MONITORING_NAMESPACE} get svc prometheus-server"
 echo "Prometheus URL:  ${PROM_URL_DISPLAY}"
 echo "Grafana access:  kubectl -n ${MONITORING_NAMESPACE} port-forward svc/grafana 3000:80"
 echo ""
 echo "Service status command:"
-echo "  systemctl status antiatropos-fastapi --no-pager"
+echo "  systemctl status antiatropos-control --no-pager"
 echo ""
-echo "If needed, edit env and restart:"
+echo "If needed, edit env and restart control service:"
 echo "  ${ENV_FILE}"
-echo "  systemctl restart antiatropos-fastapi"
+echo "  systemctl restart antiatropos-control"
+echo ""
+echo "Verify remote scaling path:"
+echo "  watch -n 1 'kubectl -n prod-sre get deploy -o custom-columns=NAME:.metadata.name,DESIRED:.spec.replicas,READY:.status.readyReplicas'"

deploy/do/uninstall-legacy-openenv.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Removes legacy VM OpenEnv service path.
+# This keeps droplet runtime focused on control API + observability only.
+
+if [[ "${EUID}" -ne 0 ]]; then
+  echo "Run as root: sudo bash deploy/do/uninstall-legacy-openenv.sh"
+  exit 1
+fi
+
+if systemctl list-unit-files | grep -q '^antiatropos-fastapi\.service'; then
+  echo "Stopping and disabling antiatropos-fastapi..."
+  systemctl disable --now antiatropos-fastapi >/dev/null 2>&1 || true
+else
+  echo "antiatropos-fastapi service not registered."
+fi
+
+if [[ -f /etc/systemd/system/antiatropos-fastapi.service ]]; then
+  rm -f /etc/systemd/system/antiatropos-fastapi.service
+  echo "Removed /etc/systemd/system/antiatropos-fastapi.service"
+fi
+
+systemctl daemon-reload
+echo "Legacy VM OpenEnv service cleanup complete."

deploy/prometheus-helm-values.yaml

@@ -27,3 +27,4 @@ extraScrapeConfigs: |
         regex: ([^:]+)(?::\d+)?;(\d+)
         replacement: $1:$2
         target_label: __address__
+