chore(kubernetes): support unbounded max replicas and enhance local setup

- Allow ANTIATROPOS_MAX_REPLICAS to be empty or 'unbounded' for no scale-up limit
- Adjust scaling logic to handle unbounded max replicas correctly
- Update .env.example with local defaults and new Prometheus queries
- Simplify start-grafana.ps1 for local Grafana setup without AWS credentials
- Add kind cluster config with max pods set to 250 for local testing
- Enhance local-laptop.yaml with prometheus metrics exposition and python http server
- Update LOCAL_LAPTOP_FASTAPI_GUIDE.md to document unbounded max replicas and setup changes

.env.example

@@ -1,34 +1,40 @@
 # AntiAtropos runtime defaults (copy to .env and adjust values)
 
 # Environment mode for FastAPI runtime
-ANTIATROPOS_ENV_MODE=aws
+ANTIATROPOS_ENV_MODE=live
 
 # Reward output to agent
 ANTIATROPOS_REWARD_OUTPUT_MODE=normalized
 
-# Prometheus/AMP endpoint (workspace URL)
-PROMETHEUS_URL=https://aps-workspaces.ap-south-1.amazonaws.com/workspaces/REPLACE_WORKSPACE_ID
+# Local Prometheus endpoint (use kubectl port-forward to localhost:9090)
+PROMETHEUS_URL=http://localhost:9090
 ANTIATROPOS_PROM_TIMEOUT_S=5.0
 ANTIATROPOS_STRICT_REAL=false
 ANTIATROPOS_METRIC_AGGREGATION=sum
 
 # Kubernetes execution settings
-KUBECONFIG=D:/AntiAtropos/deploy/aws/kubeconfig-antiatropos.yaml
+KUBECONFIG=C:/Users/your-user/.kube/config
 ANTIATROPOS_K8S_NAMESPACE=prod-sre
 ANTIATROPOS_MIN_REPLICAS=1
-ANTIATROPOS_MAX_REPLICAS=6
+ANTIATROPOS_MAX_REPLICAS=
 ANTIATROPOS_SCALE_STEP=3
 
 # Node -> deployment map used by Kubernetes executor
 ANTIATROPOS_WORKLOAD_MAP={"node-0":{"deployment":"payments","namespace":"prod-sre"},"node-1":{"deployment":"checkout","namespace":"prod-sre"},"node-2":{"deployment":"catalog","namespace":"prod-sre"},"node-3":{"deployment":"cart","namespace":"prod-sre"},"node-4":{"deployment":"auth","namespace":"prod-sre"}}
 
-# AWS defaults used by deployment scripts
-AWS_REGION=ap-south-1
-ANTIATROPOS_GRAFANA_MODE=external
+# Local defaults
+AWS_REGION=
+ANTIATROPOS_GRAFANA_MODE=local
 
 # Agent LLM provider (optional): Groq (OpenAI-compatible)
 # If GROQ_API_KEY is set and API_BASE_URL is not set, inference.py auto-uses Groq.
 GROQ_API_KEY=
 MODEL_NAME=llama-3.1-8b-instant
 ENV_URL=http://localhost:8000
-ANTIATROPOS_MODE=aws
+ANTIATROPOS_MODE=live
+ANTIATROPOS_LABEL_NODE_MAP={"payments":"node-0","checkout":"node-1","catalog":"node-2","cart":"node-3","auth":"node-4"}
+ANTIATROPOS_PROM_QUERY_REQUEST_RATE=sum(rate(http_requests_total[1m])) by (node_id)
+ANTIATROPOS_PROM_QUERY_LATENCY_MS=histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[1m])) by (node_id, le)) * 1000
+ANTIATROPOS_PROM_QUERY_ERROR_RATE=sum(rate(http_requests_total{status=~"5.."}[1m])) by (node_id) / clamp_min(sum(rate(http_requests_total[1m])) by (node_id), 1)
+ANTIATROPOS_PROM_QUERY_CPU=avg(rate(container_cpu_usage_seconds_total[1m])) by (node_id)
+ANTIATROPOS_PROM_QUERY_QUEUE_DEPTH=avg(queue_depth) by (node_id)

control/kubernetes_executor.py

@@ -18,12 +18,34 @@ class KubernetesExecutor:
         self.is_mock = not self.kubeconfig or self.kubeconfig.lower() == "mock"
         self.namespace = os.getenv("ANTIATROPOS_K8S_NAMESPACE", "default")
         self.min_replicas = int(os.getenv("ANTIATROPOS_MIN_REPLICAS", "1"))
-        self.max_replicas = int(os.getenv("ANTIATROPOS_MAX_REPLICAS", "20"))
+        self.max_replicas = self._parse_max_replicas(os.getenv("ANTIATROPOS_MAX_REPLICAS"))
         self.scale_step = int(os.getenv("ANTIATROPOS_SCALE_STEP", "3"))
         self._apps_v1_api = None
         self._node_workload_map = self._load_node_workload_map()
         self._live_supported_actions = {"NO_OP", "SCALE_UP", "SCALE_DOWN"}
 
+    @staticmethod
+    def _parse_max_replicas(raw: Optional[str]) -> Optional[int]:
+        """
+        Parse optional max replicas.
+
+        Returns:
+          - int when a positive explicit cap is provided
+          - None when scale-up should be unbounded
+        """
+        if raw is None:
+            return None
+        value = str(raw).strip().lower()
+        if value in ("", "none", "unbounded", "inf", "infinite"):
+            return None
+        try:
+            parsed = int(value)
+        except ValueError:
+            return None
+        if parsed <= 0:
+            return None
+        return parsed
+
     @staticmethod
     def _normalize_action_type(action_type) -> str:
         if hasattr(action_type, "value"):
@@ -116,14 +138,18 @@ class KubernetesExecutor:
         current = int(scale_obj.spec.replicas or self.min_replicas)
         delta = max(1, int(float(parameter) * self.scale_step))
         if action_type == "SCALE_UP":
-            desired = min(self.max_replicas, current + delta)
+            if self.max_replicas is None:
+                desired = current + delta
+            else:
+                desired = min(self.max_replicas, current + delta)
         else:
             desired = max(self.min_replicas, current - delta)
 
         if desired == current:
+            upper = "unbounded" if self.max_replicas is None else str(self.max_replicas)
             return (
                 f"Ack: {action_type} for {target} - replicas unchanged at {current} "
-                f"(bounds {self.min_replicas}-{self.max_replicas})"
+                f"(bounds {self.min_replicas}-{upper})"
             )
 
         apps_v1.patch_namespaced_deployment_scale(

deploy/LOCAL_LAPTOP_FASTAPI_GUIDE.md

@@ -24,7 +24,7 @@ The controller requires `KUBECONFIG` and `ANTIATROPOS_WORKLOAD_MAP`.
 $env:KUBECONFIG = "$HOME/.kube/config"
 $env:ANTIATROPOS_K8S_NAMESPACE = "prod-sre"
 $env:ANTIATROPOS_MIN_REPLICAS = "1"
-$env:ANTIATROPOS_MAX_REPLICAS = "6"
+$env:ANTIATROPOS_MAX_REPLICAS = ""   # empty => unbounded scale-up
 $env:ANTIATROPOS_SCALE_STEP = "3"
 $env:ANTIATROPOS_WORKLOAD_MAP = '{"node-0":{"deployment":"payments","namespace":"prod-sre"},"node-1":{"deployment":"checkout","namespace":"prod-sre"},"node-2":{"deployment":"catalog","namespace":"prod-sre"},"node-3":{"deployment":"cart","namespace":"prod-sre"},"node-4":{"deployment":"auth","namespace":"prod-sre"}}'
 ```

deploy/grafana/provisioning/dashboards/json/antiatropos-live.json

@@ -54,7 +54,7 @@
       },
       "targets": [
         {
-          "expr": "sum by (action_type, ack_class) (rate(antiatropos_actions_total{task_id=~\"$task\",mode=~\"$mode\"}[1m]))",
+          "expr": "sum by (action_type, ack_class) (rate(antiatropos_actions_total{task_id=~\"$task\",mode=~\"$mode\"}[5m]))",
           "legendFormat": "{{action_type}} {{ack_class}}",
           "refId": "A"
         }
@@ -261,7 +261,7 @@
       },
       "targets": [
         {
-          "expr": "histogram_quantile(0.95, sum(rate(antiatropos_executor_latency_ms_bucket{mode=~\"$mode\"}[2m])) by (le, mode))",
+          "expr": "histogram_quantile(0.95, sum(rate(antiatropos_executor_latency_ms_bucket{mode=~\"$mode\"}[5m])) by (le, mode))",
           "legendFormat": "p95 {{mode}}",
           "refId": "A"
         },

deploy/grafana/provisioning/dashboards/json/antiatropos-overview.json

@@ -76,8 +76,8 @@
       "targets": [
         {
           "editorMode": "code",
-          "expr": "scalar(avg(last_over_time(antiatropos_reward{mode=\"simulated\"}[1m])))",
-          "legendFormat": "reward (simulated)",
+          "expr": "scalar(avg(last_over_time(antiatropos_reward{mode=~\"live|simulated|hybrid|aws\"}[1m])))",
+          "legendFormat": "reward (all modes)",
           "range": true,
           "refId": "A"
         }
@@ -143,8 +143,8 @@
       "targets": [
         {
           "editorMode": "code",
-          "expr": "scalar(avg(last_over_time(antiatropos_total_queue_backlog{mode=\"simulated\"}[1m])))",
-          "legendFormat": "queue backlog (simulated)",
+          "expr": "scalar(avg(last_over_time(antiatropos_total_queue_backlog{mode=~\"live|simulated|hybrid|aws\"}[1m])))",
+          "legendFormat": "queue backlog (all modes)",
           "range": true,
           "refId": "A"
         }
@@ -210,8 +210,8 @@
       "targets": [
         {
           "editorMode": "code",
-          "expr": "scalar(avg(last_over_time(antiatropos_average_latency_norm{mode=\"simulated\"}[1m])))",
-          "legendFormat": "latency (simulated)",
+          "expr": "scalar(avg(last_over_time(antiatropos_average_latency_norm{mode=~\"live|simulated|hybrid|aws\"}[1m])))",
+          "legendFormat": "latency (all modes)",
           "range": true,
           "refId": "A"
         }
@@ -277,8 +277,8 @@
       "targets": [
         {
           "editorMode": "code",
-          "expr": "scalar(avg(last_over_time(antiatropos_lyapunov_energy{mode=\"simulated\"}[1m])))",
-          "legendFormat": "lyapunov energy (simulated)",
+          "expr": "scalar(avg(last_over_time(antiatropos_lyapunov_energy{mode=~\"live|simulated|hybrid|aws\"}[1m])))",
+          "legendFormat": "lyapunov energy (all modes)",
           "range": true,
           "refId": "A"
         }
@@ -369,14 +369,14 @@
       "targets": [
         {
           "editorMode": "code",
-          "expr": "antiatropos_reward{mode=\"simulated\"}",
+          "expr": "antiatropos_reward{mode=~\"live|simulated|hybrid|aws\"}",
           "legendFormat": "reward {{task_id}} ({{mode}})",
           "range": true,
           "refId": "A"
         },
         {
           "editorMode": "code",
-          "expr": "antiatropos_lyapunov_energy{mode=\"simulated\"}",
+          "expr": "antiatropos_lyapunov_energy{mode=~\"live|simulated|hybrid|aws\"}",
           "legendFormat": "lyapunov {{task_id}} ({{mode}})",
           "range": true,
           "refId": "B"
@@ -468,14 +468,14 @@
       "targets": [
         {
           "editorMode": "code",
-          "expr": "antiatropos_total_queue_backlog{mode=\"simulated\"}",
+          "expr": "antiatropos_total_queue_backlog{mode=~\"live|simulated|hybrid|aws\"}",
           "legendFormat": "queue {{task_id}} ({{mode}})",
           "range": true,
           "refId": "A"
         },
         {
           "editorMode": "code",
-          "expr": "antiatropos_average_latency_norm{mode=\"simulated\"}",
+          "expr": "antiatropos_average_latency_norm{mode=~\"live|simulated|hybrid|aws\"}",
           "legendFormat": "latency {{task_id}} ({{mode}})",
           "range": true,
           "refId": "B"
@@ -535,14 +535,14 @@
       "targets": [
         {
           "editorMode": "code",
-          "expr": "sum by (task_id, mode) (rate(antiatropos_steps_total{mode=\"simulated\"}[1m]))",
+          "expr": "sum by (task_id, mode) (rate(antiatropos_steps_total{mode=~\"live|simulated|hybrid|aws\"}[1m]))",
           "legendFormat": "steps/sec {{task_id}} ({{mode}})",
           "range": true,
           "refId": "A"
         },
         {
           "editorMode": "code",
-          "expr": "sum by (task_id, mode, action_type) (rate(antiatropos_actions_total{mode=\"simulated\"}[1m]))",
+          "expr": "sum by (task_id, mode, action_type) (rate(antiatropos_actions_total{mode=~\"live|simulated|hybrid|aws\"}[1m]))",
           "legendFormat": "actions/sec {{action_type}} ({{task_id}}, {{mode}})",
           "range": true,
           "refId": "B"
@@ -602,14 +602,14 @@
       "targets": [
         {
           "editorMode": "code",
-          "expr": "sum by (mode, error_code) (rate(antiatropos_executor_errors_total{mode=\"simulated\"}[5m]))",
+          "expr": "sum by (mode, error_code) (rate(antiatropos_executor_errors_total{mode=~\"live|simulated|hybrid|aws\"}[5m]))",
           "legendFormat": "executor errors {{error_code}} ({{mode}})",
           "range": true,
           "refId": "A"
         },
         {
           "editorMode": "code",
-          "expr": "histogram_quantile(0.95, sum(rate(antiatropos_executor_latency_ms_bucket{mode=\"simulated\"}[5m])) by (le, mode))",
+          "expr": "histogram_quantile(0.95, sum(rate(antiatropos_executor_latency_ms_bucket{mode=~\"live|simulated|hybrid|aws\"}[5m])) by (le, mode))",
           "legendFormat": "p95 executor latency {{mode}}",
           "range": true,
           "refId": "B"
@@ -640,3 +640,8 @@
   "version": 2,
   "weekStart": ""
 }
+
+
+
+
+

deploy/grafana/provisioning/datasources/prometheus.yaml

@@ -5,13 +5,6 @@ datasources:
     uid: PBFA97CFB590B2093
     type: prometheus
     access: proxy
-    url: https://aps-workspaces.ap-south-1.amazonaws.com/workspaces/ws-ba100a53-b185-4e64-8619-59c2bb3d2fce
+    url: http://host.docker.internal:9090
     isDefault: true
     editable: true
-    jsonData:
-      sigV4Auth: true
-      sigV4AuthType: keys
-      sigV4Region: ap-south-1
-    secureJsonData:
-      sigV4AccessKey: "${AWS_ACCESS_KEY_ID}"
-      sigV4SecretKey: "${AWS_SECRET_ACCESS_KEY}"

deploy/kind-maxpods-250.yaml

@@ -0,0 +1,11 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+name: antiatropos-local
+nodes:
+  - role: control-plane
+    kubeadmConfigPatches:
+      - |
+        kind: InitConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            max-pods: "250"

deploy/local-laptop.yaml

@@ -17,11 +17,63 @@ spec:
     metadata:
       labels:
         app: auth
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
+        prometheus.io/path: "/metrics"
     spec:
       containers:
       - name: auth
-        image: alpine:latest
-        command: ["sleep", "infinity"]
+        image: python:3.12-alpine
+        env:
+        - name: NODE_ID
+          value: node-4
+        - name: BASE_QUEUE
+          value: "6"
+        command: ["/bin/sh", "-lc"]
+        args:
+        - |
+          mkdir -p /www
+          echo ok > /www/index.html
+          python -m http.server 8080 --directory /www >/tmp/http.log 2>&1 &
+          req=0; err=0; cpu_total=0
+          while true; do
+            t=$(date +%s)
+            noise=$((t % 11))
+            req=$((req + 30 + noise))
+            q=$((BASE_QUEUE + (t % 20) - 10))
+            if [ "$q" -lt 0 ]; then q=0; fi
+            err=$((err + q / 20))
+            cpu_inc=$((10 + q / 10))
+            cpu_total=$((cpu_total + cpu_inc))
+            lat_ms=$((35 + q * 3))
+            b005=$((req / 5)); b01=$((req / 3)); b025=$((req / 2)); b05=$((req * 3 / 4)); b1=$req; b2=$req
+            lat_sum=$(awk "BEGIN {printf \"%.3f\", $req * $lat_ms / 1000.0}")
+            {
+              echo "# HELP http_requests_total Synthetic request counter"
+              echo "# TYPE http_requests_total counter"
+              echo "http_requests_total{node_id=\"${NODE_ID}\",status=\"200\"} ${req}"
+              echo "http_requests_total{node_id=\"${NODE_ID}\",status=\"500\"} ${err}"
+              echo "# HELP queue_depth Synthetic queue depth"
+              echo "# TYPE queue_depth gauge"
+              echo "queue_depth{node_id=\"${NODE_ID}\"} ${q}"
+              echo "# HELP container_cpu_usage_seconds_total Synthetic CPU counter"
+              echo "# TYPE container_cpu_usage_seconds_total counter"
+              echo "container_cpu_usage_seconds_total{node_id=\"${NODE_ID}\"} ${cpu_total}"
+              echo "# HELP http_request_duration_seconds Synthetic request duration histogram"
+              echo "# TYPE http_request_duration_seconds histogram"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.05\"} ${b005}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.1\"} ${b01}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.25\"} ${b025}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.5\"} ${b05}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"1\"} ${b1}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"2\"} ${b2}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"+Inf\"} ${req}"
+              echo "http_request_duration_seconds_count{node_id=\"${NODE_ID}\"} ${req}"
+              echo "http_request_duration_seconds_sum{node_id=\"${NODE_ID}\"} ${lat_sum}"
+            } > /www/metrics
+            sleep 2
+          done
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -37,11 +89,63 @@ spec:
     metadata:
       labels:
         app: cart
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
+        prometheus.io/path: "/metrics"
     spec:
       containers:
       - name: cart
-        image: alpine:latest
-        command: ["sleep", "infinity"]
+        image: python:3.12-alpine
+        env:
+        - name: NODE_ID
+          value: node-3
+        - name: BASE_QUEUE
+          value: "14"
+        command: ["/bin/sh", "-lc"]
+        args:
+        - |
+          mkdir -p /www
+          echo ok > /www/index.html
+          python -m http.server 8080 --directory /www >/tmp/http.log 2>&1 &
+          req=0; err=0; cpu_total=0
+          while true; do
+            t=$(date +%s)
+            noise=$((t % 11))
+            req=$((req + 30 + noise))
+            q=$((BASE_QUEUE + (t % 20) - 10))
+            if [ "$q" -lt 0 ]; then q=0; fi
+            err=$((err + q / 20))
+            cpu_inc=$((10 + q / 10))
+            cpu_total=$((cpu_total + cpu_inc))
+            lat_ms=$((35 + q * 3))
+            b005=$((req / 5)); b01=$((req / 3)); b025=$((req / 2)); b05=$((req * 3 / 4)); b1=$req; b2=$req
+            lat_sum=$(awk "BEGIN {printf \"%.3f\", $req * $lat_ms / 1000.0}")
+            {
+              echo "# HELP http_requests_total Synthetic request counter"
+              echo "# TYPE http_requests_total counter"
+              echo "http_requests_total{node_id=\"${NODE_ID}\",status=\"200\"} ${req}"
+              echo "http_requests_total{node_id=\"${NODE_ID}\",status=\"500\"} ${err}"
+              echo "# HELP queue_depth Synthetic queue depth"
+              echo "# TYPE queue_depth gauge"
+              echo "queue_depth{node_id=\"${NODE_ID}\"} ${q}"
+              echo "# HELP container_cpu_usage_seconds_total Synthetic CPU counter"
+              echo "# TYPE container_cpu_usage_seconds_total counter"
+              echo "container_cpu_usage_seconds_total{node_id=\"${NODE_ID}\"} ${cpu_total}"
+              echo "# HELP http_request_duration_seconds Synthetic request duration histogram"
+              echo "# TYPE http_request_duration_seconds histogram"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.05\"} ${b005}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.1\"} ${b01}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.25\"} ${b025}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.5\"} ${b05}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"1\"} ${b1}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"2\"} ${b2}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"+Inf\"} ${req}"
+              echo "http_request_duration_seconds_count{node_id=\"${NODE_ID}\"} ${req}"
+              echo "http_request_duration_seconds_sum{node_id=\"${NODE_ID}\"} ${lat_sum}"
+            } > /www/metrics
+            sleep 2
+          done
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -57,11 +161,63 @@ spec:
     metadata:
       labels:
         app: catalog
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
+        prometheus.io/path: "/metrics"
     spec:
       containers:
       - name: catalog
-        image: alpine:latest
-        command: ["sleep", "infinity"]
+        image: python:3.12-alpine
+        env:
+        - name: NODE_ID
+          value: node-2
+        - name: BASE_QUEUE
+          value: "20"
+        command: ["/bin/sh", "-lc"]
+        args:
+        - |
+          mkdir -p /www
+          echo ok > /www/index.html
+          python -m http.server 8080 --directory /www >/tmp/http.log 2>&1 &
+          req=0; err=0; cpu_total=0
+          while true; do
+            t=$(date +%s)
+            noise=$((t % 11))
+            req=$((req + 30 + noise))
+            q=$((BASE_QUEUE + (t % 20) - 10))
+            if [ "$q" -lt 0 ]; then q=0; fi
+            err=$((err + q / 20))
+            cpu_inc=$((10 + q / 10))
+            cpu_total=$((cpu_total + cpu_inc))
+            lat_ms=$((35 + q * 3))
+            b005=$((req / 5)); b01=$((req / 3)); b025=$((req / 2)); b05=$((req * 3 / 4)); b1=$req; b2=$req
+            lat_sum=$(awk "BEGIN {printf \"%.3f\", $req * $lat_ms / 1000.0}")
+            {
+              echo "# HELP http_requests_total Synthetic request counter"
+              echo "# TYPE http_requests_total counter"
+              echo "http_requests_total{node_id=\"${NODE_ID}\",status=\"200\"} ${req}"
+              echo "http_requests_total{node_id=\"${NODE_ID}\",status=\"500\"} ${err}"
+              echo "# HELP queue_depth Synthetic queue depth"
+              echo "# TYPE queue_depth gauge"
+              echo "queue_depth{node_id=\"${NODE_ID}\"} ${q}"
+              echo "# HELP container_cpu_usage_seconds_total Synthetic CPU counter"
+              echo "# TYPE container_cpu_usage_seconds_total counter"
+              echo "container_cpu_usage_seconds_total{node_id=\"${NODE_ID}\"} ${cpu_total}"
+              echo "# HELP http_request_duration_seconds Synthetic request duration histogram"
+              echo "# TYPE http_request_duration_seconds histogram"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.05\"} ${b005}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.1\"} ${b01}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.25\"} ${b025}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.5\"} ${b05}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"1\"} ${b1}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"2\"} ${b2}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"+Inf\"} ${req}"
+              echo "http_request_duration_seconds_count{node_id=\"${NODE_ID}\"} ${req}"
+              echo "http_request_duration_seconds_sum{node_id=\"${NODE_ID}\"} ${lat_sum}"
+            } > /www/metrics
+            sleep 2
+          done
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -77,11 +233,63 @@ spec:
     metadata:
       labels:
         app: checkout
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
+        prometheus.io/path: "/metrics"
     spec:
       containers:
       - name: checkout
-        image: alpine:latest
-        command: ["sleep", "infinity"]
+        image: python:3.12-alpine
+        env:
+        - name: NODE_ID
+          value: node-1
+        - name: BASE_QUEUE
+          value: "24"
+        command: ["/bin/sh", "-lc"]
+        args:
+        - |
+          mkdir -p /www
+          echo ok > /www/index.html
+          python -m http.server 8080 --directory /www >/tmp/http.log 2>&1 &
+          req=0; err=0; cpu_total=0
+          while true; do
+            t=$(date +%s)
+            noise=$((t % 11))
+            req=$((req + 30 + noise))
+            q=$((BASE_QUEUE + (t % 20) - 10))
+            if [ "$q" -lt 0 ]; then q=0; fi
+            err=$((err + q / 20))
+            cpu_inc=$((10 + q / 10))
+            cpu_total=$((cpu_total + cpu_inc))
+            lat_ms=$((35 + q * 3))
+            b005=$((req / 5)); b01=$((req / 3)); b025=$((req / 2)); b05=$((req * 3 / 4)); b1=$req; b2=$req
+            lat_sum=$(awk "BEGIN {printf \"%.3f\", $req * $lat_ms / 1000.0}")
+            {
+              echo "# HELP http_requests_total Synthetic request counter"
+              echo "# TYPE http_requests_total counter"
+              echo "http_requests_total{node_id=\"${NODE_ID}\",status=\"200\"} ${req}"
+              echo "http_requests_total{node_id=\"${NODE_ID}\",status=\"500\"} ${err}"
+              echo "# HELP queue_depth Synthetic queue depth"
+              echo "# TYPE queue_depth gauge"
+              echo "queue_depth{node_id=\"${NODE_ID}\"} ${q}"
+              echo "# HELP container_cpu_usage_seconds_total Synthetic CPU counter"
+              echo "# TYPE container_cpu_usage_seconds_total counter"
+              echo "container_cpu_usage_seconds_total{node_id=\"${NODE_ID}\"} ${cpu_total}"
+              echo "# HELP http_request_duration_seconds Synthetic request duration histogram"
+              echo "# TYPE http_request_duration_seconds histogram"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.05\"} ${b005}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.1\"} ${b01}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.25\"} ${b025}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.5\"} ${b05}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"1\"} ${b1}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"2\"} ${b2}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"+Inf\"} ${req}"
+              echo "http_request_duration_seconds_count{node_id=\"${NODE_ID}\"} ${req}"
+              echo "http_request_duration_seconds_sum{node_id=\"${NODE_ID}\"} ${lat_sum}"
+            } > /www/metrics
+            sleep 2
+          done
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -89,7 +297,7 @@ metadata:
   name: payments
   namespace: prod-sre
 spec:
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       app: payments
@@ -97,8 +305,61 @@ spec:
     metadata:
       labels:
         app: payments
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "8080"
+        prometheus.io/path: "/metrics"
     spec:
       containers:
       - name: payments
-        image: alpine:latest
-        command: ["sleep", "infinity"]
+        image: python:3.12-alpine
+        env:
+        - name: NODE_ID
+          value: node-0
+        - name: BASE_QUEUE
+          value: "30"
+        command: ["/bin/sh", "-lc"]
+        args:
+        - |
+          mkdir -p /www
+          echo ok > /www/index.html
+          python -m http.server 8080 --directory /www >/tmp/http.log 2>&1 &
+          req=0; err=0; cpu_total=0
+          while true; do
+            t=$(date +%s)
+            noise=$((t % 11))
+            req=$((req + 30 + noise))
+            q=$((BASE_QUEUE + (t % 20) - 10))
+            if [ "$q" -lt 0 ]; then q=0; fi
+            err=$((err + q / 20))
+            cpu_inc=$((10 + q / 10))
+            cpu_total=$((cpu_total + cpu_inc))
+            lat_ms=$((35 + q * 3))
+            b005=$((req / 5)); b01=$((req / 3)); b025=$((req / 2)); b05=$((req * 3 / 4)); b1=$req; b2=$req
+            lat_sum=$(awk "BEGIN {printf \"%.3f\", $req * $lat_ms / 1000.0}")
+            {
+              echo "# HELP http_requests_total Synthetic request counter"
+              echo "# TYPE http_requests_total counter"
+              echo "http_requests_total{node_id=\"${NODE_ID}\",status=\"200\"} ${req}"
+              echo "http_requests_total{node_id=\"${NODE_ID}\",status=\"500\"} ${err}"
+              echo "# HELP queue_depth Synthetic queue depth"
+              echo "# TYPE queue_depth gauge"
+              echo "queue_depth{node_id=\"${NODE_ID}\"} ${q}"
+              echo "# HELP container_cpu_usage_seconds_total Synthetic CPU counter"
+              echo "# TYPE container_cpu_usage_seconds_total counter"
+              echo "container_cpu_usage_seconds_total{node_id=\"${NODE_ID}\"} ${cpu_total}"
+              echo "# HELP http_request_duration_seconds Synthetic request duration histogram"
+              echo "# TYPE http_request_duration_seconds histogram"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.05\"} ${b005}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.1\"} ${b01}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.25\"} ${b025}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"0.5\"} ${b05}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"1\"} ${b1}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"2\"} ${b2}"
+              echo "http_request_duration_seconds_bucket{node_id=\"${NODE_ID}\",le=\"+Inf\"} ${req}"
+              echo "http_request_duration_seconds_count{node_id=\"${NODE_ID}\"} ${req}"
+              echo "http_request_duration_seconds_sum{node_id=\"${NODE_ID}\"} ${lat_sum}"
+            } > /www/metrics
+            sleep 2
+          done
+

deploy/local/grafana-local-values.yaml

@@ -0,0 +1,34 @@
+adminUser: admin
+adminPassword: antiatropos
+
+service:
+  type: ClusterIP
+
+persistence:
+  enabled: false
+
+resources:
+  requests:
+    cpu: 100m
+    memory: 192Mi
+  limits:
+    cpu: 400m
+    memory: 384Mi
+
+datasources:
+  datasources.yaml:
+    apiVersion: 1
+    datasources:
+      - name: Prometheus
+        type: prometheus
+        access: proxy
+        url: http://prometheus-server.monitoring.svc.cluster.local
+        isDefault: true
+        editable: true
+
+sidecar:
+  dashboards:
+    enabled: true
+    label: grafana_dashboard
+    labelValue: "1"
+    searchNamespace: ALL

deploy/local/prometheus-local-values.yaml

@@ -0,0 +1,49 @@
+alertmanager:
+  enabled: false
+
+kube-state-metrics:
+  enabled: false
+
+prometheus-node-exporter:
+  enabled: false
+
+prometheus-pushgateway:
+  enabled: false
+
+extraScrapeConfigs: |
+  - job_name: 'antiatropos-fastapi'
+    metrics_path: /metrics
+    static_configs:
+      - targets: ['host.docker.internal:8000']
+
+  - job_name: 'prod-sre-annotated-pods'
+    kubernetes_sd_configs:
+      - role: pod
+        namespaces:
+          names: ['prod-sre']
+    relabel_configs:
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+        action: keep
+        regex: true
+      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
+        action: replace
+        target_label: __metrics_path__
+        regex: (.+)
+      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+        action: replace
+        regex: ([^:]+)(?::\d+)?;(\d+)
+        replacement: $1:$2
+        target_label: __address__
+
+server:
+  persistentVolume:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi
+  service:
+    type: ClusterIP

server/Dockerfile

@@ -85,7 +85,7 @@ ENV KUBECONFIG="mock"
 ENV ANTIATROPOS_K8S_NAMESPACE="default"
 ENV ANTIATROPOS_DEPLOYMENT_PREFIX=""
 ENV ANTIATROPOS_MIN_REPLICAS="1"
-ENV ANTIATROPOS_MAX_REPLICAS="20"
+ENV ANTIATROPOS_MAX_REPLICAS=""
 ENV ANTIATROPOS_SCALE_STEP="3"
 ENV ANTIATROPOS_WORKLOAD_MAP="{}"
 ENV ANTIATROPOS_NODE_DEPLOYMENT_MAP="{}"

server/app.py

@@ -84,6 +84,9 @@ def runtime_config():
         except Exception:
             mapped_nodes = []
 
+    raw_max_replicas = os.getenv("ANTIATROPOS_MAX_REPLICAS", "")
+    max_replicas_display = raw_max_replicas if raw_max_replicas.strip() else "unbounded"
+
     return {
         "env_mode": os.getenv("ANTIATROPOS_ENV_MODE", "simulated"),
         "reward_output_mode": os.getenv("ANTIATROPOS_REWARD_OUTPUT_MODE", "normalized"),
@@ -91,7 +94,7 @@ def runtime_config():
         "kubeconfig_configured": bool(os.getenv("KUBECONFIG")),
         "k8s_namespace": os.getenv("ANTIATROPOS_K8S_NAMESPACE", "default"),
         "min_replicas": os.getenv("ANTIATROPOS_MIN_REPLICAS", "1"),
-        "max_replicas": os.getenv("ANTIATROPOS_MAX_REPLICAS", "20"),
+        "max_replicas": max_replicas_display,
         "scale_step": os.getenv("ANTIATROPOS_SCALE_STEP", "3"),
         "strict_real": os.getenv("ANTIATROPOS_STRICT_REAL", "false"),
         "workload_map_configured": bool(raw_map),

start-grafana.ps1

@@ -1,29 +1,12 @@
 docker stop antiatropos-grafana 2>$null
 docker rm antiatropos-grafana 2>$null
 
-Write-Host "Fetching AWS credentials..."
-$AccessKey = (aws configure get aws_access_key_id).Trim()
-$SecretKey = (aws configure get aws_secret_access_key).Trim()
-$SessionToken = aws configure get aws_session_token
-if ($SessionToken) { $SessionToken = $SessionToken.Trim() }
+Write-Host "Starting local Grafana (datasource -> host.docker.internal:9090)..."
 
-Write-Host "Starting Grafana with injected AWS credentials..."
+docker run -d --name antiatropos-grafana -p 3000:3000 `
+  -v "$PWD\deploy\grafana\provisioning:/etc/grafana/provisioning:ro" `
+  -e GF_AUTH_ANONYMOUS_ENABLED=true `
+  -e GF_AUTH_ANONYMOUS_ORG_ROLE=Admin `
+  grafana/grafana:latest | Out-Null
 
-$DockerCmd = "docker run -d --name antiatropos-grafana -p 3000:3000 " + `
-  "-v ""$PWD\deploy\grafana\provisioning:/etc/grafana/provisioning:ro"" " + `
-  "-e GF_AUTH_ANONYMOUS_ENABLED=true " + `
-  "-e GF_AUTH_ANONYMOUS_ORG_ROLE=Admin " + `
-  "-e GF_AUTH_SIGV4_AUTH_ENABLED=true " + `
-  "-e AWS_ACCESS_KEY_ID=""$AccessKey"" " + `
-  "-e AWS_SECRET_ACCESS_KEY=""$SecretKey"" " + `
-  "-e AWS_REGION=ap-south-1 "
-
-if ($SessionToken) {
-    $DockerCmd += "-e AWS_SESSION_TOKEN=""$SessionToken"" "
-}
-
-$DockerCmd += "grafana/grafana:latest"
-
-Invoke-Expression $DockerCmd
-
-Write-Host "Grafana is running! Open http://localhost:3000 in your browser."
+Write-Host "Grafana is running at http://localhost:3000"

telemetry/mapping.py

@@ -25,9 +25,11 @@ class MetricMapper:
 
         # Safe default mapping for local demos.
         return {
-            "web-node-1": "node-0",
-            "api-node-2": "node-1",
-            "db-node-3": "node-2"
+            "payments": "node-0",
+            "checkout": "node-1",
+            "catalog": "node-2",
+            "cart": "node-3",
+            "auth": "node-4",
         }
 
     def _resolve_node_id(self, labels: Dict[str, Any]) -> Optional[str]:

telemetry/prometheus_client.py

@@ -1,10 +1,13 @@
 import os
 import random
+import logging
 from typing import Any, Dict, List, Optional
 import requests
 from pydantic import BaseModel
 from .mapping import MetricMapper
 
+logger = logging.getLogger("antiatropos.telemetry")
+
 class TelemetryRecord(BaseModel):
     node_id: str
     latency_ms: float
@@ -49,7 +52,7 @@ class PrometheusClient:
             'sum(queue_depth) by (pod)'
         )
         
-    def fetch_latest_metrics(self, node_ids: List[str]) -> Dict[str, TelemetryRecord]:
+    def fetch_latest_metrics(self, node_ids: List[str]) -> Dict[str, Any]:
         """
         Query Prometheus for the latest metrics for the given nodes.
         Returns a mapping from node_id to TelemetryRecord.
@@ -65,9 +68,9 @@ class PrometheusClient:
                 raise
             return self._generate_mock_metrics(node_ids)
 
-    def _fetch_real_metrics(self, node_ids: List[str]) -> Dict[str, TelemetryRecord]:
+    def _fetch_real_metrics(self, node_ids: List[str]) -> Dict[str, Any]:
         """Fetches node telemetry from Prometheus instant queries."""
-        metrics: Dict[str, TelemetryRecord] = {}
+        metrics: Dict[str, Any] = {}
         saw_any_real_signal = False
 
         req_by_node = self._collect_metric_values("request_rate", self.request_rate_query, node_ids)
@@ -85,19 +88,35 @@ class PrometheusClient:
 
             if any(v is not None for v in (req_rate, lat_ms, err_rate, cpu, q_depth)):
                 saw_any_real_signal = True
+            else:
+                # No usable sample for this node this cycle; skip reconciliation
+                # so simulator dynamics are preserved instead of being collapsed
+                # toward zero by synthetic defaults.
+                continue
 
-            metrics[node_id] = TelemetryRecord(
-                node_id=node_id,
-                latency_ms=float(lat_ms if lat_ms is not None else 20.0),
-                request_rate=float(req_rate if req_rate is not None else 0.0),
-                error_rate=max(0.0, min(1.0, float(err_rate if err_rate is not None else 0.0))),
-                cpu_utilization=max(0.0, min(1.0, float(cpu if cpu is not None else 0.0))),
-                queue_depth=max(0.0, float(q_depth if q_depth is not None else 0.0)),
-            )
+            node_payload: Dict[str, float] = {}
+            if lat_ms is not None:
+                node_payload["latency_ms"] = float(lat_ms)
+            if req_rate is not None:
+                node_payload["request_rate"] = float(req_rate)
+            if err_rate is not None:
+                node_payload["error_rate"] = max(0.0, min(1.0, float(err_rate)))
+            if cpu is not None:
+                node_payload["cpu_utilization"] = max(0.0, min(1.0, float(cpu)))
+            if q_depth is not None:
+                node_payload["queue_depth"] = max(0.0, float(q_depth))
+            if node_payload:
+                metrics[node_id] = node_payload
 
         if self.strict_real and not saw_any_real_signal:
             raise RuntimeError("Prometheus returned no usable real telemetry for requested node IDs.")
 
+        if not saw_any_real_signal:
+            logger.warning(
+                "No per-node Prometheus samples found for configured queries; "
+                "skipping telemetry reconciliation for this step."
+            )
+
         return metrics
 
     def _collect_metric_values(