infra(k8s): add HorizontalPodAutoscaler for api (CPU-based) and worker (queue-depth-based)

kubernetes/hpa.yaml

@@ -0,0 +1,145 @@
+# HorizontalPodAutoscaler for the AI Code Review Agent
+#
+# Two HPAs:
+#
+#   api-hpa    — scales the FastAPI pod on CPU utilisation.
+#                Rationale: API pods are stateless and CPU-bound during
+#                request parsing and response serialisation. CPU at 60%
+#                is a reliable proxy for saturation without over-provisioning.
+#
+#   worker-hpa — scales the Celery worker pod on a custom metric:
+#                the depth of the Celery task queue exposed via Prometheus.
+#                Rationale: CPU is a lagging indicator for queue workers —
+#                a worker sitting idle waiting on an LLM response shows low
+#                CPU but the queue may be backing up. Queue depth is a
+#                leading indicator that triggers scale-out before jobs pile up.
+#
+# Prerequisites:
+#   - metrics-server installed in the cluster (for CPU-based HPA)
+#   - Prometheus Adapter installed and configured to expose
+#     celery_queue_length as a custom metric (for worker HPA)
+#   - kubectl apply -f kubernetes/hpa.yaml
+
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: api-hpa
+  namespace: default
+  labels:
+    app: ai-code-review-agent
+    component: api
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: api                      # must match metadata.name in deployment.yaml
+
+  minReplicas: 2                   # always run at least 2 for availability
+  maxReplicas: 10                  # cap prevents runaway LLM cost under attack
+
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 60   # scale out when avg CPU across pods exceeds 60%
+
+    - type: Resource
+      resource:
+        name: memory
+        target:
+          type: Utilization
+          averageUtilization: 75   # secondary guard — large repos spike memory
+
+  behavior:
+    scaleUp:
+      stabilizationWindowSeconds: 30     # react quickly to traffic spikes
+      policies:
+        - type: Pods
+          value: 2                       # add at most 2 pods per scale event
+          periodSeconds: 30
+    scaleDown:
+      stabilizationWindowSeconds: 300    # wait 5 min before scaling down
+      policies:                          # prevents flapping after burst traffic
+        - type: Pods
+          value: 1                       # remove at most 1 pod per scale event
+          periodSeconds: 60
+
+---
+
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: worker-hpa
+  namespace: default
+  labels:
+    app: ai-code-review-agent
+    component: worker
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: worker                   # must match metadata.name in deployment.yaml
+
+  minReplicas: 1                   # one worker is enough at idle
+  maxReplicas: 8                   # each worker holds 2 Celery slots (--concurrency=2)
+                                   # so 8 pods = 16 concurrent analysis tasks max
+
+  metrics:
+    - type: Pods
+      pods:
+        metric:
+          name: celery_queue_length   # exposed by Prometheus Adapter
+                                      # configure in prometheus-adapter ConfigMap:
+                                      #   - seriesQuery: 'celery_queue_length{queue=~"high|low"}'
+                                      #     name: { as: "celery_queue_length" }
+                                      #     metricsQuery: 'sum(celery_queue_length{<<.LabelMatchers>>})'
+        target:
+          type: AverageValue
+          averageValue: "5"           # scale out when avg queue depth per pod exceeds 5 jobs
+
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 70      # fallback CPU metric if Prometheus Adapter unavailable
+
+  behavior:
+    scaleUp:
+      stabilizationWindowSeconds: 15  # workers need to react faster than API pods
+      policies:
+        - type: Pods
+          value: 2
+          periodSeconds: 15           # burst: add 2 workers every 15s if queue keeps growing
+    scaleDown:
+      stabilizationWindowSeconds: 600 # wait 10 min — draining a long analysis task
+      policies:                       # takes time; premature scale-down kills in-flight jobs
+        - type: Pods
+          value: 1
+          periodSeconds: 120
+
+---
+
+# Prometheus Adapter ConfigMap reference (apply separately if not already present)
+# This tells the adapter how to translate the Prometheus metric into a
+# Kubernetes custom metric that the worker HPA can consume.
+#
+# apiVersion: v1
+# kind: ConfigMap
+# metadata:
+#   name: prometheus-adapter-config
+#   namespace: monitoring
+# data:
+#   config.yaml: |
+#     rules:
+#       - seriesQuery: 'celery_queue_length{queue=~"high|low"}'
+#         resources:
+#           overrides:
+#             namespace: { resource: namespace }
+#             pod:       { resource: pod }
+#         name:
+#           matches: "celery_queue_length"
+#           as:      "celery_queue_length"
+#         metricsQuery: 'sum(celery_queue_length{<<.LabelMatchers>>})'