phase-5/scripts/final-verification.sh

#!/bin/bash
# Final Verification Script for Phase 5
# Verifies all components are properly configured and deployed
#

set -e

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Phase 5 Final Verification${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""

FAILURES=0
WARNINGS=0

# 1. Check Kubernetes cluster connectivity
echo -e "${YELLOW}1. Checking Kubernetes cluster...${NC}"

if kubectl cluster-info > /dev/null 2>&1; then
  echo -e "${GREEN}✓ Kubernetes cluster is accessible${NC}"
  echo "  Cluster: $(kubectl config current-context)"
else
  echo -e "${RED}✗ Cannot connect to Kubernetes cluster${NC}"
  FAILURES=$((FAILURES + 1))
fi

# 2. Check namespace
echo ""
echo -e "${YELLOW}2. Checking namespace...${NC}"

if kubectl get namespace phase-5 > /dev/null 2>&1; then
  echo -e "${GREEN}✓ Namespace 'phase-5' exists${NC}"
else
  echo -e "${YELLOW}⚠ Namespace 'phase-5' not found${NC}"
  echo "  Run: kubectl create namespace phase-5"
  WARNINGS=$((WARNINGS + 1))
fi

# 3. Check deployments
echo ""
echo -e "${YELLOW}3. Checking deployments...${NC}"

DEPLOYMENTS=("backend" "notification" "postgres")
for deployment in "${DEPLOYMENTS[@]}"; do
  if kubectl get deployment "$deployment" -n phase-5 > /dev/null 2>&1; then
    READY_REPLICAS=$(kubectl get deployment "$deployment" -n phase-5 -o jsonpath='{.status.readyReplicas}')
    DESIRED_REPLICAS=$(kubectl get deployment "$deployment" -n phase-5 -o jsonpath='{.spec.replicas}')

    if [ "$READY_REPLICAS" == "$DESIRED_REPLICAS" ]; then
      echo -e "${GREEN}✓ Deployment '$deployment' is ready (${READY_REPLICAS}/${DESIRED_REPLICAS})${NC}"
    else
      echo -e "${YELLOW}⚠ Deployment '$deployment' not ready (${READY_REPLICAS}/${DESIRED_REPLICAS})${NC}"
      WARNINGS=$((WARNINGS + 1))
    fi
  else
    echo -e "${RED}✗ Deployment '$deployment' not found${NC}"
    FAILURES=$((FAILURES + 1))
  fi
done

# 4. Check pods
echo ""
echo -e "${YELLOW}4. Checking pods...${NC}"

PODS=$(kubectl get pods -n phase-5 --no-headers 2>/dev/null | wc -l)
RUNNING=$(kubectl get pods -n phase-5 --no-headers 2>/dev/null | grep "Running" | wc -l)

if [ "$PODS" -gt 0 ]; then
  echo -e "${GREEN}✓ Found ${PODS} pods (${RUNNING} running)${NC}"

  # Check for failing pods
  FAILED=$(kubectl get pods -n phase-5 --no-headers 2>/dev/null | grep -v "Running\|Completed" | wc -l)
  if [ "$FAILED" -gt 0 ]; then
    echo -e "${YELLOW}⚠ ${FAILED} pods are not running${NC}"
    kubectl get pods -n phase-5 | grep -v "Running\|Completed"
    WARNINGS=$((WARNINGS + 1))
  fi
else
  echo -e "${YELLOW}⚠ No pods found${NC}"
  WARNINGS=$((WARNINGS + 1))
fi

# 5. Check services
echo ""
echo -e "${YELLOW}5. Checking services...${NC}"

SERVICES=("backend-service" "notification-service" "postgres")
for service in "${SERVICES[@]}"; do
  if kubectl get service "$service" -n phase-5 > /dev/null 2>&1; then
    TYPE=$(kubectl get service "$service" -n phase-5 -o jsonpath='{.spec.type}')
    echo -e "${GREEN}✓ Service '$service' exists (${TYPE})${NC}"
  else
    echo -e "${YELLOW}⚠ Service '$service' not found${NC}"
    WARNINGS=$((WARNINGS + 1))
  fi
done

# 6. Check ingress
echo ""
echo -e "${YELLOW}6. Checking ingress...${NC}"

INGRESS=$(kubectl get ingress -n phase-5 --no-headers 2>/dev/null | wc -l)
if [ "$INGRESS" -gt 0 ]; then
  echo -e "${GREEN}✓ Found ${INGRESS} ingress resources${NC}"

  # Check TLS configuration
  TLS_INGRESS=$(kubectl get ingress -n phase-5 -o json | jq '.items[] | select(.spec.tls != null) | .metadata.name' | wc -l)
  if [ "$TLS_INGRESS" -gt 0 ]; then
    echo -e "${GREEN}✓ ${TLS_INGRESS} ingress resources have TLS configured${NC}"
  else
    echo -e "${YELLOW}⚠ No TLS configured on ingress${NC}"
    WARNINGS=$((WARNINGS + 1))
  fi
else
  echo -e "${YELLOW}⚠ No ingress resources found${NC}"
  WARNINGS=$((WARNINGS + 1))
fi

# 7. Check certificates
echo ""
echo -e "${YELLOW}7. Checking TLS certificates...${NC}"

if kubectl get certificates -n phase-5 > /dev/null 2>&1; then
  CERTS=$(kubectl get certificates -n phase-5 --no-headers | wc -l)
  echo -e "${GREEN}✓ Found ${CERTS} certificates${NC}"

  # Check certificate status
  READY_CERTS=$(kubectl get certificates -n phase-5 -o json | jq '.items[] | select(.status.conditions[].status == "True") | .metadata.name' | wc -l)
  if [ "$READY_CERTS" -eq "$CERTS" ]; then
    echo -e "${GREEN}✓ All certificates are ready${NC}"
  else
    echo -e "${YELLOW}⚠ Some certificates are not ready${NC}"
    kubectl get certificates -n phase-5
    WARNINGS=$((WARNINGS + 1))
  fi
else
  echo -e "${YELLOW}⚠ No certificates found (cert-manager may not be installed)${NC}"
  WARNINGS=$((WARNINGS + 1))
fi

# 8. Check HPA
echo ""
echo -e "${YELLOW}8. checking Horizontal Pod Autoscalers...${NC}"

if kubectl get hpa -n phase-5 > /dev/null 2>&1; then
  HPA_COUNT=$(kubectl get hpa -n phase-5 --no-headers | wc -l)
  echo -e "${GREEN}✓ Found ${HPA_COUNT} HPA resources${NC}"
  kubectl get hpa -n phase-5
else
  echo -e "${YELLOW}⚠ No HPA resources found${NC}"
  echo "  Run: kubectl apply -f k8s/autoscaler.yaml"
  WARNINGS=$((WARNINGS + 1))
fi

# 9. Check secrets
echo ""
echo -e "${YELLOW}9. Checking secrets...${NC}"

SECRETS=("db-credentials" "ollama-config")
for secret in "${SECRETS[@]}"; do
  if kubectl get secret "$secret" -n phase-5 > /dev/null 2>&1; then
    echo -e "${GREEN}✓ Secret '$secret' exists${NC}"
  else
    echo -e "${RED}✗ Secret '$secret' not found${NC}"
    echo "  Run: kubectl create secret generic $secret --from-literal=..."
    FAILURES=$((FAILURES + 1))
  fi
done

# 10. Check monitoring
echo ""
echo -e "${YELLOW}10. Checking monitoring stack...${NC}"

# Check Prometheus
if kubectl get svc prometheus-kube-prometheus-prometheus -n monitoring > /dev/null 2>&1; then
  echo -e "${GREEN}✓ Prometheus is running${NC}"
else
  echo -e "${YELLOW}⚠ Prometheus not found in monitoring namespace${NC}"
  WARNINGS=$((WARNINGS + 1))
fi

# Check Grafana
if kubectl get svc grafana -n monitoring > /dev/null 2>&1; then
  echo -e "${GREEN}✓ Grafana is running${NC}"
else
  echo -e "${YELLOW}⚠ Grafana not found in monitoring namespace${NC}"
  WARNINGS=$((WARNINGS + 1))
fi

# 11. Check Dapr
echo ""
echo -e "${YELLOW}11. Checking Dapr sidecars...${NC}"

DAPR_PODS=$(kubectl get pods -n phase-5 -o json | jq '.items[] | select(.spec.containers[].name == "daprd") | .metadata.name' | wc -l)
if [ "$DAPR_PODS" -gt 0 ]; then
  echo -e "${GREEN}✓ Dapr sidecars are injected (${DAPR_PODS} pods)${NC}"
else
  echo -e "${YELLOW}⚠ Dapr sidecars not found${NC}"
  WARNINGS=$((WARNINGS + 1))
fi

# 12. Run health check
echo ""
echo -e "${YELLOW}12. Running health check...${NC}"

# Port forward to backend
BACKEND_POD=$(kubectl get pod -n phase-5 -l app=backend -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)

if [ -n "$BACKEND_POD" ]; then
  echo "  Forwarding port to pod: ${BACKEND_POD}"

  # Start port forward in background
  kubectl port-forward -n phase-5 pod/$BACKEND_POD 8000:8000 > /dev/null 2>&1 &
  PF_PID=$!

  # Wait for port forward to be ready
  sleep 3

  # Run health check
  if curl -s http://localhost:8000/health | grep -q "healthy"; then
    echo -e "${GREEN}✓ Backend health check passed${NC}"
  else
    echo -e "${RED}✗ Backend health check failed${NC}"
    FAILURES=$((FAILURES + 1))
  fi

  # Kill port forward
  kill $PF_PID 2>/dev/null
else
  echo -e "${YELLOW}⚠ Could not find backend pod${NC}"
  WARNINGS=$((WARNINGS + 1))
fi

# Summary
echo ""
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Verification Summary${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""

if [ $FAILURES -eq 0 ] && [ $WARNINGS -eq 0 ]; then
  echo -e "${GREEN}✓ All checks passed! System is ready for production.${NC}"
  echo ""
  echo "Next steps:"
  echo "  1. Configure DNS records for your domain"
  echo "  2. Verify SSL certificates are issued"
  echo "  3. Run security scan: ./scripts/security-scan.sh"
  echo "  4. Run performance tests: ./scripts/performance-test.sh"
  echo "  5. Monitor Grafana dashboards"
  exit 0
elif [ $FAILURES -eq 0 ]; then
  echo -e "${YELLOW}⚠ System is operational with ${WARNINGS} warnings${NC}"
  echo "  Review warnings above and address if needed"
  exit 0
else
  echo -e "${RED}✗ Found ${FAILURES} failures and ${WARNINGS} warnings${NC}"
  echo "  Please address the failures before deploying to production"
  exit 1
fi