Fix monitoring stack configuration for HF Spaces: Add Alertmanager, Pushgateway, and fix Prometheus/Grafana connection

Dockerfile

@@ -40,9 +40,26 @@ RUN wget https://github.com/prometheus/prometheus/releases/download/v2.45.0/prom
     mkdir -p /etc/prometheus /var/lib/prometheus && \
     rm -rf prometheus-*
 
-COPY monitoring/grafana/provisioning /etc/grafana/provisioning
-COPY monitoring/grafana/dashboards /var/lib/grafana/dashboards
-COPY monitoring/prometheus/prometheus.yml /etc/prometheus/prometheus.yml
+# Alertmanager
+RUN wget https://github.com/prometheus/alertmanager/releases/download/v0.25.0/alertmanager-0.25.0.linux-amd64.tar.gz && \
+    tar xvfz alertmanager-*.tar.gz && \
+    mv alertmanager-*/alertmanager /usr/local/bin/ && \
+    mv alertmanager-*/amtool /usr/local/bin/ && \
+    mkdir -p /etc/alertmanager /var/lib/alertmanager && \
+    rm -rf alertmanager-*
+
+# Pushgateway
+RUN wget https://github.com/prometheus/pushgateway/releases/download/v1.6.0/pushgateway-1.6.0.linux-amd64.tar.gz && \
+    tar xvfz pushgateway-*.tar.gz && \
+    mv pushgateway-*/pushgateway /usr/local/bin/ && \
+    rm -rf pushgateway-*
+
+COPY --chown=user monitoring/grafana/provisioning /etc/grafana/provisioning
+COPY --chown=user monitoring/grafana/dashboards /var/lib/grafana/dashboards
+COPY --chown=user monitoring/prometheus/prometheus.yml /etc/prometheus/prometheus.yml
+COPY --chown=user monitoring/prometheus/alert_rules.yml /etc/prometheus/alert_rules.yml
+COPY --chown=user monitoring/alertmanager/config.yml /etc/alertmanager/config.yml
+
 
 # Copy requirements first for caching
 COPY requirements.txt .

docker/nginx.conf

@@ -33,6 +33,14 @@ http {
         server 127.0.0.1:9090;
     }
 
+    upstream alertmanager {
+        server 127.0.0.1:9093;
+    }
+
+    upstream pushgateway {
+        server 127.0.0.1:9091;
+    }
+
     server {
         listen 7860;
         server_name localhost;
@@ -108,6 +116,24 @@ http {
             proxy_set_header X-Forwarded-Proto $scheme;
         }
 
+        # Alertmanager UI
+        location /alertmanager/ {
+            proxy_pass http://alertmanager;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+        }
+
+        # Pushgateway UI
+        location /pushgateway/ {
+            proxy_pass http://pushgateway;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+        }
+
         # Streamlit (Catch-all)
         location / {
             proxy_pass http://streamlit;

docker/scripts/start_space.sh

@@ -53,23 +53,37 @@ for i in {1..30}; do
 done
 
 echo "$(date) - Configuring and starting Prometheus..."
-# Create a config for the space
-cat <<EOF > /tmp/prometheus.yml
-global:
-  scrape_interval: 15s
-  evaluation_interval: 15s
-
-scrape_configs:
-  - job_name: 'hopcroft-api'
-    metrics_path: '/metrics'
-    static_configs:
-      - targets: ['127.0.0.1:8000']
-    scrape_interval: 10s
-
-  - job_name: 'prometheus'
-    static_configs:
-      - targets: ['127.0.0.1:9090']
-EOF
+# Patch Grafana Datasource for Localhost (HF Space)
+# Replace prometheus:9090 with 127.0.0.1:9090 in all datasource configs
+find /etc/grafana/provisioning/datasources -name '*.yml' -exec sed -i 's/prometheus:9090/127.0.0.1:9090/g' {} +
+
+# Copy production configs to /tmp for modification
+cp /etc/prometheus/prometheus.yml /tmp/prometheus.yml
+cp /etc/prometheus/alert_rules.yml /tmp/alert_rules.yml
+cp /etc/alertmanager/config.yml /tmp/alertmanager.yml
+
+# Modify Prometheus config for local execution (replace docker-compose service names with localhost)
+# hopcroft-api:8080 -> 127.0.0.1:8000 (API runs on 8000 in Space)
+sed -i 's/hopcroft-api:8080/127.0.0.1:8000/g' /tmp/prometheus.yml
+# alertmanager:9093 -> 127.0.0.1:9093
+sed -i 's/alertmanager:9093/127.0.0.1:9093/g' /tmp/prometheus.yml
+# pushgateway:9091 -> 127.0.0.1:9091
+sed -i 's/pushgateway:9091/127.0.0.1:9091/g' /tmp/prometheus.yml
+# Fix alert_rules path to be absolute or relative to execution
+sed -i 's|"alert_rules.yml"|"/tmp/alert_rules.yml"|g' /tmp/prometheus.yml
+
+echo "$(date) - Starting Alertmanager..."
+alertmanager \
+    --config.file=/tmp/alertmanager.yml \
+    --storage.path=/tmp/alertmanager_data \
+    --web.route-prefix=/alertmanager/ \
+    >> /tmp/alertmanager.log 2>&1 &
+
+echo "$(date) - Starting Pushgateway..."
+pushgateway \
+    --persistence.file=/tmp/pushgateway_data \
+    --web.route-prefix=/pushgateway/ \
+    >> /tmp/pushgateway.log 2>&1 &
 
 # Determine Prometheus External URL
 # Always use relative path so it works on both huggingface.co and .hf.space domains