Deploy MoBis: live planner API + web app + dataset

hf_space/grafana/dashboards/mobaz.json

@@ -4,69 +4,92 @@
   "tags": ["mobaz"],
   "timezone": "browser",
   "schemaVersion": 39,
-  "version": 1,
+  "version": 2,
   "editable": true,
   "refresh": "30s",
   "time": { "from": "now-6h", "to": "now" },
   "annotations": { "list": [] },
   "templating": { "list": [] },
   "panels": [
+    { "id": 100, "type": "row", "title": "Overview", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "collapsed": false },
     {
-      "id": 1,
-      "type": "stat",
-      "title": "Uptime",
+      "id": 1, "type": "stat", "title": "Uptime",
       "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
       "fieldConfig": { "defaults": { "unit": "s", "color": { "mode": "fixed", "fixedColor": "green" } }, "overrides": [] },
       "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none" },
       "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "mobaz_uptime_seconds" } ]
     },
     {
-      "id": 2,
-      "type": "stat",
-      "title": "Requests / sec (5m)",
+      "id": 2, "type": "stat", "title": "Requests / sec (5m)",
       "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
       "fieldConfig": { "defaults": { "unit": "reqps", "decimals": 2, "color": { "mode": "fixed", "fixedColor": "blue" } }, "overrides": [] },
       "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area" },
       "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(mobaz_http_requests_total[5m]))" } ]
     },
     {
-      "id": 3,
-      "type": "stat",
-      "title": "p95 latency (5m)",
+      "id": 3, "type": "stat", "title": "p95 latency (5m)",
       "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
+      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
       "fieldConfig": { "defaults": { "unit": "s", "decimals": 3, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 3 } ] } }, "overrides": [] },
       "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none" },
       "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "histogram_quantile(0.95, sum(rate(mobaz_http_request_duration_seconds_bucket[5m])) by (le))" } ]
     },
     {
-      "id": 4,
-      "type": "stat",
-      "title": "Client errors (total)",
+      "id": 9, "type": "stat", "title": "Active clients (5m)",
       "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
+      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
+      "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "purple" } }, "overrides": [] },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area" },
+      "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "mobaz_active_clients" } ]
+    },
+    {
+      "id": 10, "type": "stat", "title": "Avg response time (5m)",
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 5 },
+      "fieldConfig": { "defaults": { "unit": "s", "decimals": 3, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.5 }, { "color": "red", "value": 2 } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "none" },
+      "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(mobaz_http_request_duration_seconds_sum[5m])) / sum(rate(mobaz_http_request_duration_seconds_count[5m]))" } ]
+    },
+    {
+      "id": 11, "type": "stat", "title": "In-flight requests",
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 5 },
+      "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "fixed", "fixedColor": "blue" } }, "overrides": [] },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "value", "graphMode": "area" },
+      "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "mobaz_inflight_requests" } ]
+    },
+    {
+      "id": 12, "type": "stat", "title": "Error rate (5xx, 5m)",
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 5 },
+      "fieldConfig": { "defaults": { "unit": "percent", "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 5 } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none" },
+      "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "100 * sum(rate(mobaz_http_requests_total{code=~\"5..\"}[5m])) / clamp_min(sum(rate(mobaz_http_requests_total[5m])), 0.0001)" } ]
+    },
+    {
+      "id": 4, "type": "stat", "title": "Client errors (total)",
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 5 },
       "fieldConfig": { "defaults": { "unit": "short", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] },
       "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none" },
       "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "mobaz_client_errors_total" } ]
     },
+
+    { "id": 101, "type": "row", "title": "Traffic & latency", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, "collapsed": false },
     {
-      "id": 5,
-      "type": "timeseries",
-      "title": "Request rate by route",
+      "id": 5, "type": "timeseries", "title": "Request rate by route",
       "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 4 },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 },
       "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "fillOpacity": 15, "stacking": { "mode": "normal" } } }, "overrides": [] },
       "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] }, "tooltip": { "mode": "multi" } },
       "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (route) (rate(mobaz_http_requests_total[5m]))", "legendFormat": "{{route}}" } ]
     },
     {
-      "id": 6,
-      "type": "timeseries",
-      "title": "Latency quantiles",
+      "id": 6, "type": "timeseries", "title": "Latency quantiles",
       "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 4 },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 },
       "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "fillOpacity": 10 } }, "overrides": [] },
       "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
       "targets": [
@@ -76,21 +99,81 @@
       ]
     },
     {
-      "id": 7,
-      "type": "timeseries",
-      "title": "Responses by status code",
+      "id": 7, "type": "timeseries", "title": "Responses by status code",
       "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "gridPos": { "h": 9, "w": 12, "x": 0, "y": 13 },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 },
       "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "fillOpacity": 15, "stacking": { "mode": "normal" } } }, "overrides": [] },
       "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
       "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum by (code) (rate(mobaz_http_requests_total[5m]))", "legendFormat": "{{code}}" } ]
     },
     {
-      "id": 8,
-      "type": "timeseries",
-      "title": "Cache & data gauges",
+      "id": 13, "type": "timeseries", "title": "Avg response time & in-flight",
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 },
+      "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "fillOpacity": 10 } }, "overrides": [ { "matcher": { "id": "byName", "options": "in-flight" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "custom.axisPlacement", "value": "right" } ] } ] },
+      "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
+      "targets": [
+        { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "sum(rate(mobaz_http_request_duration_seconds_sum[5m])) / sum(rate(mobaz_http_request_duration_seconds_count[5m]))", "legendFormat": "avg response" },
+        { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "mobaz_inflight_requests", "legendFormat": "in-flight" }
+      ]
+    },
+
+    { "id": 102, "type": "row", "title": "System (CPU / RAM / host)", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 }, "collapsed": false },
+    {
+      "id": 14, "type": "timeseries", "title": "CPU usage",
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 27 },
+      "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "custom": { "drawStyle": "line", "fillOpacity": 15 } }, "overrides": [] },
+      "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
+      "targets": [
+        { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "mobaz_process_cpu_percent", "legendFormat": "process" },
+        { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "mobaz_system_cpu_percent", "legendFormat": "system" }
+      ]
+    },
+    {
+      "id": 15, "type": "timeseries", "title": "Memory",
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 27 },
+      "fieldConfig": { "defaults": { "unit": "bytes", "custom": { "drawStyle": "line", "fillOpacity": 15 } }, "overrides": [ { "matcher": { "id": "byName", "options": "system used %" }, "properties": [ { "id": "unit", "value": "percent" }, { "id": "custom.axisPlacement", "value": "right" }, { "id": "max", "value": 100 } ] } ] },
+      "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
+      "targets": [
+        { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "mobaz_process_resident_memory_bytes", "legendFormat": "process RSS" },
+        { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "mobaz_system_memory_used_percent", "legendFormat": "system used %" }
+      ]
+    },
+    {
+      "id": 16, "type": "timeseries", "title": "Threads & open file descriptors",
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 35 },
+      "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "fillOpacity": 10 } }, "overrides": [] },
+      "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
+      "targets": [
+        { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "mobaz_process_threads", "legendFormat": "threads" },
+        { "refId": "B", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "mobaz_process_open_fds", "legendFormat": "open fds" }
+      ]
+    },
+    {
+      "id": 17, "type": "timeseries", "title": "System load (1m)",
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": { "h": 8, "w": 8, "x": 8, "y": 35 },
+      "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "fillOpacity": 15 } }, "overrides": [] },
+      "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } },
+      "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "mobaz_system_load1", "legendFormat": "load1" } ]
+    },
+    {
+      "id": 18, "type": "timeseries", "title": "Active clients over time",
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 35 },
+      "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "fillOpacity": 20 } }, "overrides": [] },
+      "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } },
+      "targets": [ { "refId": "A", "datasource": { "type": "prometheus", "uid": "prometheus" }, "expr": "mobaz_active_clients", "legendFormat": "active clients" } ]
+    },
+
+    { "id": 103, "type": "row", "title": "Data & caches", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 }, "collapsed": false },
+    {
+      "id": 8, "type": "timeseries", "title": "Cache & data gauges",
       "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "gridPos": { "h": 9, "w": 12, "x": 12, "y": 13 },
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 44 },
       "fieldConfig": { "defaults": { "unit": "short", "custom": { "drawStyle": "line", "fillOpacity": 5 } }, "overrides": [] },
       "options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["lastNotNull"] }, "tooltip": { "mode": "multi" } },
       "targets": [

requirements.txt

@@ -1,2 +1,3 @@
 requests>=2.25
 beautifulsoup4>=4.9
+psutil>=5.9

server.py

@@ -39,6 +39,15 @@ import geocode
 import nearby
 import roads
 
+try:
+    import psutil  # process + system CPU/RAM metrics for /metrics
+    _PROC = psutil.Process()
+    _PROC.cpu_percent(interval=None)  # prime the per-process CPU delta
+    psutil.cpu_percent(interval=None)  # prime the system CPU delta
+except Exception:  # noqa: BLE001 — metrics are best-effort; never block startup
+    psutil = None
+    _PROC = None
+
 ROOT = os.path.dirname(os.path.abspath(__file__))
 PUBLIC = os.path.join(ROOT, "public")
 PORT = int(os.environ.get("PORT", "8000"))
@@ -94,6 +103,8 @@ class _Metrics:
         self.duration_count = 0
         self.client_errors = 0
         self.beacons_dropped = 0      # rate-limited client beacons
+        self.inflight = 0             # requests currently being served
+        self.clients = []             # sliding window of (ts, client_ip)
 
     def observe(self, route, method, code, dur):
         with self._lock:
@@ -112,6 +123,54 @@ class _Metrics:
             else:
                 self.client_errors += 1
 
+    def inflight_inc(self):
+        with self._lock:
+            self.inflight += 1
+
+    def inflight_dec(self):
+        with self._lock:
+            self.inflight = max(0, self.inflight - 1)
+
+    def record_client(self, ip):
+        now = time.time()
+        with self._lock:
+            self.clients.append((now, ip))
+            if len(self.clients) > 5000:  # hard cap so a flood can't grow it unbounded
+                self.clients = self.clients[-5000:]
+
+    def active_clients(self, window=300):
+        """Unique client IPs seen in the last `window` seconds (a stateless API
+        has no real 'connections', so this is the meaningful 'active clients')."""
+        cutoff = time.time() - window
+        with self._lock:
+            self.clients = [(t, ip) for (t, ip) in self.clients if t >= cutoff]
+            return len({ip for _, ip in self.clients})
+
+    def _system_metrics(self):
+        """Process + host CPU/RAM via psutil (best-effort; empty if unavailable)."""
+        out = {}
+        if psutil is None or _PROC is None:
+            return out
+        try:
+            out["mobaz_process_cpu_percent"] = float(_PROC.cpu_percent(interval=None))
+            out["mobaz_process_resident_memory_bytes"] = int(_PROC.memory_info().rss)
+            out["mobaz_process_threads"] = int(_PROC.num_threads())
+            try:
+                out["mobaz_process_open_fds"] = int(_PROC.num_fds())
+            except Exception:  # noqa: BLE001 — not on every platform
+                pass
+            out["mobaz_system_cpu_percent"] = float(psutil.cpu_percent(interval=None))
+            vm = psutil.virtual_memory()
+            out["mobaz_system_memory_used_percent"] = float(vm.percent)
+            out["mobaz_system_memory_total_bytes"] = int(vm.total)
+            try:
+                out["mobaz_system_load1"] = float(os.getloadavg()[0])
+            except Exception:  # noqa: BLE001 — no getloadavg on some platforms
+                pass
+        except Exception:  # noqa: BLE001
+            pass
+        return out
+
     def _gauges(self):
         """Best-effort point-in-time gauges (never raise into /metrics)."""
         out = {}
@@ -141,6 +200,7 @@ class _Metrics:
             buckets = list(self.bucket_counts)
             dur_sum, dur_count = self.duration_sum, self.duration_count
             client_errors, dropped = self.client_errors, self.beacons_dropped
+            inflight = self.inflight
             uptime = time.time() - self.start
         L = []
         L.append("# HELP mobaz_up Whether the MoBaz backend is serving (always 1).")
@@ -169,9 +229,19 @@ class _Metrics:
         L.append("# HELP mobaz_client_error_beacons_dropped_total Beacons dropped by the rate limit.")
         L.append("# TYPE mobaz_client_error_beacons_dropped_total counter")
         L.append("mobaz_client_error_beacons_dropped_total %d" % dropped)
-        for name, val in self._gauges().items():
+        L.append("# HELP mobaz_inflight_requests Requests currently being served.")
+        L.append("# TYPE mobaz_inflight_requests gauge")
+        L.append("mobaz_inflight_requests %d" % inflight)
+        L.append("# HELP mobaz_active_clients Unique client IPs seen in the last 5 minutes.")
+        L.append("# TYPE mobaz_active_clients gauge")
+        L.append("mobaz_active_clients %d" % self.active_clients())
+        # Data/cache gauges + process & host CPU/RAM (psutil).
+        allg = {}
+        allg.update(self._gauges())
+        allg.update(self._system_metrics())
+        for name, val in allg.items():
             L.append("# TYPE %s gauge" % name)
-            L.append("%s %d" % (name, val))
+            L.append(("%s %d" if isinstance(val, int) else "%s %.4f") % (name, val))
         return "\n".join(L) + "\n"
 
 
@@ -189,6 +259,16 @@ def _status_html():
     gauges = METRICS._gauges()
     total = sum(reqs.values())
     avg_ms = (dur_sum / dur_count * 1000) if dur_count else 0.0
+    active = METRICS.active_clients()
+    inflight = METRICS.inflight
+    # Memory only (reading RSS is side-effect-free; CPU% would reset the delta
+    # that the Prometheus scrape relies on, so leave CPU to Grafana).
+    rss_mb = 0
+    if _PROC is not None:
+        try:
+            rss_mb = _PROC.memory_info().rss / (1024 * 1024)
+        except Exception:  # noqa: BLE001
+            rss_mb = 0
     # Requests grouped by route (sum across method/code).
     by_route = {}
     for (route, _m, _c), n in reqs.items():
@@ -203,6 +283,9 @@ def _status_html():
             ("uptime", "%dh %dm" % (uptime // 3600, (uptime % 3600) // 60)),
             ("requests", total),
             ("avg latency", "%.1f ms" % avg_ms),
+            ("active clients (5m)", active),
+            ("in-flight", inflight),
+            ("memory", "%.0f MB" % rss_mb),
             ("stops", gauges.get("mobaz_stops", 0)),
             ("events cached", gauges.get("mobaz_events_cached", 0)),
             ("geocode cache", gauges.get("mobaz_geocode_cache_entries", 0)),
@@ -350,6 +433,17 @@ class Handler(BaseHTTPRequestHandler):
             if t0 is not None:
                 route = getattr(self, "_route", _route_label(urlparse(self.path).path))
                 METRICS.observe(route, self.command or "GET", code, time.perf_counter() - t0)
+                METRICS.inflight_dec()
+
+    def _begin(self, path):
+        """Stamp route/timer, count the request as in-flight, and note the client
+        IP (behind nginx the real IP is in X-Forwarded-For)."""
+        self._t0 = time.perf_counter()
+        self._route = _route_label(path)
+        METRICS.inflight_inc()
+        xff = self.headers.get("X-Forwarded-For")
+        ip = xff.split(",")[0].strip() if xff else (self.client_address[0] if self.client_address else "?")
+        METRICS.record_client(ip)
 
     def _cors(self):
         # The packaged app calls this API from a file:// origin, so allow any.
@@ -422,8 +516,7 @@ class Handler(BaseHTTPRequestHandler):
     def do_GET(self):
         parsed = urlparse(self.path)
         path = parsed.path
-        self._t0 = time.perf_counter()
-        self._route = _route_label(path)
+        self._begin(path)
         try:
             if path == "/metrics":
                 return self._send_text(METRICS.render(), "text/plain; version=0.0.4")
@@ -540,8 +633,7 @@ class Handler(BaseHTTPRequestHandler):
 
     def do_POST(self):
         parsed = urlparse(self.path)
-        self._t0 = time.perf_counter()
-        self._route = _route_label(parsed.path)
+        self._begin(parsed.path)
         if parsed.path != "/api/clienterror":
             return self._send_json({"error": "not found"}, 404)
         # Read at most a few KB; oversized or malformed bodies are dropped quietly.