"""Aggregate daily usage counters of the SQuADDS Spaces (the planar-capacitance demo and the ML API). This file exists twice, byte for byte: deploy/geomgnn/src/geomgnn/usage.py and deploy/squadds_ml_api/squadds_ml_api/usage.py (build.py and the tests check it). Only counts leave memory: one small JSON row per UTC day per service (and per process start), written to a private dataset through huggingface_hub.CommitScheduler when the Space secret USAGE_STATS_TOKEN is set. Identities (Hugging Face usernames, client addresses) reach this module only as keyed hashes (keyed()), used to count distinct users and dropped at the end of the day. No usernames, hashes, addresses, file names, geometry or per-request rows are ever written.""" from __future__ import annotations import datetime as dt import hashlib import hmac import json import logging import os import secrets import statistics import tempfile import threading from pathlib import Path REPO = "SQuADDS/squadds-usage-stats" MAX_LATENCIES = 50_000 log = logging.getLogger(__name__) _SALT = (os.environ.get("USAGE_HASH_SALT") or "").encode() or secrets.token_bytes(32) def keyed(*parts) -> str: """Keyed hash of an identity (secret salt from USAGE_HASH_SALT, else random per process); kept in memory only.""" return hmac.new(_SALT, "\x1f".join(map(str, parts)).encode(), hashlib.sha256).hexdigest() def today() -> str: return dt.datetime.now(dt.timezone.utc).date().isoformat() def _now() -> str: return dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds") class Usage: """inc(group[, label]) counts, latency(s) collects response times, seen(group, key) counts distinct keyed hashes; row() is what gets persisted. scheduler: a CommitScheduler-like object (folder_path, lock); built from USAGE_STATS_TOKEN when None.""" def __init__(self, service: str, repo_id: str = REPO, every_min: float = 10, scheduler=None): self.service, self.started, self.lock = service, _now(), threading.Lock() self._reset(today()) self.scheduler = scheduler if scheduler is None: token = os.environ.get("USAGE_STATS_TOKEN") if not token: log.warning("usage stats: USAGE_STATS_TOKEN is not set, so the daily counts stay in memory (persistence off)") return try: from huggingface_hub import CommitScheduler folder = Path(tempfile.mkdtemp(prefix="usage_stats_")) / service self.scheduler = CommitScheduler(repo_id=repo_id, repo_type="dataset", folder_path=folder, path_in_repo=service, every=every_min, private=True, token=token) except Exception as e: # noqa: BLE001 log.warning("usage stats: persistence off (%s: %s), the daily counts stay in memory", type(e).__name__, e) def _reset(self, day: str) -> None: self.day, self.counts, self.latencies, self.distinct = day, {}, [], {} def _roll(self) -> None: d = today() if d != self.day: self._write() self._reset(d) def inc(self, group: str, label: str | None = None, n: int = 1) -> None: with self.lock: self._roll() if label is None: self.counts[group] = self.counts.get(group, 0) + n else: g = self.counts.setdefault(group, {}) g[label] = g.get(label, 0) + n self._write() def latency(self, seconds: float) -> None: with self.lock: self._roll() if len(self.latencies) < MAX_LATENCIES: self.latencies.append(float(seconds)) self._write() def seen(self, group: str, key: str | None) -> None: if not key: return with self.lock: self._roll() self.distinct.setdefault(group, set()).add(key) def row(self) -> dict: s = sorted(self.latencies) lat = {"n": len(s), "median": round(statistics.median(s), 3) if s else None, "p90": round(s[min(len(s) - 1, int(round(0.9 * (len(s) - 1))))], 3) if s else None} return {"service": self.service, "date": self.day, "process_started_utc": self.started, "updated_utc": _now(), **{k: (dict(sorted(v.items())) if isinstance(v, dict) else v) for k, v in sorted(self.counts.items())}, "latency_s": lat, **{k: len(v) for k, v in sorted(self.distinct.items())}} def _write(self) -> None: if self.scheduler is None: return f = Path(self.scheduler.folder_path) / f"{self.day}_{self.started.replace(':', '').replace('+0000', 'Z')}.json" with self.scheduler.lock: f.write_text(json.dumps(self.row(), indent=1) + "\n")