File size: 4,821 Bytes
8867798 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 | """Aggregate daily usage counters of the SQuADDS Spaces (the planar-capacitance demo and the ML API). This file exists twice, byte
for byte: deploy/geomgnn/src/geomgnn/usage.py and deploy/squadds_ml_api/squadds_ml_api/usage.py (build.py and the tests check it).
Only counts leave memory: one small JSON row per UTC day per service (and per process start), written to a private dataset through
huggingface_hub.CommitScheduler when the Space secret USAGE_STATS_TOKEN is set. Identities (Hugging Face usernames, client
addresses) reach this module only as keyed hashes (keyed()), used to count distinct users and dropped at the end of the day. No
usernames, hashes, addresses, file names, geometry or per-request rows are ever written."""
from __future__ import annotations
import datetime as dt
import hashlib
import hmac
import json
import logging
import os
import secrets
import statistics
import tempfile
import threading
from pathlib import Path
REPO = "SQuADDS/squadds-usage-stats"
MAX_LATENCIES = 50_000
log = logging.getLogger(__name__)
_SALT = (os.environ.get("USAGE_HASH_SALT") or "").encode() or secrets.token_bytes(32)
def keyed(*parts) -> str:
"""Keyed hash of an identity (secret salt from USAGE_HASH_SALT, else random per process); kept in memory only."""
return hmac.new(_SALT, "\x1f".join(map(str, parts)).encode(), hashlib.sha256).hexdigest()
def today() -> str:
return dt.datetime.now(dt.timezone.utc).date().isoformat()
def _now() -> str:
return dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds")
class Usage:
"""inc(group[, label]) counts, latency(s) collects response times, seen(group, key) counts distinct keyed hashes; row() is
what gets persisted. scheduler: a CommitScheduler-like object (folder_path, lock); built from USAGE_STATS_TOKEN when None."""
def __init__(self, service: str, repo_id: str = REPO, every_min: float = 10, scheduler=None):
self.service, self.started, self.lock = service, _now(), threading.Lock()
self._reset(today())
self.scheduler = scheduler
if scheduler is None:
token = os.environ.get("USAGE_STATS_TOKEN")
if not token:
log.warning("usage stats: USAGE_STATS_TOKEN is not set, so the daily counts stay in memory (persistence off)")
return
try:
from huggingface_hub import CommitScheduler
folder = Path(tempfile.mkdtemp(prefix="usage_stats_")) / service
self.scheduler = CommitScheduler(repo_id=repo_id, repo_type="dataset", folder_path=folder, path_in_repo=service,
every=every_min, private=True, token=token)
except Exception as e: # noqa: BLE001
log.warning("usage stats: persistence off (%s: %s), the daily counts stay in memory", type(e).__name__, e)
def _reset(self, day: str) -> None:
self.day, self.counts, self.latencies, self.distinct = day, {}, [], {}
def _roll(self) -> None:
d = today()
if d != self.day:
self._write()
self._reset(d)
def inc(self, group: str, label: str | None = None, n: int = 1) -> None:
with self.lock:
self._roll()
if label is None:
self.counts[group] = self.counts.get(group, 0) + n
else:
g = self.counts.setdefault(group, {})
g[label] = g.get(label, 0) + n
self._write()
def latency(self, seconds: float) -> None:
with self.lock:
self._roll()
if len(self.latencies) < MAX_LATENCIES:
self.latencies.append(float(seconds))
self._write()
def seen(self, group: str, key: str | None) -> None:
if not key:
return
with self.lock:
self._roll()
self.distinct.setdefault(group, set()).add(key)
def row(self) -> dict:
s = sorted(self.latencies)
lat = {"n": len(s), "median": round(statistics.median(s), 3) if s else None,
"p90": round(s[min(len(s) - 1, int(round(0.9 * (len(s) - 1))))], 3) if s else None}
return {"service": self.service, "date": self.day, "process_started_utc": self.started, "updated_utc": _now(),
**{k: (dict(sorted(v.items())) if isinstance(v, dict) else v) for k, v in sorted(self.counts.items())},
"latency_s": lat, **{k: len(v) for k, v in sorted(self.distinct.items())}}
def _write(self) -> None:
if self.scheduler is None:
return
f = Path(self.scheduler.folder_path) / f"{self.day}_{self.started.replace(':', '').replace('+0000', 'Z')}.json"
with self.scheduler.lock:
f.write_text(json.dumps(self.row(), indent=1) + "\n")
|