ac / google_dashboard.py
mendeld's picture
Upload 15 files
e46ad0b verified
Raw History Blame Contribute Delete
14.8 kB
"""Private Google Analytics 4 / Search Console reporting for AC Head Admin.
Never expose OAuth tokens, service-account JSON, or Google API payloads to browsers.
Only aggregate web/search performance data is returned. Server-side admin auth is
required on every route that calls this module.
"""
from __future__ import annotations
from datetime import date, datetime, timedelta
from functools import lru_cache
import json
import os
from threading import RLock
import time
from urllib.parse import quote
from zoneinfo import ZoneInfo
import requests
PROPERTY_ID = "559699110"
GSC_SITE = "sc-domain:5bcheder.com"
PERIODS = frozenset(("live", "last30", "last_hour", "last_day", "last_week", "all_time", "custom"))
SCOPES = (
"https://www.googleapis.com/auth/analytics.readonly",
"https://www.googleapis.com/auth/webmasters.readonly",
)
_CACHE: dict[tuple, tuple[float, dict]] = {}
_CACHE_LOCK = RLock()
class DashboardError(Exception):
def __init__(self, message: str, status_code: int = 503):
super().__init__(message)
self.status_code = status_code
def _timezone():
# Configure GOOGLE_ANALYTICS_TIMEZONE to match the GA property's reporting
# timezone if it is not US Eastern. Not a user/device location lookup.
name = os.getenv("GOOGLE_ANALYTICS_TIMEZONE", "America/New_York")
try:
return ZoneInfo(name)
except (ValueError, KeyError):
raise DashboardError("Invalid Google Analytics reporting timezone configuration.")
def _dates(period: str, custom_start: str | None, custom_end: str | None, *, gsc: bool) -> tuple[date, date, str]:
if period not in PERIODS:
raise DashboardError("Unknown reporting period.", 400)
today = datetime.now(_timezone()).date()
if gsc:
# Search Console complete/finalized daily data typically trails live
# traffic by several days. End on 3 days ago and request finalized data.
latest = today - timedelta(days=3)
if period in {"live", "last30", "last_hour"}:
return latest, latest, "Realtime search performance is not available in Google Search Console. Choose Last day, Last week, All time, or Custom."
if period == "last_day":
return latest, latest, "Latest generally finalized search day; data can still be delayed. Dates use Search Console's reporting timezone."
if period == "last_week":
return latest - timedelta(days=6), latest, "Seven generally finalized search days; Search Console data is delayed."
if period == "all_time":
return latest - timedelta(days=485), latest, "Search Console retains approximately 16 months, not the entire lifetime of the site."
else:
if period == "last_week":
return today - timedelta(days=6), today, "GA4 processed reporting may lag behind realtime traffic."
if period == "all_time":
return date(2020, 1, 1), today, "GA4 data is available only since this property's tracking began."
if period in {"last_hour", "last_day"}:
return today - timedelta(days=1), today, "GA4 processed minute-level data may not include the latest traffic."
if period in {"live", "last30"}:
return today, today, "GA4 realtime shows activity during the last 30 minutes, not a historical reporting total."
if period == "custom":
try:
start = date.fromisoformat(custom_start or "")
end = date.fromisoformat(custom_end or "")
except ValueError:
raise DashboardError("Choose both start and end dates for Custom.", 400)
if start > end or start < date(2020, 1, 1) or end > today:
raise DashboardError("Choose valid dates (2020 through today), with start on or before end.", 400)
if gsc and (end > today - timedelta(days=3) or (end - start).days > 485):
raise DashboardError("For Search Console choose an end date at least 3 days ago and a range no longer than 16 months.", 400)
if (end - start).days > 2500:
raise DashboardError("Custom date range is too long.", 400)
return start, end, "Custom date range (inclusive). Data may be delayed."
raise DashboardError("Unknown reporting period.", 400)
@lru_cache(maxsize=1)
def _credentials_info() -> dict:
raw = os.environ.get("GOOGLE_SERVICE_ACCOUNT_JSON", "").strip()
if not raw:
raise DashboardError("Google credentials are not configured. Add GOOGLE_SERVICE_ACCOUNT_JSON as a private Hugging Face Space Secret.")
try:
info = json.loads(raw)
if not isinstance(info, dict) or info.get("type") != "service_account" or not info.get("private_key"):
raise ValueError("invalid")
return info
except (ValueError, TypeError):
raise DashboardError("Google credentials Secret is not valid service-account JSON. Check the saved secret, not public files.")
@lru_cache(maxsize=1)
def _google_credentials():
try:
from google.oauth2 import service_account
return service_account.Credentials.from_service_account_info(_credentials_info(), scopes=SCOPES)
except (ImportError, ValueError, KeyError, TypeError) as exc:
raise DashboardError("Could not load Google service-account credentials. Check the JSON Secret and google-auth dependency.") from None
def _post(url: str, payload: dict) -> dict:
try:
from google.auth.transport.requests import Request
creds = _google_credentials()
with _CACHE_LOCK:
if not creds.valid or not creds.token:
creds.refresh(Request())
token = creds.token
response = requests.post(url, json=payload, headers={"Authorization": f"Bearer {token}"}, timeout=16)
if response.status_code in {401, 403}:
raise DashboardError("Google refused read access. Check the service account's Viewer access in Analytics, Restricted access in Search Console, and both enabled APIs.")
if response.status_code == 404:
raise DashboardError("Google property not found for the service account. Check the Analytics property ID or Search Console domain property.")
if response.status_code == 429:
raise DashboardError("Google reporting rate limit reached. Try again in a few minutes.")
if not response.ok:
raise DashboardError(f"Google reporting service returned HTTP {response.status_code}. Please try again later.")
data = response.json()
if not isinstance(data, dict):
raise DashboardError("Google returned an unexpected reporting response.")
return data
except DashboardError:
raise
except (requests.RequestException, ValueError, ImportError) as exc:
raise DashboardError("Could not reach Google reporting services. Try again later.") from None
except Exception as exc:
# Credential errors often embed sensitive details. Never send those raw.
raise DashboardError("Google authentication or reporting request failed. Check the private Secret and Google permissions.") from None
def _cached(key: tuple, ttl: int, load):
now = time.monotonic()
with _CACHE_LOCK:
hit = _CACHE.get(key)
if hit and now - hit[0] < ttl:
return hit[1]
result = load()
with _CACHE_LOCK:
if len(_CACHE) >= 60:
_CACHE.pop(next(iter(_CACHE)))
_CACHE[key] = (time.monotonic(), result)
return result
def _ga_call(body: dict, realtime: bool = False) -> dict:
method = "runRealtimeReport" if realtime else "runReport"
url = f"https://analyticsdata.googleapis.com/v1beta/properties/{PROPERTY_ID}:{method}"
return _post(url, body)
def _ga_metrics(result: dict, names: list[str]) -> dict:
row = (result.get("rows") or [{}])[0]
values = row.get("metricValues") or []
metrics = {}
for i, name in enumerate(names):
try:
metrics[name] = int(float(values[i].get("value", 0))) if i < len(values) else 0
except (TypeError, ValueError):
metrics[name] = 0
return metrics
def _ga_points(result: dict, metric: str, dim: str) -> list[dict]:
series = []
for item in result.get("rows", []):
try:
label = item["dimensionValues"][0]["value"]
val = max(0, int(float(item["metricValues"][0]["value"])))
except (IndexError, KeyError, ValueError, TypeError):
continue
if dim == "date" and len(label) == 8 and label.isdigit():
label = f"{label[:4]}-{label[4:6]}-{label[6:]}"
if dim == "dateHourMinute" and len(label) == 12 and label.isdigit():
label = f"{label[4:6]}/{label[6:8]} {label[8:10]}:{label[10:]}"
series.append({"label": label, "value": val})
return series
def analytics_report(period: str, start: str | None = None, end: str | None = None) -> dict:
start_d, end_d, note = _dates(period, start, end, gsc=False)
key = ("ga", period, str(start_d), str(end_d))
def load():
if period in {"live", "last30"}:
# Standard GA4 properties only have a realtime lookback of 30 min.
names = ["activeUsers", "screenPageViews", "eventCount"]
body = {"metrics": [{"name": name} for name in names],
"minuteRanges": [{"startMinutesAgo": 29, "endMinutesAgo": 0}]}
totals = _ga_metrics(_ga_call(body, realtime=True), names)
trend = _ga_call({"dimensions": [{"name": "minutesAgo"}],
"metrics": [{"name": "eventCount"}],
"minuteRanges": [{"startMinutesAgo": 29, "endMinutesAgo": 0}],
"limit": "30"}, realtime=True)
points = []
for row in trend.get("rows", []):
try:
mins = int(row["dimensionValues"][0]["value"])
count = int(float(row["metricValues"][0]["value"]))
if 0 <= mins <= 29:
points.append({"label": f"{mins}m ago", "value": count, "_m": mins})
except (KeyError, IndexError, ValueError, TypeError):
continue
points.sort(key=lambda x: -x["_m"])
for p in points: p.pop("_m", None)
return {"available": True, "source": "Google Analytics 4", "period": period,
"range": "Last 30 minutes (realtime)", "note": note,
"metrics": totals, "trend": points, "trend_metric": "Events", "pages": []}
names = ["activeUsers", "sessions", "screenPageViews", "eventCount"]
dates = {"startDate": str(start_d), "endDate": str(end_d)}
window = None
if period in {"last_hour", "last_day"}:
now = datetime.now(_timezone())
earliest = now - timedelta(hours=1 if period == "last_hour" else 24)
# dateHourMinute is numeric encoded YYYYMMDDHHMM in the GA API.
window = {"filter": {"fieldName": "dateHourMinute", "betweenFilter": {
"fromValue": {"int64Value": earliest.strftime("%Y%m%d%H%M")},
"toValue": {"int64Value": now.strftime("%Y%m%d%H%M")}}}}
base = {"dateRanges": [dates], "metrics": [{"name": name} for name in names]}
if window: base["dimensionFilter"] = window
totals = _ga_metrics(_ga_call(base), names)
dimension = "dateHourMinute" if window else "date"
series_body = {"dateRanges": [dates], "dimensions": [{"name": dimension}],
"metrics": [{"name": "screenPageViews"}], "limit": "2500",
"orderBys": [{"dimension": {"dimensionName": dimension}}]}
if window: series_body["dimensionFilter"] = window
trend = _ga_points(_ga_call(series_body), "screenPageViews", dimension)
page_body = {"dateRanges": [dates], "dimensions": [{"name": "pagePath"}],
"metrics": [{"name": "screenPageViews"}], "limit": "12",
"orderBys": [{"metric": {"metricName": "screenPageViews"}, "desc": True}]}
if window: page_body["dimensionFilter"] = window
pages = _ga_points(_ga_call(page_body), "screenPageViews", "pagePath")
return {"available": True, "source": "Google Analytics 4", "period": period,
"range": (f"{start_d} to {end_d}" if not window else ("Last 60 minutes (processed)" if period == "last_hour" else "Last 24 hours (processed)")),
"note": note, "metrics": totals, "trend": trend, "trend_metric": "Page views", "pages": pages}
return _cached(key, 50 if period in {"live", "last30"} else 300, load)
def _sc_call(start: date, end: date, dimensions: list[str]) -> dict:
body = {"startDate": str(start), "endDate": str(end), "type": "web", "dataState": "final",
"dimensions": dimensions, "rowLimit": 100 if dimensions else 1}
url = "https://www.googleapis.com/webmasters/v3/sites/" + quote(GSC_SITE, safe="") + "/searchAnalytics/query"
return _post(url, body)
def _sc_data(rows: list, dimension: str) -> list[dict]:
out = []
for row in rows:
try:
label = str(row["keys"][0])[:400]
out.append({"label": label, "clicks": float(row.get("clicks", 0)),
"impressions": float(row.get("impressions", 0)),
"ctr": float(row.get("ctr", 0)), "position": float(row.get("position", 0))})
except (IndexError, KeyError, ValueError, TypeError):
continue
if dimension == "date": out.sort(key=lambda x: x["label"])
return out
def search_console_report(period: str, start: str | None = None, end: str | None = None) -> dict:
start_d, end_d, note = _dates(period, start, end, gsc=True)
if period in {"live", "last30", "last_hour"}:
return {"available": False, "source": "Google Search Console", "period": period,
"note": note, "range": "Unavailable in realtime"}
key = ("gsc", period, str(start_d), str(end_d))
def load():
totals = _sc_call(start_d, end_d, [])
row = (totals.get("rows") or [{}])[0]
metrics = {"clicks": float(row.get("clicks", 0)), "impressions": float(row.get("impressions", 0)),
"ctr": float(row.get("ctr", 0)), "position": float(row.get("position", 0))}
tables = {}
for dimension in ("date", "query", "page", "country", "device"):
tables[dimension] = _sc_data(_sc_call(start_d, end_d, [dimension]).get("rows", []), dimension)
return {"available": True, "source": "Google Search Console", "period": period,
"range": f"{start_d} to {end_d}", "note": note,
"metrics": metrics, "tables": tables}
return _cached(key, 900, load)