"""
NEXRAD Level 2 — 0.5° sweep browser (SAILS-aware)
Hugging Face Space (Gradio)
Pulls one hour of archived NEXRAD Level 2 volumes from the AWS Open Data
buckets, extracts every ~0.5° sweep (including SAILS / MESO-SAILS
re-insertions), and displays them gate-natively: each sweep's polar grid is
shipped to the browser as a lossless grayscale PNG and reprojected per-pixel
in a WebGL fragment shader (4/3-earth beam model), so the native gate
geometry is preserved at any zoom level.
"""
import base64
import bz2
import collections
import datetime as dt
import hashlib
import time as time_mod
import gzip
import html as html_mod
import io
import json
import os
import re
import shutil
import tempfile
import threading
import zipfile
try:
import shapefile as _pyshp # pyshp, for live warning polygons
except Exception: # pragma: no cover
_pyshp = None
import traceback
import urllib.request
import xml.etree.ElementTree as ET
from concurrent.futures import (ProcessPoolExecutor, ThreadPoolExecutor,
as_completed)
import matplotlib
matplotlib.use("Agg")
import cmweather # noqa: F401 (registers ChaseSpectral & friends)
import cmasher # noqa: F401 (registers cmr.* colormaps, e.g. cmr.fusion_r)
import gradio as gr
import matplotlib.pyplot as plt
import numpy as np
import xradar as xd
from PIL import Image
# Velocity dealiasing: the Rust region_dealias (a port of Py-ART's region-based
# algorithm), used via its low-level sweep_folds() so no Py-ART is needed.
try:
import region_dealias as _region_dealias # noqa: F401
_HAVE_REGION_DEALIAS = True
except Exception:
_HAVE_REGION_DEALIAS = False
_DEALIAS_ENGINE = "Rust region-dealias" if _HAVE_REGION_DEALIAS else "(none)"
print(f"[region-dealias] velocity dealias engine = {_DEALIAS_ENGINE}"
+ (f" v{_region_dealias.__version__}" if _HAVE_REGION_DEALIAS else ""),
flush=True)
from xradar.io.backends import nexrad_level2 as _nx2
# --- patch: the last LDM record's size field is negative (signed) by spec;
# xradar reads it as unsigned -> MemoryError on most modern volumes.
_orig_init_record = _nx2.NEXRADLevel2File.init_record
def _init_record_fixed(self, recnum):
try:
return _orig_init_record(self, recnum)
except (MemoryError, OverflowError):
ldm = 0 if recnum < 134 else ((recnum - 134) // 120) + 1
if ldm >= len(self.bz2_record_indices):
return False
start = self.bz2_record_indices[ldm]
size = abs(int(self._fh[start:start + 4].view(">i4")[0]))
if self._fp is not None:
self._fp.seek(start + 4)
compressed = self._fp.read(size)
else:
compressed = self._fh[start + 4:start + 4 + size].tobytes()
dec = bz2.BZ2Decompressor()
self._ldm[ldm] = np.frombuffer(dec.decompress(compressed),
dtype=np.uint8)
return _orig_init_record(self, recnum)
_nx2.NEXRADLevel2File.init_record = _init_record_fixed
# --- patch 2: legacy Message 1 declares range-to-first-gate as signed
# halfwords (they're negative for split cuts, e.g. -375 m), but xradar reads
# them unsigned -> velocity sweeps land ~65 km downrange in pre-2008 data.
_nx2.MSG_1["sur_range_first"] = _nx2.SINT2
_nx2.MSG_1["doppler_range_first"] = _nx2.SINT2
# --- link previews: Gradio's `head=` is injected client-side, so social
# scrapers only see the static defaults baked into its index.html template.
# Patch the template on disk at startup.
APP_TITLE = "NEXRAD level 2 browser"
APP_DESC = ("Gate-native WebGL browsing of archived WSR-88D 0.5° scans "
"(SAILS-aware) from the AWS NEXRAD Level 2 archive.")
THUMB_URL = ("https://huggingface.co/spaces/snesbitt/nexrad-level2-browser/"
"resolve/main/thumbnail.png")
def _patch_og_template():
try:
import gradio as _g
p = os.path.join(os.path.dirname(_g.__file__),
"templates", "frontend", "index.html")
html = open(p).read()
html = html.replace('content="Gradio"', f'content="{APP_TITLE}"')
html = html.replace("Click to try out the app!", APP_DESC)
html = html.replace(
"https://raw.githubusercontent.com/gradio-app/gradio/main/js/"
"_website/src/lib/assets/img/header-image.jpg", THUMB_URL)
open(p, "w").write(html)
except Exception:
pass # non-fatal: previews fall back to Gradio defaults
_patch_og_template()
from sites import SITES # (icao, city, state) for all WSR-88D sites
from nexrad_sites import NEXRAD_COORDS # (lat, lon) per site — replaces pyart
SITE_CHOICES = [(f"{icao} — {city.lower()}, {st.lower()}", icao)
for icao, city, st in SITES]
# ----------------------------------------------------------------------------- config
BUCKETS = [
"https://noaa-nexrad-level2.s3.amazonaws.com",
"https://unidata-nexrad-level2.s3.amazonaws.com",
]
FIELDS = {
"Reflectivity": dict(
pyart="reflectivity", cmap="ChaseSpectral", vmin=-30, vmax=80,
units="dBZ", label="Horizontal reflectivity factor", tick=10,
),
"Radial velocity": dict(
pyart="velocity", cmap="cmr.fusion_r", vmin=-40, vmax=40,
units="m/s", label="Radial velocity", tick=10,
),
"Differential reflectivity": dict(
pyart="differential_reflectivity", cmap="HomeyerRainbow",
vmin=-2, vmax=8, units="dB", label="Differential reflectivity", tick=2,
),
"Correlation coefficient": dict(
pyart="cross_correlation_ratio", cmap="RefDiff",
vmin=0.5, vmax=1.05, units="", label="Correlation coefficient", tick=0.1,
),
}
DEALIAS_NAME = "Radial velocity (dealiased)"
DEALIAS_CFG = dict(
pyart="velocity", cmap="cmr.fusion_r", vmin=-64, vmax=64,
units="m/s", label="Dealiased radial velocity", tick=16,
)
def _field_cfg(fn):
return DEALIAS_CFG if fn == DEALIAS_NAME else FIELDS[fn]
ELEV_MAX = 0.75 # deg — treat sweeps below this as the 0.5° split cut
LOW_SWEEP_TOL = 0.15 # deg — show only the lowest cut(s); a site's 0.3°
# supplemental cut then hides its 0.5° cut (e.g. KBUF)
MAX_FRAMES = 60 # safety cap on rendered sweeps
N_PROC = max(1, min(2, os.cpu_count() or 1)) # decode workers (CPU-bound)
# persistent raw-volume cache (survives across requests; LRU-pruned)
VOL_CACHE_DIR = os.path.join(tempfile.gettempdir(), "nexrad_vol_cache")
os.makedirs(VOL_CACHE_DIR, exist_ok=True)
VOL_CACHE_BYTES = 2 << 30 # 2 GiB
VOL_CACHE_MAX_AGE_S = 24 * 3600 # also evict raw volumes older than 24 h
# ---- live Level 2 chunk feed (Phase 1) --------------------------------------
# Same objects the NewNEXRADLevel2ObjectFilterable SNS topic announces, read
# anonymously: keys are SITE/VOL#/YYYYMMDD-HHMMSS-NNN-{S|I|E}.
CHUNK_BUCKET = "https://unidata-nexrad-level2-chunks.s3.amazonaws.com"
CHUNK_DIR = os.path.join(VOL_CACHE_DIR, "chunks")
os.makedirs(CHUNK_DIR, exist_ok=True)
_CHUNK_STATE = {} # site -> {vol:int -> meta dict}
_CHUNK_LOCK = threading.Lock()
_S3NS = {"s3": "http://s3.amazonaws.com/doc/2006-03-01/"}
def _s3_page(prefix, delimiter=None):
url = (f"{CHUNK_BUCKET}/?list-type=2&max-keys=1000"
f"&prefix={urllib.request.quote(prefix)}")
if delimiter:
url += f"&delimiter={delimiter}"
root = ET.fromstring(_http_get(url, timeout=30))
keys = [k.text for k in root.findall(".//s3:Contents/s3:Key", _S3NS)]
pres = [p.text for p in
root.findall(".//s3:CommonPrefixes/s3:Prefix", _S3NS)]
return keys, pres
_CHUNK_RE = re.compile(r"(\d{8})-(\d{6})-(\d{3})-([SIE])$")
def _chunk_list_vol(site, vol):
"""List one volume dir -> meta(start, keys ordered, complete)."""
keys, _ = _s3_page(f"{site}/{vol}/")
parsed = []
for k in keys:
m = _CHUNK_RE.search(k)
if m:
parsed.append((int(m.group(3)), m.group(4), k))
if not parsed:
return None
parsed.sort()
m0 = _CHUNK_RE.search(parsed[0][2])
start = dt.datetime.strptime(m0.group(1) + m0.group(2), "%Y%m%d%H%M%S")
return dict(vol=vol, start=start,
keys=[p[2] for p in parsed],
complete=any(p[1] == "E" for p in parsed))
def _chunk_recent_vols(site, window_min=65):
"""Volumes whose scan started in the trailing window, oldest->newest.
One dir listing + probes of only the newest (and still-open) volumes."""
_, pres = _s3_page(f"{site}/", delimiter="/")
nums = sorted(int(p.split("/")[1]) for p in pres if
p.split("/")[1].isdigit())
if not nums:
return []
# handle 0-999 rotation: if the set spans the wrap, low numbers are new
eff = ([n + 1000 if (max(nums) - min(nums) > 800 and n < 500) else n
for n in nums])
order = [n for _, n in sorted(zip(eff, nums), reverse=True)]
now = dt.datetime.utcnow()
state = _CHUNK_STATE.setdefault(site, {})
out = []
for vol in order[:18]:
meta = state.get(vol)
if meta is None or not meta["complete"]:
meta = _chunk_list_vol(site, vol)
if meta is None:
continue
with _CHUNK_LOCK:
state[vol] = meta
age_min = (now - meta["start"]).total_seconds() / 60.0
if age_min <= window_min:
out.append(meta)
elif age_min > window_min + 20:
break # everything older is out of window too
with _CHUNK_LOCK: # drop state for vols far outside the window
for vol in [v for v, m in state.items()
if (now - m["start"]).total_seconds() > 5400]:
state.pop(vol, None)
return sorted(out, key=lambda m: m["start"])
def _sync_chunks(site, meta):
"""Download missing chunks for one volume; returns ordered local paths."""
vdir = os.path.join(CHUNK_DIR, site, str(meta["vol"]))
os.makedirs(vdir, exist_ok=True)
paths = []
todo = []
for k in meta["keys"]:
p = os.path.join(vdir, os.path.basename(k))
paths.append(p)
if not os.path.exists(p):
todo.append((k, p))
def _dl(job):
k, p = job
try:
data = _http_get(f"{CHUNK_BUCKET}/{urllib.request.quote(k)}",
timeout=60)
tmp = p + ".part"
with open(tmp, "wb") as f:
f.write(data)
os.replace(tmp, p)
except Exception:
pass
if todo:
with ThreadPoolExecutor(max_workers=8) as ex:
list(ex.map(_dl, todo))
return [p for p in paths if os.path.exists(p)]
def fetch_live_chunks(site, window_min=65, progress=None):
"""Phase-1 API: trailing-window volumes as ordered local chunk paths.
Returns [dict(vol, start, complete, paths)] oldest->newest."""
vols = _chunk_recent_vols(site, window_min)
out = []
for i, meta in enumerate(vols):
if progress:
progress(0.05 + 0.9 * (i + 1) / max(1, len(vols)),
f"Syncing live chunks {i + 1}/{len(vols)} "
f"(vol {meta['vol']})…")
paths = _sync_chunks(site, meta)
if paths:
out.append(dict(vol=meta["vol"], start=meta["start"],
complete=meta["complete"], paths=paths))
return out
def _prune_chunk_cache(max_age_s=7200):
try:
now = time_mod.time()
for site_d in os.listdir(CHUNK_DIR):
sdir = os.path.join(CHUNK_DIR, site_d)
for vol_d in os.listdir(sdir):
vdir = os.path.join(sdir, vol_d)
if now - os.path.getmtime(vdir) > max_age_s:
shutil.rmtree(vdir, ignore_errors=True)
except Exception:
pass
CHUNK_SITES = {"KILX"} # (vestigial) every site now uses the chunk feed
_CHUNK_TTL = 600 # s — drop a site's decoded volumes ~10 min after
# it was last viewed (every site is on chunks now)
_CHUNK_FRAMES = {} # (site, vol) -> dict(n, complete, frames, ts)
_CHUNK_DEAL = {} # (site, vol) -> dealiased frames (complete vols)
def _concat_chunks(paths, out_path):
"""S+I+...+E chunks concatenated form a valid archive volume file."""
if not os.path.exists(out_path):
tmp = out_path + ".part"
with open(tmp, "wb") as w:
for p in paths:
with open(p, "rb") as r:
shutil.copyfileobj(r, w)
os.replace(tmp, out_path)
return out_path
def _sweep_nyquist(ds, fvar):
"""Per-sweep Nyquist velocity (m/s): prefer ds['nyquist_velocity'], else
max |velocity| after masking reserved codes. Clamped to a plausible range."""
try:
nv = np.asarray(ds["nyquist_velocity"].values, dtype=float)
nv = nv[np.isfinite(nv)]
if nv.size:
n = float(np.median(nv))
if 5.0 <= n <= 80.0:
return n
except Exception:
pass
try:
v = np.asarray(ds[fvar].values, dtype=np.float32)
rsv = _reserved_mask(ds, fvar, v)
if rsv is not None:
v = np.where(rsv, np.nan, v)
n = float(np.nanmax(np.abs(v)))
if 5.0 <= n <= 80.0:
return n
except Exception:
pass
return 0.0
def _open_nexrad_xr(src):
"""Open a NEXRAD Level 2 volume with the xradar fork. `src` is a local
archive-format file path or a list of live chunk paths."""
if isinstance(src, (list, tuple)):
return xd.io.open_nexradlevel2_datatree(list(src),
incomplete_sweep="drop")
return xd.io.open_nexradlevel2_datatree(_gunzip(src))
def _dealias_dtree(dtree, vol_label):
"""Region-based velocity dealiasing of the low Doppler cuts, Py-ART free:
the Rust region_dealias.sweep_folds() folds each velocity sweep decoded by
the xradar fork; rendered via polar_frame_xr."""
frames = []
if not _HAVE_REGION_DEALIAS:
return frames
fvar = XR_FIELD["velocity"]
try:
vcp = str(dtree.ds.attrs.get("scan_name", "") or "")
except Exception:
vcp = ""
sweeps = []
for name in sorted((c for c in dtree.children if c.startswith("sweep_")),
key=lambda n: int(n.split("_")[1])):
ds = dtree[name].ds
try:
el = float(ds["sweep_fixed_angle"].values)
except Exception:
continue
if el >= ELEV_MAX or fvar not in ds:
continue
if not np.isfinite(ds[fvar].values).any():
continue
sweeps.append(dict(name=name, ds=ds, el=el,
t=_np_dt(ds["time"].values.min())))
if not sweeps:
return frames
_lo = min(s["el"] for s in sweeps) # lowest cut only (e.g. KBUF 0.3°)
sweeps = [s for s in sweeps if s["el"] <= _lo + LOW_SWEEP_TOL]
sweeps = _dedup_doppler(sweeps, lambda s: _sweep_nyquist(s["ds"], fvar))
for s in sweeps:
ds = s["ds"]
try:
v = np.asarray(ds[fvar].values, dtype=np.float32)
nyq = _sweep_nyquist(ds, fvar)
if not (5.0 <= nyq <= 80.0):
continue
rsv = _reserved_mask(ds, fvar, v)
mask = ~np.isfinite(v)
if rsv is not None:
mask = mask | rsv
folds = _region_dealias.sweep_folds(v, mask, float(nyq),
rays_wrap_around=True)
vdeal = np.where(mask, np.nan, v + folds * (2.0 * nyq))
ds2 = ds.copy()
ds2[fvar] = ds2[fvar].copy(data=vdeal)
ds2[fvar].encoding = {} # dealiased = physical m/s; no code mask
fr = polar_frame_xr(ds2, fvar, DEALIAS_CFG, s["el"],
s["name"], vol_label, vcp)
if fr is not None:
frames.append(fr)
except Exception:
continue
return frames
def dealias_volume_file(src, vol_label):
"""Region-dealias one volume — a local archive file path OR a list of live
chunk paths — with the xradar fork + Rust region_dealias (no Py-ART)."""
try:
dtree = _open_nexrad_xr(src)
except Exception:
return []
try:
return _dealias_dtree(dtree, vol_label)
finally:
try:
del dtree
except Exception:
pass
# ---- live warnings (real-time mode) ----------------------------------------
WARN_SRC = ("https://mesonet.agron.iastate.edu/data/gis/shape/4326/us/"
"current_ww.zip")
WARN_JSON = os.path.join(VOL_CACHE_DIR, "warnings.json")
WARN_URL = "/gradio_api/file=" + WARN_JSON
# city label database (GeoNames-derived, shipped with the repo) is served
# from the static cache dir alongside warnings.json
CITIES_SRC = os.path.join(os.path.dirname(__file__), "cities.json")
CITIES_JSON = os.path.join(VOL_CACHE_DIR, "cities.json")
try:
if os.path.exists(CITIES_SRC):
shutil.copyfile(CITIES_SRC, CITIES_JSON)
except Exception:
pass
CITIES_URL = "/gradio_api/file=" + CITIES_JSON
# NEXRAD site locations for the in-map site picker: the curated WSR-88D list
# (sites.py) for IDs, with coordinates from Py-ART's station table. Excludes
# TDWR terminal radars (no Level 2 archive). Served like cities.json.
SITES_JSON = os.path.join(VOL_CACHE_DIR, "sites.json")
try:
_sites = []
for _icao, _city, _st in SITES:
v = NEXRAD_COORDS.get(_icao)
if v:
_sites.append([_icao, round(float(v[0]), 4),
round(float(v[1]), 4)])
_sites.sort()
with open(SITES_JSON, "w") as _f:
json.dump(_sites, _f)
except Exception:
pass
SITES_URL = "/gradio_api/file=" + SITES_JSON
def _fetch_warnings():
"""Pull IEM current watches/warnings, keep storm-based TOR/SVR warnings,
write a GeoJSON FeatureCollection next to the volume cache."""
if _pyshp is None:
return
buf = _http_get(WARN_SRC, timeout=30)
z = zipfile.ZipFile(io.BytesIO(buf))
base = [n for n in z.namelist() if n.endswith(".shp")][0][:-4]
rdr = _pyshp.Reader(shp=io.BytesIO(z.read(base + ".shp")),
dbf=io.BytesIO(z.read(base + ".dbf")),
shx=io.BytesIO(z.read(base + ".shx")))
now = dt.datetime.utcnow()
feats = []
for sr in rdr.iterShapeRecords():
d = sr.record.as_dict()
# TO = tornado, SV = severe t'storm, FF = flash flood — all storm-based
# polygon warnings (GTYPE=P). (Flood Warning FL.W is county-based.)
if not (d.get("SIG") == "W" and d.get("PHENOM") in ("TO", "SV", "FF")
and d.get("GTYPE") == "P"):
continue
# drop warnings already past their expiration (UTC YYYYMMDDHHMM) so
# they vanish from the loop instead of lingering until IEM republishes
exp = str(d.get("EXPIRED") or "")
try:
if len(exp) >= 12 and \
dt.datetime.strptime(exp[:12], "%Y%m%d%H%M") <= now:
continue
except Exception:
pass
feats.append(dict(
type="Feature",
properties=dict(ph=d.get("PHENOM"), wfo=d.get("WFO"),
etn=d.get("ETN"), exp=exp),
geometry=sr.shape.__geo_interface__))
feats.sort(key=lambda f: (str(f["properties"]["wfo"] or ""),
f["properties"]["etn"] or 0))
# etag from the (expiry-filtered) content, NOT the raw zip — so the client
# redraws and clears a warning the moment it drops out of the valid set
etag = hashlib.md5(
json.dumps(feats, sort_keys=True).encode()).hexdigest()[:12]
out = dict(type="FeatureCollection",
etag=etag,
updated=now.isoformat() + "Z",
features=feats)
tmp = WARN_JSON + ".part"
with open(tmp, "w") as f:
json.dump(out, f)
os.replace(tmp, WARN_JSON)
def _warn_poller():
while True:
try:
_fetch_warnings()
except Exception:
pass
time_mod.sleep(60)
# small static assets that live in VOL_CACHE_DIR but must never be LRU-pruned
# (cities.json is copied once at startup and never re-touched, so without this
# guard it sorts oldest and gets evicted as radar volumes fill the 2 GiB cap)
_CACHE_KEEP = {"cities.json", "warnings.json", "sites.json"}
def _prune_vol_cache():
try:
files = [(os.path.getmtime(p), os.path.getsize(p), p)
for p in (os.path.join(VOL_CACHE_DIR, f)
for f in os.listdir(VOL_CACHE_DIR))
if os.path.basename(p) not in _CACHE_KEEP
and os.path.isfile(p)]
files.sort(reverse=True)
now = time_mod.time()
total = 0
for mt, sz, p in files:
total += sz
if total > VOL_CACHE_BYTES or (now - mt) > VOL_CACHE_MAX_AGE_S:
os.remove(p)
except Exception:
pass
YEARS = [str(y) for y in range(1991, dt.datetime.utcnow().year + 1)][::-1]
MONTHS = [f"{m:02d}" for m in range(1, 13)]
DAYS = [f"{d:02d}" for d in range(1, 32)]
HOURS = [f"{h:02d}:00" for h in range(24)]
# ----------------------------------------------------------------------------- s3 access
def _http_get(url, timeout=60):
req = urllib.request.Request(url, headers={"User-Agent": "nexrad-browser/1.0"})
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.read()
def list_hour_keys(site, date, hour):
"""Return (bucket, [keys]) for volumes starting in the given UTC hour."""
prefix = f"{date:%Y/%m/%d}/{site}/{site}{date:%Y%m%d}_{hour:02d}"
last_err = None
any_ok = False
for bucket in BUCKETS:
url = f"{bucket}/?list-type=2&prefix={urllib.request.quote(prefix)}&max-keys=200"
try:
xml = _http_get(url, timeout=30)
except Exception as e: # 403 / network — try next bucket
last_err = e
continue
ns = {"s3": "http://s3.amazonaws.com/doc/2006-03-01/"}
root = ET.fromstring(xml)
any_ok = True
keys = [k.text for k in root.findall(".//s3:Contents/s3:Key", ns)]
keys = [k for k in keys if not k.endswith("_MDM") and "NXL2" not in k]
if keys:
return bucket, sorted(keys)
if last_err and not any_ok:
raise RuntimeError(f"Could not list archive buckets: {last_err}")
return None, []
def download_volume(bucket, key, dest_dir):
path = os.path.join(dest_dir, os.path.basename(key))
if not os.path.exists(path):
data = _http_get(f"{bucket}/{urllib.request.quote(key)}", timeout=120)
with open(path, "wb") as f:
f.write(data)
return path
def _safe_download(bucket, key, dest_dir):
try:
download_volume(bucket, key, dest_dir)
except Exception:
pass
# ----------------------------------------------------------------------------- radar processing
def sweep_datetime(radar, sweep):
units = radar.time["units"] # "seconds since YYYY-MM-DDTHH:MM:SSZ"
m = re.search(r"since\s+([0-9T:\-]+)", units)
base = dt.datetime.fromisoformat(m.group(1).rstrip("Z"))
s = radar.sweep_start_ray_index["data"][sweep]
return base + dt.timedelta(seconds=float(radar.time["data"][s]))
def _scale_u8(data, mask, field_cfg):
"""float field -> uint8 (0 = no data, 1..255 spans vmin..vmax)."""
scaled = np.clip(
(data - field_cfg["vmin"]) / (field_cfg["vmax"] - field_cfg["vmin"]), 0, 1)
vals = (scaled * 254 + 1).astype(np.uint8)
vals[mask] = 0
return vals
# gates whose reflectivity is below this (or with no reflectivity return at
# all) are treated as "no return" and rendered fully transparent — applied to
# every field, so V/ZDR/CC noise outside echo disappears too
Z_MIN_DBZ = -30.0
def _noreturn_mask(z):
"""True where there is no usable return (Z missing or < Z_MIN_DBZ)."""
zf = np.ma.filled(np.ma.masked_invalid(z), -1e3)
return ~(zf >= Z_MIN_DBZ)
def _regrid_az(az, vals):
"""Gather rays onto a uniform azimuth grid (every bin takes its nearest
ray, so no empty spokes). Returns (grid, naz)."""
daz = np.median(np.abs(np.diff(np.unwrap(np.radians(az))))) * 180 / np.pi
naz = 720 if daz < 0.75 else 360
order = np.argsort(az)
az_s = az[order]
centers = (np.arange(naz) + 0.5) * (360.0 / naz)
idx = np.searchsorted(az_s, centers)
lo = (idx - 1) % len(az_s)
hi = idx % len(az_s)
d_lo = np.abs((centers - az_s[lo] + 180) % 360 - 180)
d_hi = np.abs((centers - az_s[hi] + 180) % 360 - 180)
nearest = np.where(d_lo <= d_hi, lo, hi)
return vals[order[nearest]], naz
def polar_frame(radar, sweep, field_cfg):
"""Pack one sweep onto a uniform azimuth grid -> gate-native frame dict.
The uint8 grid (rows=azimuth bins, cols=gates) is PNG-compressed
losslessly; 0 = no data, 1..255 spans [vmin, vmax].
"""
fname = field_cfg["pyart"]
data = radar.get_field(sweep, fname)
mask = np.ma.getmaskarray(data)
if mask.all():
return None
s_idx = radar.sweep_start_ray_index["data"][sweep]
e_idx = radar.sweep_end_ray_index["data"][sweep]
az = radar.azimuth["data"][s_idx:e_idx + 1]
rng = radar.range["data"]
dr = float(rng[1] - rng[0])
r0 = float(rng[0]) - dr / 2.0 # edge of first gate
ngates = data.shape[1]
try: # censor gates with no usable return (same-sweep reflectivity);
# skip when this sweep has no Z at all (legacy Doppler cuts)
z = radar.get_field(sweep, "reflectivity")
if not np.all(np.ma.getmaskarray(z)):
mask = mask | _noreturn_mask(z)
except Exception:
pass
vals = _scale_u8(np.ma.filled(data, field_cfg["vmin"]), mask, field_cfg)
grid, naz = _regrid_az(az, vals)
buf = io.BytesIO()
Image.fromarray(grid, mode="L").save(buf, format="WEBP",
lossless=True, method=4)
t = sweep_datetime(radar, sweep)
el = float(radar.fixed_angle["data"][sweep])
vcp = getattr(polar_frame, "_vcp", "")
vcp_part = f"{vcp} • " if vcp else ""
return dict(
img=base64.b64encode(buf.getvalue()).decode(),
naz=naz, ngates=ngates, r0=r0, dr=dr, el=el,
maxr=r0 + ngates * dr, time=t,
label=(f"{t:%Y-%m-%d %H:%M:%S}Z • {el:.1f}° • {vcp_part}"
f"sweep {sweep} • {polar_frame._vol}"),
)
def colorbar_cfg(field_cfg, n=256):
"""Colormap stops + range; drawn client-side and used as the WebGL LUT."""
cm = plt.get_cmap(field_cfg["cmap"])
stops = [matplotlib.colors.to_hex(cm(i / (n - 1))) for i in range(n)]
unit = f" ({field_cfg['units']})" if field_cfg["units"] else ""
return dict(stops=stops, vmin=field_cfg["vmin"], vmax=field_cfg["vmax"],
tick=field_cfg["tick"], units=field_cfg["units"],
label=f"{field_cfg['label']}{unit}")
XR_FIELD = {"reflectivity": "DBZH", "velocity": "VRADH",
"differential_reflectivity": "ZDR",
"cross_correlation_ratio": "RHOHV"}
def _gunzip(path):
if not path.endswith(".gz"):
return path
out = path[:-3]
if not os.path.exists(out):
with gzip.open(path, "rb") as fi, open(out, "wb") as fo:
shutil.copyfileobj(fi, fo)
return out
def _np_dt(t64):
return dt.datetime.utcfromtimestamp(
int(np.datetime64(t64, "ns").astype("int64")) / 1e9)
def polar_frame_xr(ds, fvar, field_cfg, el, sweep_name, vol, vcp=""):
data = ds[fvar].values
finite = np.isfinite(data)
if not finite.any():
return None
az = ds["azimuth"].values
rng = ds["range"].values
dr = float(rng[1] - rng[0])
r0 = float(rng[0]) - dr / 2.0
ngates = data.shape[1]
nodata = ~finite
rsv = _reserved_mask(ds, fvar, data)
if rsv is not None: # below-threshold / range-folded codes
nodata = nodata | rsv
if "DBZH" in ds: # censor gates with no usable return
zdat = ds["DBZH"].values
zbad = _noreturn_mask(zdat)
zrsv = _reserved_mask(ds, "DBZH", zdat)
if zrsv is not None:
zbad = zbad | zrsv
nodata = nodata | zbad
vals = _scale_u8(np.nan_to_num(data, nan=field_cfg["vmin"]),
nodata, field_cfg)
grid, naz = _regrid_az(az, vals)
buf = io.BytesIO()
Image.fromarray(grid, mode="L").save(buf, format="WEBP",
lossless=True, method=4)
t = _np_dt(ds["time"].values.min())
vcp_part = f"{vcp} • " if vcp else ""
return dict(
img=base64.b64encode(buf.getvalue()).decode(),
naz=naz, ngates=ngates, r0=r0, dr=dr, el=el,
maxr=r0 + ngates * dr, time=t,
label=(f"{t:%Y-%m-%d %H:%M:%S}Z • {el:.1f}° • {vcp_part}"
f"{sweep_name} • {vol}"),
)
def _reserved_mask(ds, fvar, data):
"""NEXRAD moment codes 0 (below threshold) and 1 (range folded) are
reserved, but xradar decodes them as physical values (no _FillValue in
its attrs). Mask anything at/below code 1 using the CF encoding."""
enc = getattr(ds[fvar], "encoding", {}) or {}
sf, ao = enc.get("scale_factor"), enc.get("add_offset")
if sf is None or ao is None:
return None
return data <= (ao + 1.5 * abs(sf))
def _dedup_doppler(cands, nyq_of, window_s=90):
"""Collapse MPDA-style multi-PRF bursts: within a cluster of Doppler
cuts closer than window_s, keep only the highest-Nyquist cut — but only
when the Nyquists actually differ (>2 m/s spread). Equal-Nyquist cuts
are SAILS/MRLE revisits and are all kept."""
out = []
cluster = []
def flush():
if not cluster:
return
nv = [nyq_of(s) for s in cluster]
if max(nv) - min(nv) > 2.0:
out.append(cluster[int(np.argmax(nv))])
else:
out.extend(cluster)
for s in cands: # cands are in scan order
if cluster and (s["t"] - cluster[0]["t"]).total_seconds() > window_s:
flush()
cluster = []
cluster.append(s)
flush()
return out
def _process_xradar(path, key, cfgs):
"""Read once with xradar (swnesbitt fork); render every requested field.
Returns ({field_name: [frames]}, site_ll)."""
dtree = xd.io.open_nexradlevel2_datatree(_gunzip(path))
return _tree_to_frames(dtree, os.path.basename(key), cfgs)
def process_chunks(paths, vol_label, cfgs):
"""Decode a live chunk list (S/I/E pieces of one volume) — the fork
assembles partial volumes; incomplete trailing sweeps are dropped."""
dtree = xd.io.open_nexradlevel2_datatree(list(paths),
incomplete_sweep="drop")
return _tree_to_frames(dtree, vol_label, cfgs)
def _tree_to_frames(dtree, vol, cfgs):
root = dtree.ds
site_ll = (float(root["latitude"].values), float(root["longitude"].values))
vcp = str(root.attrs.get("scan_name", "") or "")
dyn = str(root.attrs.get("dynamic_scan_type", "") or "")
if dyn and dyn.lower() not in ("none", "false", "", "standard"):
vcp = f"{vcp} ({dyn})" if vcp else dyn
sweeps = []
for name in sorted((c for c in dtree.children if c.startswith("sweep_")),
key=lambda n: int(n.split("_")[1])):
ds = dtree[name].ds
el = float(ds["sweep_fixed_angle"].values)
if el >= ELEV_MAX:
continue
has_v = "VRADH" in ds and bool(np.isfinite(ds["VRADH"].values).any())
sweeps.append(dict(name=name, ds=ds, el=el, has_v=has_v,
t=_np_dt(ds["time"].values.min())))
# show only the lowest-elevation cut: a site that scans below 0.5° (e.g.
# KBUF's 0.3° supplemental cut) shows just that, not it AND the 0.5° cut
if sweeps:
_lo = min(s["el"] for s in sweeps)
sweeps = [s for s in sweeps if s["el"] <= _lo + LOW_SWEEP_TOL]
out = {}
for fname, cfg in cfgs.items():
fvar = XR_FIELD[cfg["pyart"]]
cands = [s for s in sweeps
if fvar in s["ds"]
and bool(np.isfinite(s["ds"][fvar].values).any())]
if cfg["pyart"] != "velocity":
# surveillance (long-range) cuts only — never the shorter-range
# Doppler split cuts; fall back for merged-cut VCPs
surv = [s for s in cands if not s["has_v"]]
cands = surv or cands
else:
# MPDA (VCP 121) runs several near-simultaneous Doppler cuts at
# different PRFs: keep only the highest-Nyquist member of each
# cluster. With reserved codes masked, max |V| == the Nyquist.
def _est_nyq(s):
v = s["ds"][fvar].values
rsv = _reserved_mask(s["ds"], fvar, v)
if rsv is not None:
v = np.where(rsv, np.nan, v)
return float(np.nanmax(np.abs(v)))
cands = _dedup_doppler(cands, _est_nyq)
frames = []
for s in cands:
try:
fr = polar_frame_xr(s["ds"], fvar, cfg, s["el"],
s["name"], vol, vcp)
except Exception:
continue
if fr is not None:
frames.append(fr)
out[fname] = frames
del dtree
return out, site_ll
def dealias_volume(bucket, key, dest_dir):
"""Download a volume and region-dealias its low Doppler cuts
(xradar fork + Rust region_dealias; no Py-ART)."""
try:
path = download_volume(bucket, key, dest_dir)
except Exception:
return []
return dealias_volume_file(path, os.path.basename(key))
def process_volume(bucket, key, cfgs, dest_dir):
"""Download + read one volume with the xradar fork (sole reader).
Decodes every requested field in a single read; raw files stay cached."""
empty = {fn: [] for fn in cfgs}
try:
path = download_volume(bucket, key, dest_dir)
except Exception:
return empty, None
try:
return _process_xradar(path, key, cfgs)
except Exception:
return empty, None
# (Py-ART fallback reader removed — the xradar fork is the sole ingest path)
# ----------------------------------------------------------------------------- leaflet + webgl page
LEAFLET_PAGE = """
"""
# small base64 of the CliMAS logo for export stamping (from logo.png)
def _export_logo_b64():
p = os.path.join(os.path.dirname(__file__), "logo.png")
if not os.path.exists(p):
return ""
try:
im = Image.open(p).convert("RGB").resize((512, 512), Image.LANCZOS)
buf = io.BytesIO()
im.save(buf, format="PNG", optimize=True)
return base64.b64encode(buf.getvalue()).decode()
except Exception:
return ""
EXPORT_LOGO_B64 = _export_logo_b64()
def _bundle_data(by_field):
"""The per-field DATA payload (textures + colorbar) the bundle renders."""
return [dict(name=fn, cbar=colorbar_cfg(_field_cfg(fn)),
frames=[{k: f[k] for k in ("img", "naz", "ngates", "r0",
"dr", "el", "maxr", "label")}
for f in frames])
for fn, frames in by_field.items()]
def _live_sig(data):
"""Cheap content signature: changes whenever any frame is added/updated."""
parts = []
for d in data:
parts.append(d["name"] + ":" + str(len(d["frames"])))
parts.append(",".join(str(fr["label"]) for fr in d["frames"]))
return hashlib.md5("|".join(parts).encode()).hexdigest()[:12]
def _live_paths(site):
s = site.upper()
return (os.path.join(VOL_CACHE_DIR, f"live_{s}.json"),
os.path.join(VOL_CACHE_DIR, f"live_{s}.tag.json"))
def _write_live(site, by_field):
"""Write the live frame set (+ a tiny etag-only sidecar) so the in-page
poller can merge new frames WITHOUT reloading the whole iframe."""
try:
data = _bundle_data(by_field)
etag = _live_sig(data)
ts = time_mod.time() # bumps every compute, even when etag is unchanged
big, tag = _live_paths(site)
for path, payload in ((big, dict(etag=etag, ts=ts, fields=data)),
(tag, dict(etag=etag, ts=ts))):
tmp = path + ".part"
with open(tmp, "w") as f:
json.dump(payload, f)
os.replace(tmp, path)
return etag
except Exception:
return ""
def build_bundle_page(by_field, site, slat, slon, share_base="",
rt_refresh_s=300):
"""One page carrying ALL fields' polar textures; the requested view
(single field or 2×2) is substituted into __MODE__ at serve time, so
every field/4-panel switch after the first load is instant."""
data = _bundle_data(by_field)
etag = _live_sig(data)
big, tag = _live_paths(site)
page = (QUAD_PAGE
.replace("__DATA__", json.dumps(data))
.replace("__SLAT__", f"{slat:.5f}")
.replace("__SLON__", f"{slon:.5f}")
.replace("__SITE__", site)
.replace("__WARNURL__", WARN_URL)
.replace("__CITIESURL__", CITIES_URL)
.replace("__SITESURL__", SITES_URL)
.replace("__LIVEURL__", "/gradio_api/file=" + big)
.replace("__LIVETAGURL__", "/gradio_api/file=" + tag)
.replace("__LIVETAG0__", json.dumps(etag))
.replace("__LOGOB64__", EXPORT_LOGO_B64)
.replace("__RTSEC__", str(int(rt_refresh_s)))
.replace("__SHAREBASE__", share_base))
return (f'')
def _mode_page(tpl, field_name, view=None, deal=False, rt=False):
"""Substitute the initial view mode, optional [lat, lon, zoom] and the
dealias flag into a cached bundle template. The template is already
HTML-escaped (srcdoc), so escape the JSON too."""
if field_name == QUAD:
mode = "quad"
elif field_name == ZVF:
mode = "zv"
elif field_name == DEALIAS_NAME:
mode, deal = "Radial velocity", True
else:
mode = field_name
return (tpl
.replace("__MODE__", html_mod.escape(json.dumps(mode)))
.replace("__VIEW__", html_mod.escape(json.dumps(view)))
.replace("__DEAL__", "true" if deal else "false")
.replace("__RT__", "true" if rt else "false"))
# ----------------------------------------------------------------------------- gradio callback
# rendered-page cache: default case is pre-rendered at startup, and recent
# views are served instantly (share links, repeat visits)
DEFAULT_VIEW = ("KILX", "Reflectivity", "2023", "06", "29", "18:00")
_PAGE_CACHE = collections.OrderedDict()
_PAGE_CACHE_CAP = 6 # hours; each bundle holds all four fields (~30-50 MB)
_FRAME_CACHE = collections.OrderedDict() # hour -> raw frames (for dealias)
_FRAME_CACHE_CAP = 4
_INFLIGHT = {}
_CACHE_LOCK = threading.Lock()
def _rt_keys(site):
"""Volumes whose start time lies in the trailing 60 minutes."""
now = dt.datetime.utcnow()
bucket, keys = None, []
for hdt in (now - dt.timedelta(hours=1), now):
try:
b, k = list_hour_keys(site, hdt.date(), hdt.hour)
except Exception:
b, k = None, []
if k:
bucket = bucket or b
keys += k
cutoff = now - dt.timedelta(minutes=62)
sel = []
for k in keys:
m = re.search(r"(\d{8})_(\d{6})", k)
if m:
t = dt.datetime.strptime(m.group(1) + m.group(2),
"%Y%m%d%H%M%S")
if t >= cutoff:
sel.append(k)
return bucket, sorted(set(sel)), now
_RT_CACHE = {} # site -> dict(ts, tpl, by_field, site_ll, has_deal)
_RT_TTL = 150 # seconds — refresh cadence is 300 s
# sites with a recent live viewer -> (last_request_ts, want_deal). A background
# daemon keeps these refreshed so ingest never depends on a browser tab driving
# the client timer (backgrounded tabs freeze their timers).
_LIVE_ACTIVE = {}
_LIVE_ACTIVE_TTL = 20 * 60 # keep polling a site for 20 min after last view
def _realtime_compute(site, field_name, _p, view, want_deal=False):
"""Decode the trailing hour. Short-lived cache + in-flight dedup keep
reloads and concurrent viewers from re-decoding live data. Dealiasing
is supported: the cached raw decode is upgraded in place."""
ttl = 45 # short-lived live cache (every site is on the chunk feed)
c = _RT_CACHE.get(site)
fresh = c and time_mod.time() - c["ts"] < ttl
if fresh and (not want_deal or c["has_deal"]):
return "", _mode_page(c["tpl"], field_name, view,
deal=want_deal, rt=True)
key_rt = ("RT", site)
with _CACHE_LOCK:
evt = _INFLIGHT.get(key_rt)
owner = evt is None
if owner:
_INFLIGHT[key_rt] = threading.Event()
if not owner:
_p(0.5, "Live hour already rendering — attaching…")
evt.wait(600)
c = _RT_CACHE.get(site)
if c and (not want_deal or c["has_deal"]):
return "", _mode_page(c["tpl"], field_name, view,
deal=want_deal, rt=True)
try:
return _realtime_compute_inner(site, field_name, _p, view,
want_deal)
finally:
if owner:
with _CACHE_LOCK:
ev = _INFLIGHT.pop(key_rt, None)
if ev:
ev.set()
def _realtime_chunks_inner(site, field_name, _p, view, want_deal=False):
"""Live mode on the Level 2 chunk feed: per-volume incremental decode —
only the in-progress volume (and newly completed ones) cost CPU."""
_p(0.02, f"Polling the live chunk feed for {site}…")
vols = fetch_live_chunks(
site, progress=lambda f, d: _p(0.02 + 0.28 * f, d))
if not vols:
raise RuntimeError("no live chunks")
by_field = {fn: [] for fn in FIELDS}
site_ll = None
todo = []
for v in vols:
ck = (site, v["vol"])
cached = _CHUNK_FRAMES.get(ck)
if (cached and cached["n"] == len(v["paths"])
and cached["complete"] == v["complete"]):
continue
todo.append(v)
done = 0
for i in range(0, len(todo), 2):
grp = todo[i:i + 2]
with _DECODE_SEM:
with ProcessPoolExecutor(max_workers=N_PROC) as px:
futs = {px.submit(
process_chunks, v["paths"],
"live vol %d%s" % (v["vol"],
"" if v["complete"] else
" (updating)"),
FIELDS): v for v in grp}
for fut in as_completed(futs):
v = futs[fut]
done += 1
_p(0.32 + 0.40 * done / max(1, len(todo)),
f"Decoding live volume {v['vol']} "
f"({done}/{len(todo)})…")
try:
frames, ll = fut.result()
except Exception:
continue
_CHUNK_FRAMES[(site, v["vol"])] = dict(
n=len(v["paths"]), complete=v["complete"],
frames=frames, site_ll=ll, ts=time_mod.time())
for v in vols:
cached = _CHUNK_FRAMES.get((site, v["vol"]))
if not cached:
continue
cached["ts"] = time_mod.time() # touch -> LRU expiry by last view
site_ll = site_ll or cached.get("site_ll")
for fn, fr in cached["frames"].items():
by_field[fn].extend(fr)
if want_deal:
comp = [v for v in vols if v["complete"]]
todo_d = [v for v in comp if (site, v["vol"]) not in _CHUNK_DEAL]
dn = 0
for i in range(0, len(todo_d), 2):
grp = todo_d[i:i + 2]
with _DECODE_SEM:
with ProcessPoolExecutor(max_workers=N_PROC) as px:
futs = {px.submit(
dealias_volume_file, v["paths"],
"live vol %d" % v["vol"]): v for v in grp}
for fut in as_completed(futs):
v = futs[fut]
dn += 1
_p(0.74 + 0.18 * dn / max(1, len(todo_d)),
f"Dealiasing live volume {v['vol']} "
f"({dn}/{len(todo_d)}, {_DEALIAS_ENGINE})…")
try:
_CHUNK_DEAL[(site, v["vol"])] = dict(
ts=time_mod.time(), frames=fut.result())
except Exception:
_CHUNK_DEAL[(site, v["vol"])] = dict(
ts=time_mod.time(), frames=[])
dframes = []
for v in comp:
d = _CHUNK_DEAL.get((site, v["vol"]))
if d:
d["ts"] = time_mod.time()
dframes.extend(d["frames"])
dframes.sort(key=lambda f: f["time"])
by_field[DEALIAS_NAME] = dframes[:MAX_FRAMES]
for fn in by_field:
by_field[fn].sort(key=lambda f: f["time"])
by_field[fn] = by_field[fn][:MAX_FRAMES]
# short-lived caches: drop volumes for sites not viewed within _CHUNK_TTL,
# with a hard count backstop (every site uses the chunk feed now)
_now = time_mod.time()
for cache in (_CHUNK_FRAMES, _CHUNK_DEAL):
for k in [k for k, vv in cache.items()
if _now - vv.get("ts", 0) > _CHUNK_TTL]:
cache.pop(k, None)
while len(cache) > 80:
cache.pop(next(iter(cache)))
_prune_chunk_cache(4800) # files: cover the ~65-min loop window
return _rt_finish(site, field_name, _p, view, want_deal,
by_field, site_ll)
def _realtime_compute_inner(site, field_name, _p, view, want_deal=False):
# Every site uses the low-latency chunk feed (so in-progress volumes show);
# fall back to the assembled-archive bucket only if a site has no chunks.
try:
return _realtime_chunks_inner(site, field_name, _p, view, want_deal)
except Exception:
pass # fall back to the archive-bucket live path below
_p(0.02, f"Listing the last 60 minutes for {site}…")
bucket, keys, now = _rt_keys(site)
if not keys:
return _msg(f"No Level 2 volumes from {site} in the last hour — "
f"the radar may be down or data is still in transit.")
n = len(keys)
c = _RT_CACHE.get(site)
if (c and time_mod.time() - c["ts"] < _RT_TTL and want_deal
and not c["has_deal"]):
# fresh raw decode exists — only the dealias pass is needed
by_field = dict(c["by_field"])
site_ll = c["site_ll"]
else:
with ThreadPoolExecutor(max_workers=6) as ex:
futs = [ex.submit(_safe_download, bucket, k, VOL_CACHE_DIR)
for k in keys]
for i, _ in enumerate(as_completed(futs)):
_p(0.04 + 0.16 * (i + 1) / n,
f"Fetching live volumes… {i + 1}/{n}")
by_field, site_ll = _decode_keys(bucket, keys, _p, 0.22, 0.50,
"Decoding live volumes")
_prune_vol_cache()
if want_deal:
dframes, done = [], 0
for i in range(0, n, 2):
chunk = keys[i:i + 2]
with _DECODE_SEM:
with ProcessPoolExecutor(max_workers=N_PROC) as px:
pfuts = [px.submit(dealias_volume, bucket, k,
VOL_CACHE_DIR) for k in chunk]
for fut in as_completed(pfuts):
done += 1
_p(0.72 + 0.22 * done / n,
f"Dealiasing live volumes {done}/{n} "
f"({_DEALIAS_ENGINE})…")
dframes.extend(fut.result())
dframes.sort(key=lambda f: f["time"])
by_field[DEALIAS_NAME] = dframes[:MAX_FRAMES]
return _rt_finish(site, field_name, _p, view, want_deal,
by_field, site_ll)
def _rt_finish(site, field_name, _p, view, want_deal, by_field, site_ll):
if site_ll is not None and abs(site_ll[0]) < 0.1 and abs(site_ll[1]) < 0.1:
site_ll = None
if site_ll is None:
loc = NEXRAD_COORDS.get(site)
if loc:
site_ll = (loc[0], loc[1])
if site_ll is None or not any(by_field.values()):
return _msg(f"Could not decode any live data from {site}.")
_p(0.96, "Packing live bundle…")
rtsec = 120 # in-browser live self-refresh cadence (s)
tpl = build_bundle_page(by_field, site, site_ll[0], site_ll[1],
f"?site={site}&rt=1", rt_refresh_s=rtsec)
_write_live(site, by_field) # refresh the in-place poll file
_RT_CACHE[site] = dict(ts=time_mod.time(), tpl=tpl, by_field=by_field,
site_ll=site_ll,
has_deal=DEALIAS_NAME in by_field)
for k in [k for k in _RT_CACHE if k != site][4:]:
_RT_CACHE.pop(k, None)
return "", _mode_page(tpl, field_name, view, deal=want_deal, rt=True)
def browse(site, field_name, year, month, day, hour, progress=None,
view=None, want_deal=False, realtime=False):
def _p(frac, desc):
if progress is not None:
try:
progress(frac, desc=desc)
except Exception:
pass
try:
m = re.match(r"\s*([A-Za-z]{4})\b", site or "")
if not m:
return _msg("Pick a NEXRAD site (e.g. KTLX, KILX, PHWA).")
site = m.group(1).upper()
if realtime:
# remember this site so the background poller keeps it fresh even
# if the client timer stops (inactive/backgrounded tab)
_LIVE_ACTIVE[site] = (time_mod.time(), bool(want_deal))
return _realtime_compute(site, field_name, _p, view, want_deal)
try:
date = dt.date(int(year), int(month), int(day))
except ValueError:
return _msg("That calendar date doesn't exist — check day/month.")
hr = int(str(hour)[:2])
if field_name not in FIELDS and field_name not in (QUAD, ZVF,
DEALIAS_NAME):
field_name = "Reflectivity"
# ---- per-hour bundle cache / in-flight dedup ---------------------
key_h = _hour_key(site, date, hr)
with _CACHE_LOCK:
tpl = _PAGE_CACHE.get(key_h)
if tpl is not None:
_PAGE_CACHE.move_to_end(key_h)
if tpl is not None:
if ((field_name == DEALIAS_NAME or want_deal)
and _DEAL_MARKER not in tpl):
tpl = _dealias_compute(site, date, hr, _p)
return "", _mode_page(tpl, field_name, view, want_deal)
with _CACHE_LOCK:
evt = _INFLIGHT.get(key_h)
owner = evt is None
if owner:
_INFLIGHT[key_h] = threading.Event()
if not owner:
_p(0.5, "This hour is already rendering — attaching to it…")
evt.wait(900)
with _CACHE_LOCK:
tpl = _PAGE_CACHE.get(key_h)
if tpl is not None:
if field_name == DEALIAS_NAME and _DEAL_MARKER not in tpl:
tpl = _dealias_compute(site, date, hr, _p)
return "", _mode_page(tpl, field_name, view)
# fall through and compute ourselves if the other run failed
try:
err, tpl = _browse_compute(site, field_name, date, hr, _p)
if err is not None:
return err
if ((field_name == DEALIAS_NAME or want_deal)
and _DEAL_MARKER not in tpl):
tpl = _dealias_compute(site, date, hr, _p)
return "", _mode_page(tpl, field_name, view, want_deal)
finally:
if owner:
with _CACHE_LOCK:
ev = _INFLIGHT.pop(key_h, None)
if ev:
ev.set()
except Exception:
return _msg("Unexpected error:\n```\n"
+ traceback.format_exc()[-1500:] + "\n```")
# marker proving the dealiased field is inside a bundle template (must not
# collide with the JS SHORT-name literal, hence the json "name" form)
_DEAL_MARKER = html_mod.escape(json.dumps("name") + ": "
+ json.dumps(DEALIAS_NAME))
# Only one decode batch may use the CPUs at a time, and work is chunked so
# concurrent requests (boot pre-render vs. a live user) interleave fairly
# instead of thrashing two process pools on two cores.
_DECODE_SEM = threading.Semaphore(1)
def _decode_keys(bucket, keys, _p, base, span, label):
"""Decode volumes (all fields) in semaphore-guarded chunks of 2."""
by_field = {fn: [] for fn in FIELDS}
site_ll = None
n, done = len(keys), 0
for i in range(0, n, 2):
chunk = keys[i:i + 2]
with _DECODE_SEM:
with ProcessPoolExecutor(max_workers=N_PROC) as px:
futs = [px.submit(process_volume, bucket, k, FIELDS,
VOL_CACHE_DIR) for k in chunk]
for fut in as_completed(futs):
done += 1
_p(base + span * done / n, f"{label} {done}/{n}…")
volframes, ll = fut.result()
for fn, fr in volframes.items():
by_field[fn].extend(fr)
site_ll = site_ll or ll
for fn in by_field:
by_field[fn].sort(key=lambda f: f["time"])
by_field[fn] = by_field[fn][:MAX_FRAMES]
return by_field, site_ll
def _hour_key(site, date, hr):
return (site, f"{date.year}", f"{date.month:02d}",
f"{date.day:02d}", f"{hr:02d}")
def _share_base(site, date, hr):
qs = (f"?site={site}&year={date.year}&month={date.month:02d}"
f"&day={date.day:02d}&hour={hr:02d}")
host = os.environ.get("SPACE_HOST", "")
return f"https://{host}/{qs}" if host else qs
def _browse_compute(site, field_name, date, hr, _p):
"""Decode ALL fields in one pass over the hour's volumes; build and
cache the bundle template. Returns (error_tuple_or_None, tpl_or_None)."""
try:
_p(0.02, f"Listing Level 2 volumes for {site} {date} {hr:02d}Z…")
bucket, keys = list_hour_keys(site, date, hr)
if not keys:
return _msg(
f"No Level 2 volumes found for {site} on {date} "
f"{hr:02d}:00–{hr:02d}:59 UTC. Check the site ID and date "
f"(dual-pol fields require 2011+ for most sites)."
), None
n = len(keys)
with ThreadPoolExecutor(max_workers=6) as ex:
futs = [ex.submit(_safe_download, bucket, k, VOL_CACHE_DIR)
for k in keys]
for i, _ in enumerate(as_completed(futs)):
_p(0.04 + 0.20 * (i + 1) / n,
f"Downloading volumes from AWS… {i + 1}/{n} "
f"(cached volumes skip this)")
by_field, site_ll = _decode_keys(
bucket, keys, _p, 0.26, 0.66,
"Decoding volumes (all four fields)")
_prune_vol_cache()
# legacy (Message 1 era, pre-~2008) volumes carry no site coords and
# decode as 0°N 0°E — fall back to the WSR-88D station table
if site_ll is not None and abs(site_ll[0]) < 0.1 and abs(site_ll[1]) < 0.1:
site_ll = None
if site_ll is None:
loc = NEXRAD_COORDS.get(site)
if loc:
site_ll = (loc[0], loc[1])
if site_ll is None or not any(by_field.values()):
return _msg(
f"Volumes were found for {site} in that hour, but no 0.5° "
f"sweep data could be decoded. Pre-dual-pol data (before "
f"~2011-2013 depending on site) has no ZDR/CC."
), None
_p(0.94, "Packing the all-field WebGL bundle (textures for every "
"field load once)…")
slat, slon = site_ll
# keep empty fields in the bundle: their panels show an explanatory
# message (e.g. dual-pol fields on pre-upgrade data)
tpl = build_bundle_page(by_field, site, slat, slon,
_share_base(site, date, hr))
with _CACHE_LOCK:
_PAGE_CACHE[_hour_key(site, date, hr)] = tpl
while len(_PAGE_CACHE) > _PAGE_CACHE_CAP:
_PAGE_CACHE.popitem(last=False)
_FRAME_CACHE[_hour_key(site, date, hr)] = dict(
by_field=by_field, slat=slat, slon=slon)
while len(_FRAME_CACHE) > _FRAME_CACHE_CAP:
_FRAME_CACHE.popitem(last=False)
return None, tpl
except Exception:
return _msg("Unexpected error:\n```\n"
+ traceback.format_exc()[-1500:] + "\n```"), None
def _dealias_compute(site, date, hr, _p):
"""Region-based dealiasing for the hour; merges the new field into the
cached bundle and returns the updated template."""
key_h = _hour_key(site, date, hr)
with _CACHE_LOCK:
fc = _FRAME_CACHE.get(key_h)
if fc is None: # server restarted since the hour was decoded
err, _tpl = _browse_compute(site, "Reflectivity", date, hr, _p)
if err is not None:
raise RuntimeError(err[0])
with _CACHE_LOCK:
fc = _FRAME_CACHE.get(key_h)
bucket, keys = list_hour_keys(site, date, hr)
n = len(keys)
dframes = []
done = 0
for i in range(0, n, 2):
chunk = keys[i:i + 2]
with _DECODE_SEM:
with ProcessPoolExecutor(max_workers=N_PROC) as px:
pfuts = [px.submit(dealias_volume, bucket, k, VOL_CACHE_DIR)
for k in chunk]
for fut in as_completed(pfuts):
done += 1
_p(0.05 + 0.85 * done / n,
f"Region-based dealiasing {done}/{n} volumes "
f"({_DEALIAS_ENGINE})…")
dframes.extend(fut.result())
dframes.sort(key=lambda f: f["time"])
dframes = dframes[:MAX_FRAMES]
_p(0.95, "Rebuilding bundle with dealiased velocity…")
by_field = dict(fc["by_field"])
by_field[DEALIAS_NAME] = dframes
tpl = build_bundle_page(by_field, site, fc["slat"], fc["slon"],
_share_base(site, date, hr))
with _CACHE_LOCK:
_PAGE_CACHE[key_h] = tpl
_FRAME_CACHE[key_h] = dict(by_field=by_field,
slat=fc["slat"], slon=fc["slon"])
return tpl
def _msg(text):
return text, ""
def prepare_archive(site, year, month, day, hour):
"""Bundle the hour's raw Level 2 volumes into SITEyyyymmdd_HHUTC.zip
(files are pulled from the volume cache; anything missing is fetched).
The volumes are internally compressed already, so members are stored."""
m = re.match(r"\s*([A-Za-z]{4})\b", site or "")
if not m:
raise gr.Error("Pick a site first.")
site = m.group(1).upper()
date = dt.date(int(year), int(month), int(day))
hr = int(str(hour)[:2])
bucket, keys = list_hour_keys(site, date, hr)
if not keys:
raise gr.Error("No Level 2 volumes for this selection.")
with ThreadPoolExecutor(max_workers=6) as ex:
list(ex.map(lambda k: _safe_download(bucket, k, VOL_CACHE_DIR), keys))
out = os.path.join(VOL_CACHE_DIR, f"{site}{date:%Y%m%d}_{hr:02d}UTC.zip")
if not os.path.exists(out):
tmp = out + ".part"
with zipfile.ZipFile(tmp, "w", zipfile.ZIP_STORED) as zf:
for k in keys:
p = os.path.join(VOL_CACHE_DIR, os.path.basename(k))
if os.path.exists(p):
zf.write(p, arcname=os.path.basename(k))
os.replace(tmp, out)
return out
# ----------------------------------------------------------------------------- ui
ILLINI = dict(orange="#FF5F05", blue="#13294B", blue2="#1D3866", blue3="#25457F",
storm_light="#C8C6C7", severe="#C84113")
ILLINI_THEME = gr.themes.Default(
primary_hue=gr.themes.Color(
c50="#FFF1E8", c100="#FFE1CC", c200="#FFC39A", c300="#FFA268",
c400="#FF8136", c500="#FF5F05", c600="#E25504", c700="#C84113",
c800="#9E3610", c900="#7A2A0C", c950="#571E08"),
font=[gr.themes.GoogleFont("Source Sans 3"), "system-ui", "sans-serif"],
).set(
body_background_fill=ILLINI["blue"],
body_background_fill_dark=ILLINI["blue"],
body_text_color="#FFFFFF", body_text_color_dark="#FFFFFF",
block_background_fill=ILLINI["blue2"],
block_background_fill_dark=ILLINI["blue2"],
border_color_primary=ILLINI["blue3"],
border_color_primary_dark=ILLINI["blue3"],
input_background_fill=ILLINI["blue"],
input_background_fill_dark=ILLINI["blue"],
button_primary_background_fill=ILLINI["orange"],
button_primary_background_fill_hover="#E25504",
button_primary_text_color="#FFFFFF",
)
ILLINI_CSS = """
@import url('https://fonts.googleapis.com/css2?family=Montserrat:wght@600;700;800&family=Source+Sans+3:wght@400;600;700&display=swap');
.gradio-container { background: #13294B !important; max-width: 100% !important;
padding: 0 !important;
font-family: 'Source Sans 3','Source Sans Pro',system-ui,Arial,sans-serif !important; }
/* the app wrapper () is centred at 1280px with 32px
padding by default — make the content span the full window with 2px edges */
.gradio-container > main, .gradio-container .fillable, .gradio-container main.app {
max-width: 100% !important; margin: 0 !important; padding: 2px !important; }
/* collapse all inter-element spacing so the map gets the window */
.gradio-container .gap, .gradio-container .wrap,
.gradio-container .contain { gap: 2px !important; }
.gradio-container .html-container { padding: 0 !important; margin: 0 !important; }
.gradio-container .block { margin: 0 !important; }
/* the map fills edge-to-edge */
#map-html, #map-html .html-container, #map-html .block { padding: 0 !important; }
.gradio-container h1 { font-family: 'Montserrat','Arial Black',sans-serif !important;
font-weight: 800 !important; text-transform: uppercase; letter-spacing: .06em;
font-size: 17px !important; margin: 0 !important;
color: #fff !important; border-bottom: 3px solid #FF5F05;
padding-bottom: 2px; display: inline-block; }
.gradio-container .prose a { color: #FF8136 !important; }
.gradio-container .prose, .gradio-container .prose p { color: #C8C6C7 !important;
font-size: 13px !important; }
.gradio-container .prose p { margin: 2px 0 !important; }
.gradio-container .block { padding: 2px 6px !important; }
.gradio-container .form { gap: 2px !important; }
.gradio-container .gap, .gradio-container .gradio-row { gap: 3px !important; }
/* compact one-row control strip */
#ctrl-row { flex-wrap: nowrap !important; }
.gradio-container .block > label,
.gradio-container span[data-testid="block-info"] {
font-size: 11px !important; margin-bottom: 1px !important; }
.gradio-container input, .gradio-container .gradio-dropdown input {
font-size: 13px !important; }
.gradio-container .gradio-dropdown { min-height: 0 !important; }
#mode-sw .wrap { flex-direction: row !important; gap: 4px !important;
flex-wrap: nowrap !important; }
#mode-sw label { padding: 0 11px !important; font-size: 13px !important;
height: 38px !important; display: flex !important;
align-items: center !important; box-sizing: border-box !important; }
#ctrl-row button { font-size: 13px !important; padding: 6px 8px !important; }
/* visually grey the time dropdowns when disabled (Live mode) — Gradio sets
the input disabled but doesn't dim it, so the lock wasn't obvious */
#ctrl-row .block:has(input:disabled) { opacity: .45 !important; }
#ctrl-row .block:has(input:disabled) input { cursor: not-allowed !important; }
@media (max-width: 900px) { #ctrl-row { flex-wrap: wrap !important; } }
footer { display: none !important; }
#dax-trigger, #rt-refresh, #rt-force, #site-go, #site-jump { display: none !important; }
/* Map panel: one consistent height across states; 100dvh so iOS Safari's
address-bar show/hide doesn't resize the radar mid-scroll. */
#map-html iframe {
height: calc(100vh - 170px) !important;
height: calc(100dvh - 170px) !important;
min-height: 460px !important; box-sizing: border-box !important; }
@media (max-width: 700px) {
.gradio-container { padding: 8px 10px 4px !important; }
.gradio-container h1 { font-size: 14px !important; letter-spacing: .03em !important; }
.gradio-container .prose, .gradio-container .prose p {
font-size: 10px !important; line-height: 1.3 !important; }
#ctrl-row .block > label,
#ctrl-row span[data-testid="block-info"] { font-size: 10px !important; }
#map-html iframe {
height: calc(100vh - 300px) !important;
height: calc(100dvh - 300px) !important;
min-height: 380px !important; }
}
"""
# CliMAS logo: use logo.png if present in the repo, else an inline SVG replica
if os.path.exists(os.path.join(os.path.dirname(__file__), "logo.png")):
_b64 = base64.b64encode(
open(os.path.join(os.path.dirname(__file__), "logo.png"), "rb").read()
).decode()
LOGO_HTML = (f'')
else:
LOGO_HTML = """
"""
HEADER_HTML = f"""
"""
# block-I favicon (Illini Orange I on Illini Blue), as an SVG data URI
_FAVICON_SVG = (
''
)
_FAVICON_B64 = base64.b64encode(_FAVICON_SVG.encode()).decode()
OG_HEAD = f"""
"""
gr.set_static_paths(paths=[VOL_CACHE_DIR]) # serves warnings.json
with gr.Blocks(title="NEXRAD Level 2 low level sweep browser", head=OG_HEAD,
theme=ILLINI_THEME, css=ILLINI_CSS) as demo:
gr.HTML(HEADER_HTML)
with gr.Row(elem_id="ctrl-row"):
mode_sw = gr.Radio(["Archive", "Live"], value="Archive",
label="Mode", scale=1, min_width=172,
elem_id="mode-sw")
site_tb = gr.Dropdown(SITE_CHOICES, value="KILX", label="Site",
allow_custom_value=True, scale=2, min_width=150)
# field is chosen via the in-map selector (Z/V/ZDR/CC/Z+V/2×2); this
# dropdown is kept hidden so handlers + share-link round-trips still work
field_dd = gr.Dropdown(list(FIELDS) + [QUAD, ZVF], value="Reflectivity",
label="Field", scale=2, min_width=140,
visible=False)
year_dd = gr.Dropdown(YEARS, value="2023", label="Year (UTC)",
scale=1, min_width=84)
month_dd = gr.Dropdown(MONTHS, value="06", label="Month (UTC)",
scale=1, min_width=72)
day_dd = gr.Dropdown(DAYS, value="29", label="Day (UTC)",
scale=1, min_width=72)
hour_dd = gr.Dropdown(HOURS, value="18:00", label="Hour (UTC)",
scale=1, min_width=84)
with gr.Column(scale=1, min_width=150):
go = gr.Button("Go", variant="primary")
dl = gr.DownloadButton("Download raw (.zip)",
interactive=False, size="sm",
visible=False)
# hidden trigger: the in-map "Dealias V" checkbox clicks this
# via parent.document when the hour hasn't been dealiased yet
dax = gr.Button("Dealias velocity", elem_id="dax-trigger",
size="sm")
# hidden: the live page clicks this every 5 minutes
rtr = gr.Button("Refresh live", elem_id="rt-refresh", size="sm")
# hidden: the in-map refresh button clicks this to force a live
# re-decode now (picks up new scans) without reloading the iframe
frr = gr.Button("Force live", elem_id="rt-force", size="sm")
# hidden: the in-map site picker writes "\t" here and
# clicks site-go to switch radars in place (with a progress bar)
site_jump = gr.Textbox(elem_id="site-jump")
sgo = gr.Button("Go site", elem_id="site-go", size="sm")
status = gr.Markdown()
map_html = gr.HTML(elem_id="map-html")
shared = gr.State(False)
view_st = gr.State(None)
deal_st = gr.State(False) # live dealiasing on for this session
def browse_h(mode, site, field, year, month, day, hour,
progress=gr.Progress()):
rt = (mode == "Live")
info, page = browse(site, field, year, month, day, hour,
progress=progress, realtime=rt)
return info, page, gr.DownloadButton(
interactive=bool(page) and not rt)
go.click(browse_h,
[mode_sw, site_tb, field_dd, year_dd, month_dd, day_dd, hour_dd],
[status, map_html, dl], show_progress_on=map_html)
def set_mode(mode, site, field, year, month, day, hour,
progress=gr.Progress()):
rt = (mode == "Live")
info, page = browse(site, field, year, month, day, hour,
progress=progress, realtime=rt)
dd = [gr.Dropdown(interactive=not rt) for _ in range(4)]
return (*dd, info, page,
gr.DownloadButton(interactive=bool(page) and not rt),
False) # reset live-dealias preference on mode change
mode_sw.change(set_mode,
[mode_sw, site_tb, field_dd, year_dd, month_dd, day_dd,
hour_dd],
[year_dd, month_dd, day_dd, hour_dd, status, map_html, dl,
deal_st],
show_progress_on=map_html)
def rt_refresh(site, field, progress=gr.Progress()):
info, page = browse(site, field, "", "", "", "",
progress=progress, realtime=True)
return info, page
rtr.click(rt_refresh, [site_tb, field_dd], [status, map_html],
show_progress_on=map_html)
dl.click(prepare_archive,
[site_tb, year_dd, month_dd, day_dd, hour_dd], dl)
def dealias_h(mode, site, field, year, month, day, hour,
progress=gr.Progress()):
if mode == "Live":
info, page = browse(site, "Radial velocity", year, month, day,
hour, progress=progress, realtime=True,
want_deal=True)
else:
info, page = browse(site, DEALIAS_NAME, year, month, day, hour,
progress=progress)
return info, page, (mode == "Live")
dax.click(dealias_h,
[mode_sw, site_tb, field_dd, year_dd, month_dd, day_dd,
hour_dd],
[status, map_html, deal_st], show_progress_on=map_html)
def site_jump_h(jump, mode, year, month, day, hour, progress=gr.Progress()):
# jump = "\t" from the in-map site picker; load that
# radar in place (live -> latest; archive -> same hour), keeping field.
parts = (jump or "").split("\t")
site = parts[0].strip()
field = parts[1] if len(parts) > 1 and parts[1] else "Reflectivity"
rt = (mode == "Live")
info, page = browse(site, field, year, month, day, hour,
progress=progress, realtime=rt)
return (info, page,
gr.DownloadButton(interactive=bool(page) and not rt),
gr.Dropdown(value=site))
sgo.click(site_jump_h,
[site_jump, mode_sw, year_dd, month_dd, day_dd, hour_dd],
[status, map_html, dl, site_tb], show_progress_on=map_html)
# server-side live refresh: a Timer tick every 2 minutes replaces the
# in-page hidden-button click (programmatic DOM clicks proved unreliable
# across browsers). Archive sessions skip at zero cost.
rt_timer = gr.Timer(60) # re-check the live feed for new chunks every minute
def rt_tick(mode, site, field, deal, progress=gr.Progress()):
# Only refresh the live frame file; the in-page poller merges the new
# frames in place when ready. Crucially this has NO Gradio outputs, so
# the display is never marked "generating"/greyed during an autoupdate.
if mode == "Live":
try:
browse(site, "Radial velocity" if deal else field,
"", "", "", "", progress=progress,
realtime=True, want_deal=deal)
except Exception:
pass
rt_timer.tick(rt_tick, [mode_sw, site_tb, field_dd, deal_st],
None, show_progress="hidden")
def force_refresh(mode, site, field, deal, progress=gr.Progress()):
# user hit the in-map refresh: drop the short-lived RT cache so we
# re-check the feed for new scans now and rewrite live_.json.
# NO Gradio outputs -> the display is never greyed; the in-page poller
# (with its own spinning icon) merges the result when ready.
if mode == "Live":
try:
_RT_CACHE.pop(site, None)
browse(site, "Radial velocity" if deal else field,
"", "", "", "", progress=progress,
realtime=True, want_deal=deal)
except Exception:
pass
frr.click(force_refresh, [mode_sw, site_tb, field_dd, deal_st], None,
show_progress="hidden")
gr.Markdown(
"Level 2 decoding by [xradar](https://github.com/swnesbitt/xradar) "
"(openradar; S. Nesbitt fork) "
"· velocity dealiasing by the Rust region-based dealiaser "
"[region-dealias](https://github.com/swnesbitt/region-dealias) (S. Nesbitt) "
"· colormaps from [cmweather](https://github.com/openradar/cmweather) "
"and [CMasher](https://cmasher.readthedocs.io) "
"· data from the [NOAA NEXRAD Level II archive on AWS](https://registry.opendata.aws/noaa-nexrad/) "
"· [Source code on GitHub](https://github.com/swnesbitt/nexrad-level2-browser)."
)
def init_values(request: gr.Request):
"""Fast: restore dropdown values (and map view) from URL params."""
q = dict(request.query_params) if request else {}
deal = q.get("dealias") == "1"
rt = q.get("rt") == "1"
view = None
try:
if "lat" in q and "lon" in q and "zoom" in q:
view = [float(q["lat"]), float(q["lon"]),
max(3, min(14, int(float(q["zoom"]))))]
except (TypeError, ValueError):
view = None
site = q.get("site", "KILX").upper()[:4]
field = q.get("field", "Reflectivity")
if field not in FIELDS and field not in (QUAD, ZVF, DEALIAS_NAME):
field = "Reflectivity"
year = q.get("year", "2023")
month = q.get("month", "06").zfill(2)
day = q.get("day", "29").zfill(2)
hour = q.get("hour", "18")[:2].zfill(2) + ":00"
year = year if year in YEARS else "2023"
month = month if month in MONTHS else "06"
day = day if day in DAYS else "29"
hour = hour if hour in HOURS else "18:00"
arch = not rt
return (site, field,
gr.Dropdown(value=year, interactive=arch),
gr.Dropdown(value=month, interactive=arch),
gr.Dropdown(value=day, interactive=arch),
gr.Dropdown(value=hour, interactive=arch),
("site" in q or "year" in q), view, deal,
"Live" if rt else "Archive")
def maybe_browse(mode, view, deal, site, field, year, month, day,
hour, progress=gr.Progress()):
"""Slow: auto-load on every visit — the default case on a plain
visit, the shared view (incl. map center/zoom) when params exist."""
rt = (mode == "Live")
info, page = browse(site, field, year, month, day, hour,
progress=progress, view=view, want_deal=deal,
realtime=rt)
return info, page, gr.DownloadButton(
interactive=bool(page) and not rt)
demo.load(
init_values, None,
[site_tb, field_dd, year_dd, month_dd, day_dd, hour_dd, shared,
view_st, deal_st, mode_sw],
).then(
maybe_browse,
[mode_sw, view_st, deal_st, site_tb, field_dd, year_dd, month_dd,
day_dd, hour_dd],
[status, map_html, dl], show_progress_on=map_html,
)
def _live_poller():
"""Keep live_.json current for recently-viewed live sites regardless
of any browser tab (client timers freeze when a tab is backgrounded). Calls
_realtime_compute directly so it does NOT refresh the activity timestamp —
sites age out _LIVE_ACTIVE_TTL after the last real view."""
noop = lambda *a, **k: None
while True:
try:
now = time_mod.time()
active = [(s, d) for s, (ts, d) in list(_LIVE_ACTIVE.items())
if now - ts < _LIVE_ACTIVE_TTL]
for s, deal in active:
try:
_realtime_compute(s, "Reflectivity", noop, None, deal)
except Exception:
pass
except Exception:
pass
time_mod.sleep(60)
def start_background():
"""Start the startup pre-render + the warnings/live poller daemons. Kept
out of module import so that (a) ProcessPoolExecutor workers, which re-import
this module under the macOS 'spawn' start method, don't relaunch the daemons
(or recurse), and (b) the desktop launcher can start them explicitly."""
# pre-render the default case at startup so first visitors get it instantly
threading.Thread(target=lambda: browse(*DEFAULT_VIEW), daemon=True).start()
# keep live warnings fresh for real-time mode
threading.Thread(target=_warn_poller, daemon=True).start()
# keep live radar frames fresh for recently-viewed sites (server-side, so it
# doesn't depend on a client tab driving the refresh timer)
threading.Thread(target=_live_poller, daemon=True).start()
if __name__ == "__main__":
start_background()
# ssr_mode=False -> Python serves the OG-patched index.html, so social
# link previews show our title/thumbnail instead of Gradio defaults
demo.launch(ssr_mode=False)