Spaces:
Running
Running
| """Gradio app: compare estimators of a smooth level on sparse, gappy air-quality data. | |
| Reads the tables written by dataset/precompute.py: from DATA_DIR if set, else from an outputs/ folder | |
| next to the app (on the Space) or at the repo root (run from the repo), otherwise from the Hugging Face dataset | |
| HF_DATASET. | |
| """ | |
| import json | |
| import os | |
| from pathlib import Path | |
| import gradio as gr | |
| import pandas as pd | |
| import plotly.graph_objects as go | |
| import plotly.io as pio | |
| from plotly.subplots import make_subplots | |
| pio.templates.default = "ggplot2" | |
| HF_DATASET = "prteek/sparse-panel-demo" | |
| BLOG_URL = "https://huggingface.co/blog/prteek/sparse-panel-modelling" | |
| LAMBDA_NAMES = {0.5: "Sparse: ~0.5 hours a day", 1.2: "As in the blog post: ~1.2 hours a day", 3.0: "Rich: ~3 hours a day"} | |
| LABEL_NOTES = { | |
| "regular": "Thinned at the global setting, with two 20–60 day gaps per pollutant.", | |
| "sparse": "Thinned to 40% of the global setting's hours.", | |
| "very sparse": "As regular, then only 15% of the observed days kept.", | |
| "long gap": "As regular, but the two gaps are 90–120 days long.", | |
| "dropped series": "One pollutant removed entirely; each panel shows the dropped one. " | |
| "The joint network can rebuild it from the station's other pollutant.", | |
| } | |
| STYLE = { | |
| "joint": dict(color="#1f77b4", width=2), | |
| "single-target": dict(color="#9467bd", width=1.5, dash="dash"), | |
| "±15-day mean": dict(color="#ff7f0e", width=1.2), | |
| } | |
| DAY_TYPES = ["all", "observed", "unobserved", "gap"] | |
| TRUTH_COLOR = "#2ca02c" # visible on light and dark themes | |
| def load() -> dict: | |
| here = Path(__file__).resolve().parent | |
| candidates = ([Path(os.environ["DATA_DIR"])] if "DATA_DIR" in os.environ | |
| else [here / "outputs", here.parent / "outputs"]) | |
| root = next((c for c in candidates if c.exists()), None) | |
| if root is None: | |
| from huggingface_hub import snapshot_download | |
| root = Path(snapshot_download(HF_DATASET, repo_type="dataset", allow_patterns=["*.parquet", "meta.json"])) | |
| tables = {f.stem: pd.read_parquet(f) for f in root.glob("*.parquet")} | |
| tables["meta"] = json.loads((root / "meta.json").read_text()) | |
| return tables | |
| def gap_spans(dates: pd.Series): | |
| d = dates.sort_values() | |
| block = (d.diff().dt.days != 1).cumsum() | |
| return d.groupby(block).agg(["min", "max"]).itertuples(index=False) | |
| def figure(t: dict, lam: float, rows: pd.DataFrame) -> go.Figure: | |
| titles = [f"{r.station} · {r.pollutant} ({r.area})" for r in rows.itertuples()] | |
| fig = make_subplots(rows=len(rows), cols=1, shared_xaxes=True, subplot_titles=titles, vertical_spacing=0.06) | |
| readings = t["readings"][t["readings"]["lam"] == lam] | |
| preds = t["predictions"][t["predictions"]["lam"] == lam] | |
| for i, r in enumerate(rows.itertuples(), 1): | |
| sel = lambda d: d[(d["station"] == r.station) & (d["pollutant"] == r.pollutant)] | |
| first = i == 1 | |
| s = sel(readings) | |
| fig.add_trace(go.Scatter(x=s["date"], y=s["value"], mode="markers", name="sparse readings", | |
| marker=dict(size=3 + 1.5 * s["n"], color="rgba(128,128,128,0.45)"), | |
| legendgroup="readings", showlegend=first), row=i, col=1) | |
| tr = sel(t["truth"]) | |
| fig.add_trace(go.Scatter(x=tr["date"], y=tr["value"], mode="lines", name="31-day MA (truth)", | |
| line=dict(color=TRUTH_COLOR, width=2.5), legendgroup="truth", showlegend=first), | |
| row=i, col=1) | |
| for name, style in STYLE.items(): | |
| p = sel(preds[preds["estimator"] == name]) | |
| fig.add_trace(go.Scatter(x=p["date"], y=p["pred"], mode="lines", name=name, line=style, | |
| legendgroup=name, showlegend=first), row=i, col=1) | |
| # after the traces: plotly skips shapes on subplots that have no traces yet | |
| for a, b in gap_spans(sel(t["gaps"])["date"]): | |
| fig.add_vrect(x0=a, x1=b, fillcolor="gold", opacity=0.25, line_width=0, layer="below", row=i, col=1) | |
| fig.update_yaxes(type="log", title_text="µg/m³", row=i, col=1) | |
| height, top = 260 * len(rows) + 110, 100 | |
| fig.update_layout(height=height, margin=dict(l=10, r=10, t=top, b=10), hovermode="x unified", | |
| # legend sits in the top margin, clear of the first subplot title | |
| legend=dict(orientation="h", y=1 + 45 / (height - top - 10), yanchor="bottom", x=0)) | |
| return fig | |
| def error_table(m: pd.DataFrame) -> pd.DataFrame: | |
| tab = m.pivot_table(index="estimator", columns="day_type", values="error") | |
| days = m.groupby("day_type")["days"].max() | |
| cols = [c for c in DAY_TYPES if c in tab] | |
| tab = tab[cols].reindex(list(STYLE)).map(lambda v: f"{v:.3f}") | |
| tab.loc["days"] = [f"{days[c]:,}" for c in cols] # counts as a row keeps the headers narrow | |
| return tab.reset_index() | |
| def trend_figure(m: pd.DataFrame) -> go.Figure: | |
| trend = m[m["day_type"] == "all"].pivot_table(index="lam", columns="estimator", values="error")[list(STYLE)] | |
| fig = go.Figure([go.Scatter(x=trend.index, y=trend[c], mode="lines+markers", name=c, | |
| line={k: v for k, v in STYLE[c].items() if k != "width"}) for c in trend]) | |
| fig.update_layout(height=300, margin=dict(l=10, r=10, t=10, b=10), xaxis_title="mean hours kept per day", | |
| yaxis_title="error", legend=dict(orientation="h", y=-0.3)) | |
| return fig | |
| T = load() | |
| LAMBDAS = T["meta"]["lambdas"] | |
| def view(lam: float, label: str, pollutant: str): | |
| group = T["labels"][T["labels"]["label"] == label] | |
| if label == "dropped series": | |
| pollutant = "dropped" | |
| rows = group.assign(pollutant=group["dropped"]) | |
| else: | |
| rows = group.assign(pollutant=pollutant) | |
| m = T["metrics"][(T["metrics"]["label"] == label) & (T["metrics"]["pollutant"] == pollutant)] | |
| note = f"### {label.capitalize()} stations\n{LABEL_NOTES[label]} Shaded: gap blocks." | |
| return (note, figure(T, lam, rows), error_table(m[m["lam"] == lam]), trend_figure(m), | |
| gr.update(visible=label != "dropped series")) | |
| with gr.Blocks(title="Sparse panel demo") as demo: | |
| gr.Markdown( | |
| "# Smooth levels from sparse, gappy data\n" | |
| "Hourly PM2.5 and PM10 from 12 Beijing monitoring stations are thinned to a few readings a day, with gaps, " | |
| "and three estimators try to recover the slow underlying level: a **joint network** that models both " | |
| "pollutants together, the same network fitted to **one pollutant at a time**, and a **±15-day mean** of the " | |
| f"readings. The truth is the 31-day moving average of the full data. Method and discussion: [blog post]({BLOG_URL})." | |
| ) | |
| with gr.Sidebar(): | |
| gr.Markdown("## Scenario") | |
| lam = gr.Radio([(LAMBDA_NAMES.get(l, str(l)), l) for l in LAMBDAS], value=1.2, | |
| label="Readings kept (Poisson mean hours per station-day)") | |
| gr.Markdown("## View") | |
| label = gr.Dropdown(list(LABEL_NOTES), value="regular", label="Station group") | |
| pollutant = gr.Radio(["PM2.5", "PM10"], value="PM2.5", label="Pollutant") | |
| note = gr.Markdown() | |
| plot = gr.Plot(show_label=False) | |
| with gr.Row(): | |
| with gr.Column(scale=3): | |
| gr.Markdown("**Error against the 31-day MA** (mean absolute log error; 0.15 ≈ 15%)") | |
| table = gr.Dataframe(interactive=False, show_label=False) | |
| gr.Markdown("observed: days with a reading · unobserved: no reading, outside the gaps · " | |
| "gap: inside a gap block") | |
| with gr.Column(scale=2): | |
| gr.Markdown("**All days, by thinning level**") | |
| trend = gr.Plot(show_label=False) | |
| gr.Markdown("Data: Beijing Multi-Site Air Quality, UCI Machine Learning Repository (CC BY 4.0). " | |
| "Labels are assigned once with a fixed seed: each degraded group has one urban and one suburban " | |
| "station.") | |
| outputs = [note, plot, table, trend, pollutant] | |
| for control in (lam, label, pollutant): | |
| control.change(view, [lam, label, pollutant], outputs) | |
| demo.load(view, [lam, label, pollutant], outputs) | |
| if __name__ == "__main__": | |
| demo.launch() | |