Gridlock / app /frontend /src /pages /Models.tsx
parvmittal07's picture
Improved Map UI and minor fixes
2c64d54
Raw
History Blame Contribute Delete
21 kB
import { useEffect, useState } from "react";
import { Activity, ShieldAlert, Clock, Flame } from "lucide-react";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { Badge } from "@/components/ui/badge";
import { InfoTip } from "@/components/ui/popover";
import { LoadingState, ErrorState } from "@/components/ui/feedback";
import {
MetricStat,
ConfusionMatrix,
HBarChart,
OperatingPointsChart,
ChartCard,
} from "@/components/charts";
import { api } from "@/lib/api";
import type {
MetricsData,
TaskMetrics,
DurationMetrics,
HotspotMetrics,
} from "@/lib/types";
import { pct, num, minutesToHuman, titleCase } from "@/lib/format";
function FigureImage({ name, alt }: { name: string; alt: string }) {
const [ok, setOk] = useState(true);
if (!ok) return null;
return (
<img
src={api.figureUrl(name)}
alt={alt}
loading="lazy"
onError={() => setOk(false)}
className="w-full rounded-md border border-border bg-white"
/>
);
}
function Hero({
headline,
unit,
caption,
badge,
}: {
headline: string;
unit?: string;
caption: string;
badge?: { text: string; variant: "success" | "warning" | "danger" | "accent" };
}) {
return (
<div className="flex flex-col justify-between gap-3 rounded-lg border border-border bg-card p-5 sm:flex-row sm:items-center">
<div>
<div className="flex items-baseline gap-2">
<span className="text-4xl font-bold tracking-tight text-primary">
{headline}
</span>
{unit && <span className="text-sm text-muted-foreground">{unit}</span>}
</div>
<p className="mt-1 max-w-xl text-sm text-muted-foreground">{caption}</p>
</div>
{badge && <Badge variant={badge.variant} className="self-start text-sm">{badge.text}</Badge>}
</div>
);
}
/* ----------------------------- Priority ----------------------------- */
function PriorityPanel({ m }: { m: TaskMetrics }) {
return (
<div className="space-y-5">
<Hero
headline={m.average_precision.toFixed(3)}
unit="PR-AUC"
caption="The saturated, genuinely-easy task: priority is near-deterministic from location (is this on a designated priority corridor?), which is legitimately knowable at report time. High scores reflect honest geofencing, not leakage."
badge={{ text: "Best task", variant: "success" }}
/>
<div className="grid grid-cols-2 gap-3 md:grid-cols-4">
<MetricStat label="ROC-AUC" value={m.roc_auc.toFixed(3)} accent="text-accent" />
<MetricStat label="F1" value={m.f1.toFixed(3)} />
<MetricStat
label="MCC"
value={m.mcc.toFixed(3)}
accent="text-success"
info={{
title: "Why MCC?",
body: "Matthews Correlation balances all four confusion cells in one number that stays honest under class imbalanceunlike accuracy, which a trivial classifier can game. 0.90 here is near-perfect agreement.",
}}
/>
<MetricStat label="Brier" value={m.brier.toFixed(3)} info={{ title: "Brier score", body: "Mean squared error of the predicted probabilities. Lower = better calibrated. 0.04 means the probabilities are trustworthy, not just the rankings." }} />
<MetricStat label="Precision" value={m.precision.toFixed(3)} />
<MetricStat label="Recall" value={m.recall.toFixed(3)} />
<MetricStat label="Balanced acc." value={m.balanced_accuracy.toFixed(3)} />
<MetricStat label="Base rate" value={pct(m.positive_rate)} sub="High-priority share" />
</div>
<div className="grid grid-cols-1 gap-5 lg:grid-cols-2">
<ChartCard
title="Confusion at deployed threshold"
info="Counts on the chronological test set (n=1,611) at the deployed decision threshold."
>
<ConfusionMatrix {...m.confusion} positiveLabel="High" negativeLabel="Low" />
</ChartCard>
{m.oof && (
<ChartCard
title="Base learners → stack (OOF AP)"
info="Three decorrelated gradient-boosting models are combined by a logistic meta-learner on out-of-fold predictions. The stack matches or beats every individual base model."
>
<HBarChart
data={[
{ name: "LightGBM", value: m.oof.base_oof_ap.lightgbm },
{ name: "XGBoost", value: m.oof.base_oof_ap.xgboost },
{ name: "CatBoost", value: m.oof.base_oof_ap.catboost },
{ name: "Stack", value: m.oof.oof_ap },
]}
domain={[0.95, 1]}
highlightLast
/>
</ChartCard>
)}
</div>
<div className="grid grid-cols-1 gap-5 lg:grid-cols-2">
<ChartCard title="Precision–recall & calibration">
<FigureImage name="priority_pr_calibration.png" alt="Priority PR and calibration curves" />
</ChartCard>
<ChartCard title="SHAP feature impact" info="Which features move the priority prediction most, and in which direction.">
<FigureImage name="priority_shap_summary.png" alt="Priority SHAP summary" />
</ChartCard>
</div>
</div>
);
}
/* ----------------------------- Closure ----------------------------- */
function ClosurePanel({ m }: { m: TaskMetrics }) {
const ops = m.operating_points;
const opData = [
{ key: "mcc_optimal", name: "Balanced" },
{ key: "f2_optimal", name: "Deployed (F2)" },
{ key: "recall>=0.8", name: "Never-miss" },
]
.filter((o) => ops[o.key])
.map((o) => ({
name: o.name,
recall: ops[o.key].recall,
precision: ops[o.key].precision,
mcc: ops[o.key].mcc,
}));
return (
<div className="space-y-5">
<Hero
headline={m.average_precision.toFixed(3)}
unit="PR-AUC"
caption="The genuinely hard operational task. End-point/route geometry — the strongest raw predictor — was removed as leakage, so the model must forecast from real report-time signal."
badge={{ text: `${m.ap_lift_over_base.toFixed(1)}× base rate`, variant: "accent" }}
/>
<div className="grid grid-cols-2 gap-3 md:grid-cols-4">
<MetricStat
label="PR-AUC lift"
value={`${m.ap_lift_over_base.toFixed(1)}×`}
accent="text-primary"
info={{
title: "Why 0.326 is strong",
body: `With only ${pct(m.positive_rate)} of events needing a closure, a random model scores ~0.072 PR-AUC. 0.326 is ${m.ap_lift_over_base.toFixed(1)}× that base ratefor a rare, partly-discretionary event with the leakage features removed, this is substantial real signal.`,
}}
/>
<MetricStat label="ROC-AUC" value={m.roc_auc.toFixed(3)} accent="text-accent" />
<MetricStat
label="MCC"
value={m.mcc.toFixed(3)}
info={{ title: "Why MCC?", body: "Under 7% positives, accuracy is meaningless (predict 'never' → 93% accurate, 0 closures caught). MCC and PR-AUC reward actually finding the rare closures." }}
/>
<MetricStat label="Brier" value={m.brier.toFixed(3)} sub="well calibrated" />
<MetricStat label="Recall" value={pct(m.recall)} accent="text-success" sub="of real closures caught" />
<MetricStat label="Precision" value={pct(m.precision)} sub="at deployed threshold" />
<MetricStat
label="F2"
value={m.f_beta.toFixed(3)}
info={{ title: "Why F2 (recall-leaning)?", body: "Missing a real closure (no barricade planned) is far costlier than a false alarm (an officer briefly on stand-by). F2 weights recall 2× precision, so the deployed threshold favours catching closures." }}
/>
<MetricStat label="Base rate" value={pct(m.positive_rate)} sub="closure share" />
</div>
<div className="grid grid-cols-1 gap-5 lg:grid-cols-2">
<ChartCard
title="The threshold is a policy choice"
info="The same calibrated model, scored at three operating points. Slide toward recall to never miss a closure, or toward balance for the best MCC. The demo deploys the F2 (recall-leaning) point."
legend={[
{ color: "hsl(var(--primary))", label: "Recall" },
{ color: "hsl(var(--accent))", label: "Precision" },
{ color: "hsl(var(--warning))", label: "MCC" },
]}
>
<OperatingPointsChart data={opData} />
</ChartCard>
<ChartCard title="Confusion at deployed threshold" info={`Deployed F2 threshold ${m.threshold.toFixed(2)} on the test set (n=${m.n.toLocaleString()}).`}>
<ConfusionMatrix {...m.confusion} positiveLabel="Closure" negativeLabel="None" />
</ChartCard>
</div>
<div className="grid grid-cols-1 gap-5 lg:grid-cols-2">
{m.oof && (
<ChartCard title="Base learners → stack (OOF AP)" info="Out-of-fold average precision of each base model and the stacked ensemble.">
<HBarChart
data={[
{ name: "LightGBM", value: m.oof.base_oof_ap.lightgbm },
{ name: "XGBoost", value: m.oof.base_oof_ap.xgboost },
{ name: "CatBoost", value: m.oof.base_oof_ap.catboost },
{ name: "Stack", value: m.oof.oof_ap },
]}
domain={[0.35, 0.42]}
highlightLast
/>
</ChartCard>
)}
<ChartCard title="Precision–recall & calibration">
<FigureImage name="closure_pr_calibration.png" alt="Closure PR and calibration curves" />
</ChartCard>
</div>
<ChartCard title="SHAP feature impact" info="Causal target-rate features (past-only closure rates per corridor/junction) and text embeddings carry most of the signal.">
<div className="mx-auto max-w-2xl">
<FigureImage name="closure_shap_summary.png" alt="Closure SHAP summary" />
</div>
</ChartCard>
</div>
);
}
/* ----------------------------- Duration ----------------------------- */
function DurationPanel({ m }: { m: DurationMetrics }) {
return (
<div className="space-y-5">
<Hero
headline={m.r2_log.toFixed(3)}
unit="log-scale R²"
caption="Minutes from report to clearance — a heavy-tailed target spanning minutes to multi-week construction. The honest fit is measured on the log scale; the 74-minute median error is what a control room feels on normal incidents."
badge={{ text: `median error ${num(m.median_ae_min)} min`, variant: "warning" }}
/>
<div className="grid grid-cols-2 gap-3 md:grid-cols-4">
<MetricStat
label="Median abs. error"
value={`${num(m.median_ae_min)} min`}
accent="text-warning"
info={{ title: "Typical incident error", body: "Half of all events are predicted within this many minutes. Robust to the multi-week outliers that distort the mean." }}
/>
<MetricStat label="MAE (log)" value={m.mae_log.toFixed(2)} sub="central-tendency error" />
<MetricStat label="MAPE" value={`${num(m.mape)}%`} />
<MetricStat
label="R² (raw min)"
value={m.r2.toFixed(3)}
info={{ title: "Why two R² numbers?", body: "Raw-minute R² (0.09) is dominated by a handful of days-to-weeks construction events. The log-scale R² (0.25) is the honest measure of central-tendency fit on normal incidents." }}
/>
<MetricStat
label="80% interval coverage"
value={pct(m.interval_coverage_80)}
accent="text-success"
info={{ title: "Calibrated uncertainty", body: "The conformalised 80% prediction interval actually contains the true clearance time ~78% of the timeclose to the nominal 80%. You get an honest band, not just a point." }}
/>
<MetricStat label="Median interval width" value={`${num(m.interval_width_med_min)} min`} sub="actionable band" />
<MetricStat label="Test events" value={m.n.toLocaleString()} sub={`train ${m.n_train.toLocaleString()}`} />
<MetricStat label="RMSE (raw)" value={`${num(m.rmse_min)} min`} sub="inflated by tail" />
</div>
<div className="grid grid-cols-1 gap-5 lg:grid-cols-2">
<ChartCard
title="Why the log scale"
info="The target spans 4+ orders of magnitude. Modelling log1p(minutes) stops a few multi-week events from dominating the loss, and is why log-R² (0.25) is the metric to trust over raw-minute R² (0.09)."
>
<div className="flex h-[220px] flex-col justify-center gap-4">
<ScaleBar label="Log-scale R² (honest)" value={m.r2_log} max={0.4} color="hsl(var(--primary))" />
<ScaleBar label="Raw-minute R² (tail-dominated)" value={m.r2} max={0.4} color="hsl(var(--muted-foreground))" />
<ScaleBar label="80% interval coverage" value={m.interval_coverage_80} max={1} color="hsl(var(--success))" reference={0.8} />
</div>
</ChartCard>
<ChartCard
title="Prediction interval"
info="Quantile models (p10/p50/p90) with a conformal correction produce an 80% interval around every point estimate."
>
<div className="flex h-[220px] flex-col items-center justify-center gap-3">
<div className="text-center">
<div className="text-5xl font-bold text-warning">
{minutesToHuman(m.median_ae_min)}
</div>
<div className="mt-1 text-sm text-muted-foreground">median absolute error</div>
</div>
<div className="w-full max-w-xs">
<div className="mb-1 flex justify-between text-[11px] text-muted-foreground">
<span>p10</span>
<span className="font-medium text-foreground">
median width {num(m.interval_width_med_min)} min
</span>
<span>p90</span>
</div>
<div className="h-2 w-full rounded-full bg-warning/25" />
</div>
</div>
</ChartCard>
</div>
</div>
);
}
function ScaleBar({
label,
value,
max,
color,
reference,
}: {
label: string;
value: number;
max: number;
color: string;
reference?: number;
}) {
return (
<div>
<div className="mb-1 flex justify-between text-xs">
<span className="text-muted-foreground">{label}</span>
<span className="font-semibold tabular-nums text-foreground">{value.toFixed(3)}</span>
</div>
<div className="relative h-2.5 w-full rounded-full bg-secondary">
<div
className="h-full rounded-full"
style={{ width: `${Math.min((value / max) * 100, 100)}%`, backgroundColor: color }}
/>
{reference != null && (
<div
className="absolute top-1/2 h-3.5 w-0.5 -translate-y-1/2 bg-foreground"
style={{ left: `${(reference / max) * 100}%` }}
title={`nominal ${reference}`}
/>
)}
</div>
</div>
);
}
/* ----------------------------- Hotspot ----------------------------- */
function HotspotPanel({ m }: { m: HotspotMetrics }) {
const t = m.test;
const ops = m.test_operating_points;
const opData = [
{ key: "mcc_optimal", name: "Balanced" },
{ key: "f2_optimal", name: "Deployed (F2)" },
]
.filter((o) => ops[o.key])
.map((o) => ({
name: o.name,
recall: ops[o.key].recall,
precision: ops[o.key].precision,
mcc: ops[o.key].mcc,
}));
const topFeatures = Object.entries(m.top_features)
.slice(0, 12)
.map(([name, value]) => ({ name: name, value }))
.reverse();
return (
<div className="space-y-5">
<Hero
headline={t.average_precision.toFixed(3)}
unit="PR-AUC"
caption="A new forward-looking target engineered from scratch: will this ~110 m spot generate ≥2 more events in the next 14 days? An early warning to send a root-cause fix instead of firefighting."
badge={{ text: `recall ${pct(t.recall)}`, variant: "success" }}
/>
<div className="grid grid-cols-2 gap-3 md:grid-cols-4">
<MetricStat
label="PR-AUC lift"
value={`${(t.average_precision / t.base_rate).toFixed(1)}×`}
accent="text-primary"
info={{ title: "vs base rate", body: `With a ${pct(t.base_rate)} base rate, 0.441 PR-AUC is ~2.8× better than randomstrong for a rare, strictly forward-looking target.` }}
/>
<MetricStat label="ROC-AUC" value={t.roc_auc.toFixed(3)} accent="text-accent" />
<MetricStat
label="Recall"
value={pct(t.recall)}
accent="text-success"
info={{ title: "Catch emerging hotspots", body: `At the deployed threshold the model catches ${t.confusion.tp} of ${t.confusion.tp + t.confusion.fn} emerging hotspotsonly ${t.confusion.fn} missed. Early warning favours recall.` }}
/>
<MetricStat label="Precision" value={pct(t.precision)} sub="early-warning favours recall" />
<MetricStat label="MCC" value={t.mcc.toFixed(3)} />
<MetricStat label="Brier" value={t.brier.toFixed(3)} />
<MetricStat label="Base rate" value={pct(t.base_rate)} sub="chronic share" />
<MetricStat
label="Cold-start"
value={pct(m.cold_start_test.base_rate)}
sub={`${m.cold_start_test.n_pos}/${m.cold_start_test.n} new sites`}
info={{ title: "Honest limitation", body: "Brand-new locations (no prior history) almost never turn chronic, and the model correctly assigns them low risk rather than inventing signal." }}
/>
</div>
<div className="grid grid-cols-1 gap-5 lg:grid-cols-2">
<ChartCard title="Confusion at deployed threshold" info={`Deployed F2 threshold ${t.threshold.toFixed(3)} on the test set (n=${t.n.toLocaleString()}).`}>
<ConfusionMatrix {...t.confusion} positiveLabel="Chronic" negativeLabel="Stable" />
</ChartCard>
<ChartCard
title="Operating points"
info="Slide from the recall-favouring deployed point to a balanced MCC-optimal point."
legend={[
{ color: "hsl(var(--primary))", label: "Recall" },
{ color: "hsl(var(--accent))", label: "Precision" },
{ color: "hsl(var(--warning))", label: "MCC" },
]}
>
<OperatingPointsChart data={opData} />
</ChartCard>
</div>
<ChartCard
title="Top features (LightGBM gain)"
info="Recency counts per junction / zone / police-station dominate — recurring locations are the signal. Every feature is strictly past-only (causal)."
>
<HBarChart
data={topFeatures}
color="hsl(var(--primary))"
suffix=""
height={320}
/>
</ChartCard>
</div>
);
}
/* ----------------------------- Page ----------------------------- */
export default function Models() {
const [metrics, setMetrics] = useState<MetricsData | null>(null);
const [error, setError] = useState<string | null>(null);
useEffect(() => {
api.metrics().then(setMetrics).catch((e) => setError((e as Error).message));
}, []);
if (error) return <ErrorState message={error} />;
if (!metrics) return <LoadingState label="Loading model metrics…" />;
return (
<div>
<div className="mb-5">
<h1 className="text-xl font-bold tracking-tight">Model report</h1>
<p className="mt-1 text-sm text-muted-foreground">
Four leakage-controlled targets on a chronological hold-out (train on
the past, test on the future). {metrics.dataset.n_events_scored.toLocaleString()} events ·{" "}
{metrics.dataset.date_span}. Tap any{" "}
<span className="font-medium text-foreground"></span> for the “why”.
</p>
</div>
<Tabs defaultValue="priority">
<TabsList className="flex-wrap">
<TabsTrigger value="priority">
<Activity className="h-4 w-4" /> Priority
</TabsTrigger>
<TabsTrigger value="closure">
<ShieldAlert className="h-4 w-4" /> Closure
</TabsTrigger>
<TabsTrigger value="duration">
<Clock className="h-4 w-4" /> Duration
</TabsTrigger>
<TabsTrigger value="hotspot">
<Flame className="h-4 w-4" /> Hotspot
</TabsTrigger>
</TabsList>
<TabsContent value="priority">
<PriorityPanel m={metrics.priority} />
</TabsContent>
<TabsContent value="closure">
<ClosurePanel m={metrics.closure} />
</TabsContent>
<TabsContent value="duration">
<DurationPanel m={metrics.duration} />
</TabsContent>
<TabsContent value="hotspot">
<HotspotPanel m={metrics.hotspot} />
</TabsContent>
</Tabs>
</div>
);
}