"use client"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useTranslations } from "next-intl"; import Link from "next/link"; import Card from "@/shared/components/Card"; import ProviderIcon from "@/shared/components/ProviderIcon"; import ModelCooldownsCard from "./components/ModelCooldownsCard"; import { useProviderNodeMap, resolveProviderName } from "@/lib/display/useProviderNodeMap"; type KnownBreakerState = "CLOSED" | "OPEN" | "HALF_OPEN" | "DEGRADED"; type BreakerState = KnownBreakerState | (string & {}); type ProviderBreaker = { provider: string; state: BreakerState; failureCount: number; lastFailure: string | null; retryAfterMs: number; }; type LockoutEntry = { reason?: string; until?: number | string | null; remainingMs?: number; model?: string; accountId?: string; }; type SessionTop = { sessionId: string; requestCount: number; connectionId?: string | null; ageMs: number; idleMs: number; createdAt?: string; lastActiveAt?: string; }; type QuotaMonitor = { sessionId?: string; accountId?: string; provider?: string; window?: string; status?: "ok" | "alerting" | "exhausted" | "error" | string; remainingPercent?: number; }; type HealthPayload = { timestamp?: string; providerBreakers?: ProviderBreaker[]; lockouts?: Record; quotaMonitor?: { active?: number; alerting?: number; exhausted?: number; errors?: number; monitors?: QuotaMonitor[]; }; sessions?: { activeCount?: number; stickyBoundCount?: number; byApiKey?: Record; top?: SessionTop[]; }; }; type Connection = { id: string; provider: string; name?: string; displayName?: string; email?: string; authType?: string; rateLimitedUntil?: string | null; testStatus?: string; lastError?: string; lastErrorType?: string; errorCode?: string | number; backoffLevel?: number; }; type FeedEventKind = | "circuit-opened" | "circuit-degraded" | "circuit-recovered" | "circuit-closed" | "cooldown-added" | "cooldown-cleared" | "lockout-added" | "lockout-cleared" | "session-new" | "quota-alert" | "quota-exhausted" | "quota-recovered"; type FeedEvent = { id: string; ts: number; kind: FeedEventKind; title: string; detail: string; }; type FeedFilter = "all" | "circuits" | "cooldowns" | "lockouts" | "sessions" | "quotas"; // ───────────────────────────────────────────────────────────────────────────── // Constants & helpers // ───────────────────────────────────────────────────────────────────────────── const REFRESH_INTERVAL_MS = 5000; const FEED_MAX_EVENTS = 50; const EMPTY_PROVIDER_BREAKERS: ProviderBreaker[] = []; type BreakerTone = { dot: string; bg: string; ring: string; label: string; icon: string }; const BREAKER_TONE: Record = { CLOSED: { dot: "#22c55e", bg: "rgba(34,197,94,0.10)", ring: "rgba(34,197,94,0.30)", label: "OK", icon: "check_circle", }, HALF_OPEN: { dot: "#eab308", bg: "rgba(234,179,8,0.10)", ring: "rgba(234,179,8,0.30)", label: "RECOV", icon: "sync", }, DEGRADED: { dot: "#f97316", bg: "rgba(249,115,22,0.10)", ring: "rgba(249,115,22,0.30)", label: "DEG", icon: "warning", }, OPEN: { dot: "#ef4444", bg: "rgba(239,68,68,0.10)", ring: "rgba(239,68,68,0.30)", label: "OPEN", icon: "block", }, }; const FALLBACK_BREAKER_TONE: BreakerTone = { dot: "#64748b", bg: "rgba(100,116,139,0.10)", ring: "rgba(100,116,139,0.30)", label: "UNK", icon: "help", }; const FEED_KIND_META: Record = { "circuit-opened": { icon: "block", color: "#ef4444", group: "circuits" }, "circuit-degraded": { icon: "warning", color: "#f97316", group: "circuits" }, "circuit-recovered": { icon: "sync", color: "#eab308", group: "circuits" }, "circuit-closed": { icon: "check_circle", color: "#22c55e", group: "circuits" }, "cooldown-added": { icon: "ac_unit", color: "#3b82f6", group: "cooldowns" }, "cooldown-cleared": { icon: "lock_open", color: "#22c55e", group: "cooldowns" }, "lockout-added": { icon: "lock", color: "#f97316", group: "lockouts" }, "lockout-cleared": { icon: "lock_open", color: "#22c55e", group: "lockouts" }, "session-new": { icon: "fingerprint", color: "#06b6d4", group: "sessions" }, "quota-alert": { icon: "warning", color: "#eab308", group: "quotas" }, "quota-exhausted": { icon: "error", color: "#ef4444", group: "quotas" }, "quota-recovered": { icon: "check_circle", color: "#22c55e", group: "quotas" }, }; function fmtMs(ms: number | undefined | null): string { if (!ms || ms <= 0) return "0s"; if (ms < 1000) return `${ms}ms`; const s = Math.ceil(ms / 1000); if (s < 60) return `${s}s`; const m = Math.floor(s / 60); const rs = s % 60; if (m < 60) return rs > 0 ? `${m}m ${rs}s` : `${m}m`; const h = Math.floor(m / 60); const rm = m % 60; return `${h}h ${rm}m`; } function fmtClock(ts: number): string { return new Date(ts).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit", hour12: false, }); } function shortId(value: string | null | undefined, max = 12): string { if (!value) return "—"; return value.length > max ? `${value.slice(0, max)}…` : value; } function untilMs(value: number | string | null | undefined): number { if (typeof value === "number") return value - Date.now(); if (typeof value === "string") { const ts = new Date(value).getTime(); if (Number.isFinite(ts)) return ts - Date.now(); } return 0; } function normalizeBreakerState(state: string | null | undefined): string { return String(state || "") .trim() .toUpperCase(); } function getBreakerTone(normalizedState: string): BreakerTone { return BREAKER_TONE[normalizedState] || FALLBACK_BREAKER_TONE; } function pushFeed(prev: FeedEvent[], events: FeedEvent[]): FeedEvent[] { if (events.length === 0) return prev; const merged = [...events, ...prev]; return merged.slice(0, FEED_MAX_EVENTS); } // Compute new feed events by diffing previous vs next snapshot. // Conservative: only emit transitions (added/cleared/state-change), never // emit for items that simply persist between polls. function diffSnapshots( prev: { health: HealthPayload | null; conns: Connection[] }, next: { health: HealthPayload | null; conns: Connection[] }, nowTs: number ): FeedEvent[] { const out: FeedEvent[] = []; // Circuit breakers const prevBreakers = new Map((prev.health?.providerBreakers ?? []).map((b) => [b.provider, b])); const nextBreakers = new Map((next.health?.providerBreakers ?? []).map((b) => [b.provider, b])); for (const [provider, nextB] of nextBreakers) { const prevB = prevBreakers.get(provider); if (!prevB) continue; const prevState = normalizeBreakerState(prevB.state); const nextState = normalizeBreakerState(nextB.state); if (prevState === nextState) continue; if (nextState === "OPEN") { out.push({ id: `cb-open-${provider}-${nowTs}`, ts: nowTs, kind: "circuit-opened", title: `${provider} circuit OPEN`, detail: `threshold hit · retry in ${fmtMs(nextB.retryAfterMs)}`, }); } else if (nextState === "HALF_OPEN") { out.push({ id: `cb-half-${provider}-${nowTs}`, ts: nowTs, kind: "circuit-recovered", title: `${provider} HALF_OPEN`, detail: `probing recovery`, }); } else if (nextState === "DEGRADED") { out.push({ id: `cb-deg-${provider}-${nowTs}`, ts: nowTs, kind: "circuit-degraded", title: `${provider} DEGRADED`, detail: `${nextB.failureCount} failures · degraded but serving`, }); } else if (nextState === "CLOSED" && prevState !== "CLOSED") { out.push({ id: `cb-close-${provider}-${nowTs}`, ts: nowTs, kind: "circuit-closed", title: `${provider} circuit CLOSED`, detail: `recovered to healthy`, }); } } // Cooldowns (per-connection rateLimitedUntil) const prevCooldowns = new Map( prev.conns .filter((c) => c.rateLimitedUntil && untilMs(c.rateLimitedUntil) > 0) .map((c) => [c.id, c]) ); const nextCooldowns = new Map( next.conns .filter((c) => c.rateLimitedUntil && untilMs(c.rateLimitedUntil) > 0) .map((c) => [c.id, c]) ); for (const [id, conn] of nextCooldowns) { if (!prevCooldowns.has(id)) { const label = conn.name || conn.email || conn.displayName || shortId(id); out.push({ id: `cd-add-${id}-${nowTs}`, ts: nowTs, kind: "cooldown-added", title: `${conn.provider}/${label} cooling`, detail: `${fmtMs(untilMs(conn.rateLimitedUntil))} · ${conn.lastError || "unavailable"}`, }); } } for (const [id, conn] of prevCooldowns) { if (!nextCooldowns.has(id)) { const label = conn.name || conn.email || conn.displayName || shortId(id); out.push({ id: `cd-clr-${id}-${nowTs}`, ts: nowTs, kind: "cooldown-cleared", title: `${conn.provider}/${label} resumed`, detail: `cooldown cleared`, }); } } // Lockouts const prevLockKeys = new Set(Object.keys(prev.health?.lockouts ?? {})); const nextLockKeys = new Set(Object.keys(next.health?.lockouts ?? {})); for (const key of nextLockKeys) { if (!prevLockKeys.has(key)) { const entry = next.health?.lockouts?.[key]; out.push({ id: `lk-add-${key}-${nowTs}`, ts: nowTs, kind: "lockout-added", title: `${key} locked`, detail: entry?.reason ? `reason: ${entry.reason}` : "rate-limit lockout", }); } } for (const key of prevLockKeys) { if (!nextLockKeys.has(key)) { out.push({ id: `lk-clr-${key}-${nowTs}`, ts: nowTs, kind: "lockout-cleared", title: `${key} unlocked`, detail: `lockout expired`, }); } } // Sessions (additions only — sessions are short-lived, deletions are noise) const prevSessions = new Set((prev.health?.sessions?.top ?? []).map((s) => s.sessionId)); for (const s of next.health?.sessions?.top ?? []) { if (prevSessions.has(s.sessionId)) continue; out.push({ id: `ss-new-${s.sessionId}-${nowTs}`, ts: nowTs, kind: "session-new", title: `new session ${shortId(s.sessionId, 8)}`, detail: s.connectionId ? `bound to ${shortId(s.connectionId, 10)}` : "no binding yet", }); } // Quota monitors (alerting / exhausted transitions) const prevQuota = new Map( (prev.health?.quotaMonitor?.monitors ?? []).map((m) => [ `${m.accountId ?? ""}:${m.provider ?? ""}:${m.window ?? ""}`, m, ]) ); for (const m of next.health?.quotaMonitor?.monitors ?? []) { const key = `${m.accountId ?? ""}:${m.provider ?? ""}:${m.window ?? ""}`; const prevM = prevQuota.get(key); const prevStatus = prevM?.status; if (m.status === prevStatus) continue; if (m.status === "exhausted") { out.push({ id: `qe-${key}-${nowTs}`, ts: nowTs, kind: "quota-exhausted", title: `${m.accountId ?? "?"} EXHAUSTED`, detail: `${m.window ?? ""}${m.provider ? ` · ${m.provider}` : ""}`, }); } else if (m.status === "alerting") { out.push({ id: `qa-${key}-${nowTs}`, ts: nowTs, kind: "quota-alert", title: `${m.accountId ?? "?"} ALERTING`, detail: `${m.window ?? ""}${ typeof m.remainingPercent === "number" ? ` · ${Math.round(m.remainingPercent)}% left` : "" }`, }); } else if (prevStatus === "exhausted" || prevStatus === "alerting") { out.push({ id: `qr-${key}-${nowTs}`, ts: nowTs, kind: "quota-recovered", title: `${m.accountId ?? "?"} recovered`, detail: `${m.window ?? ""} back to OK`, }); } } return out; } // ───────────────────────────────────────────────────────────────────────────── // Component // ───────────────────────────────────────────────────────────────────────────── export default function RuntimePageClient() { const t = useTranslations("runtime"); const nodeMap = useProviderNodeMap(); const [health, setHealth] = useState(null); const [connections, setConnections] = useState([]); const [loading, setLoading] = useState(true); const [paused, setPaused] = useState(false); const [lastUpdated, setLastUpdated] = useState(null); const [feed, setFeed] = useState([]); const [feedFilter, setFeedFilter] = useState("all"); const prevSnapshotRef = useRef<{ health: HealthPayload | null; conns: Connection[] }>({ health: null, conns: [], }); const initialLoadRef = useRef(true); const fetchAll = useCallback(async () => { try { const [healthRes, connsRes] = await Promise.all([ fetch("/api/monitoring/health"), fetch("/api/providers/client"), ]); const healthData: HealthPayload | null = healthRes.ok ? await healthRes.json() : null; const connsData = connsRes.ok ? await connsRes.json() : null; const conns: Connection[] = Array.isArray(connsData?.connections) ? connsData.connections : []; const nowTs = Date.now(); const prev = prevSnapshotRef.current; // Skip diff on first load — everything is "new" but we don't want a // burst of fake events on mount. if (!initialLoadRef.current) { const events = diffSnapshots(prev, { health: healthData, conns }, nowTs); if (events.length > 0) setFeed((cur) => pushFeed(cur, events)); } initialLoadRef.current = false; prevSnapshotRef.current = { health: healthData, conns }; setHealth(healthData); setConnections(conns); setLastUpdated(nowTs); } catch (err) { console.error("[Runtime] fetch failed", err); } finally { setLoading(false); } }, []); useEffect(() => { fetchAll(); if (paused) return; const id = setInterval(fetchAll, REFRESH_INTERVAL_MS); return () => clearInterval(id); }, [fetchAll, paused]); // ── Derived data ──────────────────────────────────────────────────────────── const cooldowns = useMemo(() => { return connections.filter((c) => c.rateLimitedUntil && untilMs(c.rateLimitedUntil) > 0); }, [connections]); const breakers = health?.providerBreakers ?? EMPTY_PROVIDER_BREAKERS; const lockoutEntries = useMemo>( () => Object.entries(health?.lockouts ?? {}), [health] ); const counts = useMemo(() => { let openCircuits = 0; let halfCircuits = 0; let degradedCircuits = 0; let unknownCircuits = 0; for (const breaker of breakers) { const state = normalizeBreakerState(breaker.state); if (state === "OPEN") { openCircuits++; } else if (state === "HALF_OPEN") { halfCircuits++; } else if (state === "DEGRADED") { degradedCircuits++; } else if (state !== "CLOSED") { unknownCircuits++; } } const totalBreakers = breakers.length; const affectedCircuits = openCircuits + halfCircuits + degradedCircuits + unknownCircuits; const sessions = health?.sessions?.activeCount ?? 0; const lockouts = lockoutEntries.length; const quota = health?.quotaMonitor; const quotaAlertingTotal = (quota?.alerting ?? 0) + (quota?.exhausted ?? 0) + (quota?.errors ?? 0); return { sessions, stickyBound: health?.sessions?.stickyBoundCount ?? 0, openCircuits, halfCircuits, degradedCircuits, unknownCircuits, affectedCircuits, totalBreakers, cooldowns: cooldowns.length, lockouts, quotaAlerting: quotaAlertingTotal, quotaExhausted: quota?.exhausted ?? 0, }; }, [breakers, cooldowns, health, lockoutEntries]); const filteredFeed = useMemo(() => { if (feedFilter === "all") return feed; return feed.filter((ev) => FEED_KIND_META[ev.kind].group === feedFilter); }, [feed, feedFilter]); const overallHealthy = counts.totalBreakers - counts.affectedCircuits; const overallPercent = counts.totalBreakers > 0 ? Math.round((overallHealthy / counts.totalBreakers) * 100) : 100; // ── Render ────────────────────────────────────────────────────────────────── return (
{/* Header */}

bolt {t("title")}

{t("description")}

{lastUpdated ? `↻ ${fmtClock(lastUpdated)}` : "—"}
{/* Row 1 — KPIs */}
setFeedFilter("sessions")} active={feedFilter === "sessions"} /> 0 ? t("hintRecovering", { count: counts.halfCircuits + counts.degradedCircuits + counts.unknownCircuits, }) : counts.openCircuits === 0 ? t("hintAllHealthy") : t("hintOpen") } tone={ counts.openCircuits > 0 ? "#ef4444" : counts.halfCircuits + counts.degradedCircuits + counts.unknownCircuits > 0 ? "#eab308" : "#22c55e" } onClick={() => setFeedFilter("circuits")} active={feedFilter === "circuits"} /> 0 ? "#3b82f6" : "#22c55e"} onClick={() => setFeedFilter("cooldowns")} active={feedFilter === "cooldowns"} /> 0 ? "#f97316" : "#22c55e"} onClick={() => setFeedFilter("lockouts")} active={feedFilter === "lockouts"} />
{/* Row 2 — Resilience layers (left, 2/3) + Live Feed (right, 1/3) */}
✓ {overallHealthy} ⚠ {counts.halfCircuits + counts.degradedCircuits + counts.unknownCircuits} ⛔ {counts.openCircuits}
} />
{t("providersHealthy", { percent: overallPercent })}
{/* Layer 1 */} 0 ? "red" : counts.halfCircuits + counts.degradedCircuits + counts.unknownCircuits > 0 ? "amber" : "green" } > {breakers.length === 0 ? ( ) : (
{breakers.map((b) => { const state = normalizeBreakerState(b.state); const tone = getBreakerTone(state); return (
{resolveProviderName(b.provider, nodeMap)} {tone.label}
{state === "OPEN" ? `retry ${fmtMs(b.retryAfterMs)}` : `${b.failureCount} failures`}
); })}
)}
{/* Layer 2 */} 0 ? "blue" : "green"} > {cooldowns.length === 0 ? ( ) : (
{cooldowns.slice(0, 8).map((c) => { const remaining = untilMs(c.rateLimitedUntil); const label = c.name || c.email || c.displayName || shortId(c.id); return (
{resolveProviderName(c.provider, nodeMap)}/{label}
{c.lastErrorType && (
{c.lastErrorType}
)}
{fmtMs(remaining)}
{c.lastError || "—"}
L{c.backoffLevel ?? 0}
{c.errorCode ?? ""}
); })} {cooldowns.length > 8 && (
{t("moreCooldowns", { count: cooldowns.length - 8 })}
)}
)}
{/* Layer 3 */} 0 ? "orange" : "green"} > {lockoutEntries.length === 0 ? ( ) : (
{lockoutEntries.slice(0, 8).map(([key, lk]) => { const remaining = typeof lk.remainingMs === "number" ? lk.remainingMs : untilMs(lk.until); return (
lock {key}
{lk.reason || "rate limit"}
{remaining > 0 ? fmtMs(remaining) : "—"}
); })} {lockoutEntries.length > 8 && (
{t("moreLockouts", { count: lockoutEntries.length - 8 })}
)}
)}
{/* Live Feed */}
} />
{filteredFeed.length === 0 ? (
hourglass_empty

{feed.length === 0 ? t("feedEmptyWaiting") : t("feedEmptyFiltered")}

) : ( filteredFeed.map((ev) => { const meta = FEED_KIND_META[ev.kind]; return (
{meta.icon}
{ev.title}
{fmtClock(ev.ts)}
{ev.detail}
); }) )}
{/* Row 3 — Sessions table + Quota Monitors */}
{t("sessionsActive", { count: counts.sessions })} } /> {(health?.sessions?.top ?? []).length === 0 ? (
fingerprint

{t("sessionsEmptyTitle")}

{t("sessionsEmptyHint")}

) : (
{(health?.sessions?.top ?? []).map((s) => ( ))}
{t("tblSession")} {t("tblAge")} {t("tblIdle")} {t("tblReqs")} {t("tblBoundTo")}
{shortId(s.sessionId, 14)} {fmtMs(s.ageMs)} {fmtMs(s.idleMs)} {s.requestCount} {s.connectionId ? ( {shortId(s.connectionId, 12)} ) : ( )}
{Object.keys(health?.sessions?.byApiKey ?? {}).length > 0 && (
{t("topApiKeys")}:{" "} {Object.entries(health?.sessions?.byApiKey ?? {}) .sort((a, b) => b[1] - a[1]) .slice(0, 3) .map(([k, n]) => `${shortId(k, 8)}(${n})`) .join(" · ")}
)}
)}
{t("openQuota")} → } /> {(() => { const monitors = health?.quotaMonitor?.monitors ?? []; const exhausted = monitors.filter((m) => m.status === "exhausted"); const alerting = monitors.filter((m) => m.status === "alerting"); const errors = monitors.filter((m) => m.status === "error"); const total = exhausted.length + alerting.length + errors.length; if (total === 0) { return (
radar

{t("allQuotasHealthy")}

); } return (
{exhausted.length > 0 && ( )} {alerting.length > 0 && ( )} {errors.length > 0 && ( )}
); })()}
); } // ───────────────────────────────────────────────────────────────────────────── // Sub-components // ───────────────────────────────────────────────────────────────────────────── function KpiCard({ icon, label, value, hint, tone, onClick, active, }: { icon: string; label: string; value: string | number; hint: string; tone: string; onClick: () => void; active: boolean; }) { return ( ); } function SectionHeader({ icon, title, subtitle, trailing, }: { icon: string; title: string; subtitle?: string; trailing?: React.ReactNode; }) { return (
{icon}

{title}

{subtitle &&

{subtitle}

}
{trailing &&
{trailing}
}
); } function LayerSection({ id, title, description, badge, badgeTone, children, }: { id: 1 | 2 | 3; title: string; description: string; badge: string; badgeTone: "red" | "amber" | "green" | "blue" | "orange"; children: React.ReactNode; }) { const toneMap = { red: { bg: "rgba(239,68,68,0.10)", text: "#ef4444" }, amber: { bg: "rgba(234,179,8,0.10)", text: "#eab308" }, green: { bg: "rgba(34,197,94,0.10)", text: "#22c55e" }, blue: { bg: "rgba(59,130,246,0.10)", text: "#3b82f6" }, orange: { bg: "rgba(249,115,22,0.10)", text: "#f97316" }, } as const; const tone = toneMap[badgeTone]; return (
Layer {id}

{title}

{badge}

{description}

{children}
); } function EmptyHint({ text }: { text: string }) { return (
{text}
); } function QuotaGroup({ tone, label, items, }: { tone: "red" | "amber" | "orange"; label: string; items: QuotaMonitor[]; }) { const t = useTranslations("runtime"); const toneMap = { red: { text: "#ef4444", bg: "rgba(239,68,68,0.08)", border: "rgba(239,68,68,0.20)" }, amber: { text: "#eab308", bg: "rgba(234,179,8,0.08)", border: "rgba(234,179,8,0.20)" }, orange: { text: "#f97316", bg: "rgba(249,115,22,0.08)", border: "rgba(249,115,22,0.20)" }, } as const; const tc = toneMap[tone]; return (
{label}
{items.slice(0, 6).map((m, i) => (
{m.accountId ?? "—"} {m.provider ? ` / ${resolveProviderName(m.provider, nodeMap)}` : ""}
{m.window ?? ""}
{typeof m.remainingPercent === "number" && ( {Math.round(m.remainingPercent)}% )}
))} {items.length > 6 && (
{t("moreSuffix", { count: items.length - 6 })}
)}
); }