import { getCorsHeaders, isDisallowedOrigin } from './_cors.js'; import { jsonResponse } from './_json-response.js'; import { USER_API_KEY_GATEWAY_VALIDATION_ERROR, validateApiKey } from './_api-key.js'; // Seed-envelope helper. PR 1 imports it here so PR 2 can wire envelope-aware // reads at specific call sites without further plumbing. It's a no-op on // legacy-shape seed-meta values (they have no `_seed` wrapper and pass through // as `.data`), so importing it is behavior-preserving. import { unwrapEnvelope } from './_seed-envelope.js'; // @ts-expect-error — JS module, no declaration file import { redisPipeline, getRedisCredentials } from './_upstash-json.js'; import { CII_RISK_SCORE_CACHE_KEYS } from './_cii-risk-cache-keys.js'; import { BOOTSTRAP_CACHE_KEYS } from './_bootstrap-tier-keys.js'; export const config = { runtime: 'edge' }; // HTTP responses stay `no-store`, but the expensive Redis-wide verdict is // shared briefly at the origin. At the measured browser poll rate this turns // ~390 Redis commands per request into one GET, plus one sweep per minute. const HEALTH_VERDICT_SNAPSHOT_BASE_KEY = 'health:verdict:v1'; function healthVerdictRedisKey(baseKey, vercelEnv, commitSha) { if (!vercelEnv || vercelEnv === 'production') return baseKey; return `${vercelEnv}:${commitSha?.slice(0, 8) || 'dev'}:${baseKey}`; } const HEALTH_VERDICT_SNAPSHOT_KEY = healthVerdictRedisKey( HEALTH_VERDICT_SNAPSHOT_BASE_KEY, process.env.VERCEL_ENV, process.env.VERCEL_GIT_COMMIT_SHA, ); // The memoized verdict is stored TWICE, and the difference is the whole point. // The full snapshot carries the entire `checks` map (~228 entries, ~20 KB). The // compact snapshot carries what `?compact=1` actually returns — status, summary, // checkedAt and the handful of problem entries — about 1 KB. // // `?compact=1` is the browser poll: every client hits it every 5 minutes (~115k // times/day). Before this split each of those reads pulled the full 20 KB blob out // of Redis to render ~1 KB of it, which is ~2.2 GB/day of egress for bytes the // caller throws away (#5300). One sweep writes both keys, so they can never // disagree. const HEALTH_VERDICT_COMPACT_SNAPSHOT_BASE_KEY = 'health:verdict:compact:v1'; const HEALTH_VERDICT_COMPACT_SNAPSHOT_KEY = healthVerdictRedisKey( HEALTH_VERDICT_COMPACT_SNAPSHOT_BASE_KEY, process.env.VERCEL_ENV, process.env.VERCEL_GIT_COMMIT_SHA, ); const HEALTH_VERDICT_SNAPSHOT_TTL_SECONDS = 60; // Edge runtime mirror of scripts/china-coverage-manifest.mjs. Edge functions // cannot import scripts/; tests enforce key and status-projection parity. const CHINA_COVERAGE_SUMMARY_KEY = 'health:china-coverage:v1'; const HEALTH_VERDICT_SNAPSHOT_TTL_MS = HEALTH_VERDICT_SNAPSHOT_TTL_SECONDS * 1_000; const HEALTH_VERDICT_REFRESH_LOCK_KEY = `${HEALTH_VERDICT_SNAPSHOT_KEY}:refresh-lock`; // The sweep can consume its full 8s timeout, followed by a 4s failure-log read // and 4s snapshot write. Keep the lease comfortably above that worst case. const HEALTH_VERDICT_REFRESH_LOCK_TTL_SECONDS = 30; const HEALTH_VERDICT_REFRESH_WAIT_ATTEMPTS = 45; const HEALTH_VERDICT_REFRESH_WAIT_MS = 3_000; // Avoid starting a Redis HTTP request that cannot realistically complete // inside the remaining contention budget. The direct-sweep fallback below is // preferable to turning a deadline-boundary timeout into false REDIS_DOWN. const HEALTH_VERDICT_MIN_REDIS_TIMEOUT_MS = 100; const HEALTH_VERDICT_RELEASE_LOCK_SCRIPT = [ "if redis.call('get', KEYS[1]) == ARGV[1] then", " return redis.call('del', KEYS[1])", 'end', 'return 0', ].join('\n'); // Iran-events domain sunset (war ended 2026-07). Default OFF everywhere; set // IRAN_EVENTS_ENABLED=true to restore the whole domain. Mirrors the backend // *_ENABLED env idiom (server/worldmonitor/resilience/v1/_shared.ts). const IRAN_EVENTS_ENABLED = (process.env.IRAN_EVENTS_ENABLED ?? 'false').toLowerCase() === 'true'; const BOOTSTRAP_KEYS = { earthquakes: 'seismology:earthquakes:v1', outages: 'infra:outages:v1', sectors: 'market:sectors:v2', etfFlows: 'market:etf-flows:v1', climateAnomalies: 'climate:anomalies:v2', climateDisasters: 'climate:disasters:v1', climateAirQuality: 'climate:air-quality:v1', co2Monitoring: 'climate:co2-monitoring:v1', oceanIce: 'climate:ocean-ice:v1', wildfires: 'wildfire:fires:v1', wildfiresBootstrap: 'wildfire:fires-bootstrap:v1', marketQuotes: 'market:stocks-bootstrap:v1', commodityQuotes: 'market:commodities-bootstrap:v1', cyberThreats: 'cyber:threats-bootstrap:v2', techReadiness: 'economic:worldbank-techreadiness:v1', progressData: 'economic:worldbank-progress:v1', renewableEnergy: 'economic:worldbank-renewable:v1', positiveGeoEvents: 'positive_events:geo-bootstrap:v1', riskScores: CII_RISK_SCORE_CACHE_KEYS.stale, naturalEvents: 'natural:events:v1', flightDelays: 'aviation:delays-bootstrap:v2', newsInsights: 'news:insights:v1', predictionMarkets: 'prediction:markets-bootstrap:v1', cryptoQuotes: 'market:crypto:v1', gulfQuotes: 'market:gulf-quotes:v1', stablecoinMarkets: 'market:stablecoins:v1', unrestEvents: 'unrest:events:v1', iranEvents: 'conflict:iran-events:v1', ucdpEvents: 'conflict:ucdp-events:v1', ucdpEventsBootstrap: 'conflict:ucdp-events-bootstrap:v1', weatherAlerts: 'weather:alerts:v1', spending: 'economic:spending:v1', techEvents: 'research:tech-events-bootstrap:v1', gdeltIntel: 'intelligence:gdelt-intel:v1', correlationCards: 'correlation:cards-bootstrap:v1', crossStraitActivity: 'military:cross-strait-activity:v1', crossStraitActivityBootstrap: 'military:cross-strait-activity-bootstrap:v1', forecasts: 'forecast:predictions:v2', forecastsBootstrap: 'forecast:predictions-bootstrap:v1', securityAdvisories: 'intelligence:advisories-bootstrap:v1', customsRevenue: 'trade:customs-revenue:v1', comtradeFlows: 'comtrade:flows:v1', blsSeries: 'bls:series:v1', sanctionsPressure: 'sanctions:pressure:v1', crossSourceSignals: 'intelligence:cross-source-signals:v1', sanctionsEntities: 'sanctions:entities:v1', radiationWatch: 'radiation:observations:v1', consumerPricesOverview: 'consumer-prices:overview:ae', consumerPricesCategories: 'consumer-prices:categories:ae:30d', consumerPricesMovers: 'consumer-prices:movers:ae:30d', consumerPricesSpread: 'consumer-prices:retailer-spread:ae:essentials-ae', consumerPricesFreshness: 'consumer-prices:freshness:ae', groceryBasket: 'economic:grocery-basket:v1', bigmac: 'economic:bigmac:v1', fuelPrices: 'economic:fuel-prices:v1', faoFoodPriceIndex: 'economic:fao-ffpi:v1', nationalDebt: 'economic:national-debt:v1', defiTokens: 'market:defi-tokens:v1', aiTokens: 'market:ai-tokens:v1', otherTokens: 'market:other-tokens:v1', fredBatch: 'economic:fred:v1:FEDFUNDS:0', ecbEstr: 'economic:fred:v1:ESTR:0', ecbEuribor3m: 'economic:fred:v1:EURIBOR3M:0', ecbEuribor6m: 'economic:fred:v1:EURIBOR6M:0', ecbEuribor1y: 'economic:fred:v1:EURIBOR1Y:0', fearGreedIndex: 'market:fear-greed:v1', breadthHistory: 'market:breadth-history:v1', euYieldCurve: 'economic:yield-curve-eu:v1', earningsCalendar: 'market:earnings-calendar:v1', econCalendar: 'economic:econ-calendar:v1', chinaMacro: BOOTSTRAP_CACHE_KEYS.chinaMacro, chinaReleaseCalendar: BOOTSTRAP_CACHE_KEYS.chinaReleaseCalendar, chinaCorporateDisclosures: 'market:china:corporate-disclosures:v1', chinaPolicyEvents: 'china:policy-events:v1', chinaDecisionSignals: BOOTSTRAP_CACHE_KEYS.chinaDecisionSignals, cotPositioning: 'market:cot:v1', hyperliquidFlow: 'market:hyperliquid:flow:v1', crudeInventories: 'economic:crude-inventories:v1', natGasStorage: 'economic:nat-gas-storage:v1', spr: 'economic:spr:v1', refineryInputs: 'economic:refinery-inputs:v1', ecbFxRates: 'economic:ecb-fx-rates:v1', eurostatCountryData: 'economic:eurostat-country-data:v1', eurostatHousePrices: 'economic:eurostat:house-prices:v1', eurostatGovDebtQ: 'economic:eurostat:gov-debt-q:v1', eurostatIndProd: 'economic:eurostat:industrial-production:v1', euGasStorage: 'economic:eu-gas-storage:v1', euFsi: 'economic:fsi-eu:v1', shippingStress: 'supply_chain:shipping_stress:v1', diseaseOutbreaks: 'health:disease-outbreaks:v1', healthAirQuality: 'health:air-quality:v1', socialVelocity: 'intelligence:social:reddit:v1', wsbTickers: 'intelligence:wsb-tickers:v1', vpdTrackerRealtime: 'health:vpd-tracker:realtime:v1', vpdTrackerHistorical: 'health:vpd-tracker:historical:v1', electricityPrices: 'energy:electricity:v1:index', gasStorageCountries: 'energy:gas-storage:v1:_countries', aaiiSentiment: 'market:aaii-sentiment:v1', cryptoSectors: 'market:crypto-sectors:v1', ddosAttacks: 'cf:radar:ddos:v1', economicStress: 'economic:stress-index:v1', trafficAnomalies: 'cf:radar:traffic-anomalies:v1', }; const STANDALONE_KEYS = { chinaCoverage: CHINA_COVERAGE_SUMMARY_KEY, hkoWarnings: 'weather:hko-warnings:v1', humanitarianSummary: 'conflict:humanitarian:v1', // #4920 completeness measurement (daily GH Actions publishers) — ops // keys: health-monitored but NOT bootstrap-hydrated into page loads. newsFeedHealth: 'news:feed-health:v1', newsRecallBenchmark: 'news:recall-benchmark:v1', serviceStatuses: 'infra:service-statuses:v1', macroSignals: 'economic:macro-signals:v1', energyPrices: 'economic:energy:v1:all', bisPolicy: 'economic:bis:policy:v1', bisExchange: 'economic:bis:eer:v1', fxYoy: 'economic:fx:yoy:v1', sharedFxRates: 'shared:fx-rates:v1', bisCredit: 'economic:bis:credit:v1', bisDsr: 'economic:bis:dsr:v1', bisPropertyResidential: 'economic:bis:property-residential:v1', bisPropertyCommercial: 'economic:bis:property-commercial:v1', imfMacro: 'economic:imf:macro:v2', imfGrowth: 'economic:imf:growth:v1', imfLabor: 'economic:imf:labor:v1', imfExternal: 'economic:imf:external:v1', // plan 2026-04-25-004 Phase 2: financialSystemExposure data keys. wbExternalDebt: 'economic:wb-external-debt:v1', bisLbs: 'economic:bis-lbs:v1', fatfListing: 'economic:fatf-listing:v1', climateZoneNormals: 'climate:zone-normals:v1', shippingRates: 'supply_chain:shipping:v2', chokepoints: 'supply_chain:chokepoints:v4', minerals: 'supply_chain:minerals:v2', giving: 'giving:summary:v2', gpsjam: 'intelligence:gpsjam:v2', // Corporate intelligence (issue #5695): SEC ticker→CIK registry + 8-K // material-events stream. Health-monitored, not bootstrap-hydrated. secCikMap: 'intelligence:sec-cik-map:v1', sec8kStream: 'intelligence:sec-8k-stream:v1', theaterPosture: 'theater_posture:sebuf:stale:v1', theaterPostureLive: 'theater-posture:sebuf:v1', theaterPostureBackup: 'theater-posture:sebuf:backup:v1', riskScoresLive: CII_RISK_SCORE_CACHE_KEYS.live, usniFleet: 'usni-fleet:sebuf:v1', usniFleetStale: 'usni-fleet:sebuf:stale:v1', faaDelays: 'aviation:delays:faa:v1', intlDelays: 'aviation:delays:intl:v3', notamClosures: 'aviation:notam:closures:v2', positiveEventsLive: 'positive-events:geo:v1', cableHealth: 'cable-health-v1', submarineCables: 'infrastructure:submarine-cables:v1', cyberThreatsRpc: 'cyber:threats:v2', militaryBases: 'military:bases:active', militaryFlights: 'military:flights:v1', militaryFlightsStale: 'military:flights:stale:v1', // STANDALONE (not BOOTSTRAP) by design — value is a single keyed payload // (asOf + per-country aggregate), no per-record gating needed; health just // verifies existence + freshness via the matching SEED_META entry. Same // shape as militaryFlights above. militaryCii: 'intelligence:military-cii:v1', globalTendersSam: 'economic:global-tenders:v1:source:sam', globalTendersTed: 'economic:global-tenders:v1:source:ted', globalTendersContractsFinder: 'economic:global-tenders:v1:source:contracts-finder', globalTendersCanadaBuys: 'economic:global-tenders:v1:source:canada-buys', globalTendersGets: 'economic:global-tenders:v1:source:gets', globalTendersWorldBank: 'economic:global-tenders:v1:source:world-bank', crossStraitActivityTaiwanMnd: 'military:cross-strait-activity:v1:source:taiwan-mnd', crossStraitActivityJapanMod: 'military:cross-strait-activity:v1:source:japan-mod', defensePatents: 'patents:defense:latest', temporalAnomalies: 'temporal:anomalies:v1', displacement: `displacement:summary:v1:${new Date().getUTCFullYear()}`, displacementPrev: `displacement:summary:v1:${new Date().getUTCFullYear() - 1}`, acledIntel: 'conflict:acled:v1:all:0:0', satellites: 'intelligence:satellites:tle:v1', portwatch: 'supply_chain:portwatch:v1', portwatchDisruptions: 'portwatch:disruptions:active:v1', portwatchPortActivity: 'supply_chain:portwatch-ports:v1:_countries', corridorrisk: 'supply_chain:corridorrisk:v1', chokepointTransits: 'supply_chain:chokepoint_transits:v1', transitSummaries: 'supply_chain:transit-summaries:v1', // Meta-only aggregate: payloads are sharded by country, so use the seed-meta // key as the probe target rather than pretending one country key is global. comtradeBilateralHs4: 'seed-meta:comtrade:bilateral-hs4', thermalEscalation: 'thermal:escalation:v1', thermalEscalationBootstrap: 'thermal:escalation-bootstrap:v1', tariffTrendsUs: 'trade:tariffs:v1:840:all:10', militaryForecastInputs: 'military:forecast-inputs:stale:v1', gscpi: 'economic:fred:v1:GSCPI:0', forecastFredWalcl: 'economic:fred:v1:WALCL:0', forecastFredT10y2y: 'economic:fred:v1:T10Y2Y:0', forecastFredUnrate: 'economic:fred:v1:UNRATE:0', forecastFredCpiaucsl: 'economic:fred:v1:CPIAUCSL:0', forecastFredDgs10: 'economic:fred:v1:DGS10:0', forecastFredVixcls: 'economic:fred:v1:VIXCLS:0', forecastFredGdp: 'economic:fred:v1:GDP:0', forecastFredM2sl: 'economic:fred:v1:M2SL:0', forecastFredDcoilwtico: 'economic:fred:v1:DCOILWTICO:0', marketImplications: 'intelligence:market-implications:v1', hormuzTracker: 'supply_chain:hormuz_tracker:v1', simulationPackageLatest: 'forecast:simulation-package:latest', simulationOutcomeLatest: 'forecast:simulation-outcome:latest', newsThreatSummary: 'news:threat:summary:v1', climateNews: 'climate:news-intelligence:v1', pizzint: 'intelligence:pizzint:seed:v1', resilienceStaticIndex: 'resilience:static:index:v1', resilienceStaticFao: 'resilience:static:fao', resilienceRanking: 'resilience:ranking:v25', productCatalog: 'product-catalog:v3', energySpineCountries: 'energy:spine:v1:_countries', energyExposure: 'energy:exposure:v1:index', energyMixAll: 'energy:mix:v1:_all', regulatoryActions: 'regulatory:actions:v1', energyIntelligence: 'energy:intelligence:feed:v1', ieaOilStocks: 'energy:iea-oil-stocks:v1:index', oilStocksAnalysis: 'energy:oil-stocks-analysis:v1', eiaPetroleum: 'energy:eia-petroleum:v1', jodiGas: 'energy:jodi-gas:v1:_countries', lngVulnerability: 'energy:lng-vulnerability:v1', jodiOil: 'energy:jodi-oil:v1:_countries', chokepointBaselines: 'energy:chokepoint-baselines:v1', portwatchChokepointsRef: 'portwatch:chokepoints:ref:v1', chokepointFlows: 'energy:chokepoint-flows:v1', emberElectricity: 'energy:ember:v1:_all', resilienceIntervals: 'resilience:intervals:v9:US', sprPolicies: 'energy:spr-policies:v1', pipelinesGas: 'energy:pipelines:gas:v1', pipelinesOil: 'energy:pipelines:oil:v1', storageFacilities: 'energy:storage-facilities:v1', fuelShortages: 'energy:fuel-shortages:v1', energyDisruptions: 'energy:disruptions:v1', energyCrisisPolicies: 'energy:crisis-policies:v1', regionalSnapshots: 'intelligence:regional-snapshots:summary:v1', regionalBriefs: 'intelligence:regional-briefs:summary:v1', recoveryFiscalSpace: 'resilience:recovery:fiscal-space:v1', recoveryReserveAdequacy: 'resilience:recovery:reserve-adequacy:v1', recoveryExternalDebt: 'resilience:recovery:external-debt:v1', recoveryImportHhi: 'resilience:recovery:import-hhi:v1', // recoveryFuelStocks: probe removed. The seeder // (seed-recovery-fuel-stocks.mjs) still runs in seed-bundle-resilience- // recovery so the data is preserved for a future replacement dimension, // but `scoreFuelStockDays` in _dimension-scorers.ts is hard-wired to // return coverage=0/imputationClass=null (retired in PR 3 §3.5) and // never calls getCachedJson on this key. Probing it surfaced a green // STATUS:OK that meant "the cron ran", not "the data is used"; that's // an actively-misleading signal so it's gone. Re-add this slot if and // when a future PR wires scoreFuelStockDays to read real data again. recoveryReexportShare: 'resilience:recovery:reexport-share:v1', recoverySovereignWealth: 'resilience:recovery:sovereign-wealth:v1', // PR 1 v2 energy-construct seeds. STRICT SEED_META (not ON_DEMAND): // plan 2026-04-24-001 removed these from ON_DEMAND_KEYS so /api/health // reports CRIT (not WARN) when they are absent. This is the intended // alarm on the Railway bundle-not-provisioned state. See the ON_DEMAND_KEYS // comment block below for the full rationale. lowCarbonGeneration: 'resilience:low-carbon-generation:v1', fossilElectricityShare: 'resilience:fossil-electricity-share:v1', powerLosses: 'resilience:power-losses:v1', goldExtended: 'market:gold-extended:v1', goldEtfFlows: 'market:gold-etf-flows:v1', goldCbReserves: 'market:gold-cb-reserves:v1', // Relay-side loop heartbeats. ais-relay.cjs writes these on successful child // exit for the two execFile-spawned seeders (chokepoint-flows, climate-news). // A stale heartbeat means the relay loop itself is broken (child dying at // import, parent event-loop blocked, container in a restart loop, etc.) // and alarms earlier than the underlying seed-meta staleness window. chokepointFlowsRelayHeartbeat: 'relay:heartbeat:chokepoint-flows', climateNewsRelayHeartbeat: 'relay:heartbeat:climate-news', telegramFeed: 'intelligence:telegram-feed:v1', digestNotifications: 'digest:last-run', webcams: 'webcam:cameras:active', forecastResolutions: 'forecast:resolutions:v1', forecastScorecard: 'forecast:scorecard:v1', forecastBets: 'forecast:bets:history:v1', forecastFunnel: 'forecast:funnel:health:v1', researchArxivHnTrending: 'research:arxiv:v1:cs.AI::50', // #5736 — historical-intelligence ingest health, one record per collector. // These are NOT the collectors' canonical keys: scripts/_seed-history.mjs // appends to the Convex intel-history store fail-open, so a permanently // broken relay leg (missing RELAY_SHARED_SECRET, rejecting relay, dead // embedder) left every canonical key fresh and the store empty. The paired // seed-meta below carries the last HEALTHY append, so "history stopped // accumulating" ages into STALE_SEED while canonical publication stays OK. intelHistoryIngestConflictAcled: 'intel-history:ingest-health:conflict:acled-intel:v1', intelHistoryIngestMilitaryCrossStrait: 'intel-history:ingest-health:military:cross-strait-activity:v1', intelHistoryIngestEnergyIntelligence: 'intel-history:ingest-health:energy:intelligence:v1', }; const SEED_META = { // scripts/seed-conflict-intel.mjs cron runs every 15min, while HAPI is gated // to one bulk refresh every 2h. Bounded above by // HAPI_TTL (360min / 6h) — the per-country data key's own Redis TTL — not // by cron headroom alone: this MUST fire before a per-country key can expire, // or users see empty data for up to 6h before health notices (#5554 review; // an earlier 720min value left exactly that blind spot). 300 (5h) is ~10x // the HAPI refresh interval (headroom for missed/lock-contended ticks) while staying // a full hour below the 6h data-expiry floor. humanitarianSummary: { key: 'seed-meta:conflict:humanitarian', maxStaleMin: 300, minRecordCount: 23 }, chinaCoverage: { key: 'seed-meta:health:china-coverage', maxStaleMin: 180 }, earthquakes: { key: 'seed-meta:seismology:earthquakes', maxStaleMin: 30 }, wildfires: { key: 'seed-meta:wildfire:fires', maxStaleMin: 360 }, // FIRMS NRT resets at midnight UTC; new-day data takes 3-6h to accumulate wildfiresBootstrap: { key: 'seed-meta:wildfire:fires-bootstrap', maxStaleMin: 360 }, // Compact CDN payload is a distinct publish target; monitor it so canonical fallback cannot hide transform/write failures. outages: { key: 'seed-meta:infra:outages', maxStaleMin: 30 }, climateAnomalies: { key: 'seed-meta:climate:anomalies', maxStaleMin: 540 }, // bundled into seed-bundle-climate (cron `0 */3 * * *`, every 3h); 540 = 3× cron cadence per project convention. Prior 240 (1.33× cron) flipped to silent-EMPTY between minute 180 (TTL_DATA expiry) and 240 (alarm trigger) on every routine cron-jitter cycle — see scripts/seed-climate-anomalies.mjs CACHE_TTL comment. climateDisasters: { key: 'seed-meta:climate:disasters', maxStaleMin: 720 }, // runs every 6h; 720min = 2x interval climateAirQuality:{ key: 'seed-meta:health:air-quality', maxStaleMin: 180 }, // hourly cron; 180 = 3x interval — shares meta key with healthAirQuality (same seeder run) climateZoneNormals: { key: 'seed-meta:climate:zone-normals', maxStaleMin: 89280 }, // monthly cron on the 1st; 62d = 2x 31-day cadence co2Monitoring: { key: 'seed-meta:climate:co2-monitoring', maxStaleMin: 4320 }, // daily cron at 06:00 UTC; 72h tolerates two missed runs oceanIce: { key: 'seed-meta:climate:ocean-ice', maxStaleMin: 2880 }, // daily cron at 08:00 UTC; 48h = 2× interval, tolerates one missed run climateNews: { key: 'seed-meta:climate:news-intelligence', maxStaleMin: 90 }, // relay loop every 30min; 90 = 3× interval unrestEvents: { key: 'seed-meta:unrest:events', maxStaleMin: 120 }, // 45min cron; 120 = 2h grace (was 75 = 30min buffer, too tight) cyberThreats: { key: 'seed-meta:cyber:threats', maxStaleMin: 240 }, // 2h interval; 240min = 2x interval cryptoQuotes: { key: 'seed-meta:market:crypto', maxStaleMin: 30 }, etfFlows: { key: 'seed-meta:market:etf-flows', maxStaleMin: 60 }, gulfQuotes: { key: 'seed-meta:market:gulf-quotes', maxStaleMin: 30 }, stablecoinMarkets:{ key: 'seed-meta:market:stablecoins', maxStaleMin: 60 }, naturalEvents: { key: 'seed-meta:natural:events', maxStaleMin: 540 }, // 3h Railway climate bundle; 3x cadence preserves a full missed run. hkoWarnings: { key: 'seed-meta:weather:hko-warnings', maxStaleMin: 540 }, // successful HKO responses publish a snapshot even when no tropical-cyclone warning is active. flightDelays: { key: 'seed-meta:aviation:faa', maxStaleMin: 90 }, // CACHE_TTL=7200s; matches notamClosures from same cron notamClosures: { key: 'seed-meta:aviation:notam', maxStaleMin: 240 }, // 2h interval; 240min = 2x interval predictionMarkets: { key: 'seed-meta:prediction:markets', maxStaleMin: 90, minRecordCount: 20 }, // #5733: declareRecords now counts DISTINCT markets (it triple-counted across the pre-partition near-duplicate pools, reporting ~75 for 46 real markets), so a volume collapse is finally measurable. Floor is deliberately well under the ~46 steady state and well over a broken run — the publish gate only catches mislabeling, so without this a run that shrinks every pool while each stays non-empty stays GREEN. newsInsights: { key: 'seed-meta:news:insights', maxStaleMin: 30 }, // #4920: daily GH Actions cadence; 2880 = 2x — one fully missed day alarms newsFeedHealth: { key: 'seed-meta:news:feed-health', maxStaleMin: 2880 }, newsRecallBenchmark: { key: 'seed-meta:news:recall-benchmark', maxStaleMin: 2880 }, marketQuotes: { key: 'seed-meta:market:stocks', maxStaleMin: 30 }, commodityQuotes: { key: 'seed-meta:market:commodities', maxStaleMin: 30 }, goldExtended: { key: 'seed-meta:market:gold-extended', maxStaleMin: 30 }, goldEtfFlows: { key: 'seed-meta:market:gold-etf-flows', maxStaleMin: 2880 }, // SPDR publishes daily; 2× = 48h tolerance goldCbReserves: { key: 'seed-meta:market:gold-cb-reserves', maxStaleMin: 44640 }, // IMF IFS is monthly w/ ~2-3mo lag; 31d tolerance // RPC/warm-ping keys — seed-meta written by relay loops or handlers // serviceStatuses: moved to ON_DEMAND — RPC-populated, no dedicated seed, goes stale when no users visit cableHealth: { key: 'seed-meta:cable-health', maxStaleMin: 90 }, // ais-relay warm-ping runs every 30min; 90min = 3× interval catches missed pings without false positives submarineCables: { key: 'seed-meta:infrastructure:submarine-cables', maxStaleMin: 25200 }, macroSignals: { key: 'seed-meta:economic:macro-signals', maxStaleMin: 150 }, // seed-economy afterPublish-derived stress/macro key chinaMacro: { key: 'seed-meta:economic:china-macro-transport', maxStaleMin: 4_320 }, // oldest required-source last success; preserved failures do not refresh it chinaReleaseCalendar: { key: 'seed-meta:economic:china-release-calendar', maxStaleMin: 4_320 }, chinaCorporateDisclosures: { key: 'seed-meta:market:china-corporate-disclosures', maxStaleMin: 180 }, crossStraitActivity: { key: 'seed-meta:military:cross-strait-activity', maxStaleMin: 720 }, crossStraitActivityBootstrap: { key: 'seed-meta:military:cross-strait-activity-bootstrap', maxStaleMin: 720 }, crossStraitActivityTaiwanMnd: { key: 'seed-meta:military:cross-strait-activity:taiwan-mnd', maxStaleMin: 720 }, crossStraitActivityJapanMod: { key: 'seed-meta:military:cross-strait-activity:japan-mod', maxStaleMin: 720 }, chinaPolicyEvents: { key: 'seed-meta:china:policy-events', maxStaleMin: 2_160 }, chinaDecisionSignals: { key: 'seed-meta:intelligence:china-decision-signals', maxStaleMin: 60, minRecordCount: 6 }, energyPrices: { key: 'seed-meta:economic:energy-prices', maxStaleMin: 150 }, // seed-economy primary runSeed resource bisPolicy: { key: 'seed-meta:economic:bis', maxStaleMin: 10080 }, // runSeed('economic','bis',...) writes seed-meta:economic:bis // seed-bis-extended.mjs is a child-process section spawned by // scripts/seed-bundle-macro.mjs. The bundle's Railway cron fires more // often than the per-section `intervalMs: 12 * HOUR` gate (production // logs 2026-04-26T08:00:45 show "BIS-Extended Skipped, last seeded // 175min ago, interval: 720min" — gate actively load-bearing on every // bundle tick). So the EFFECTIVE write cadence is governed by the 12h // gate, not the bundle cron. Per-dataset seed-meta is only written when // THAT dataset published fresh entries — so a single-dataset BIS outage // (e.g. WS_DSR 500s) goes STALE in health without dragging down the // healthy ones. // // The previous 1440 (= 2× the 12h gate, but only 1× the actual rolled-up // cadence after typical cron drift) had ZERO grace. All three keys // flipped to STALE_SEED synchronously at minute 1442 on 2026-04-27 // (gate-eligible run delayed by ~24h after one failed intermediate cron). // 2160min = 3× the 12h gate covers cron drift + one degraded-to-24h // cycle, fires within 36h on a real outage. bisDsr: { key: 'seed-meta:economic:bis-dsr', maxStaleMin: 2160 }, bisPropertyResidential:{ key: 'seed-meta:economic:bis-property-residential', maxStaleMin: 2160 }, bisPropertyCommercial: { key: 'seed-meta:economic:bis-property-commercial', maxStaleMin: 2160 }, imfMacro: { key: 'seed-meta:economic:imf-macro', maxStaleMin: 100800 }, // monthly seed; 100800min = 70 days = 2× interval (absorbs one missed run) imfGrowth: { key: 'seed-meta:economic:imf-growth', maxStaleMin: 100800 }, // monthly seed; 70d threshold matches imfMacro (same WEO release cadence) imfLabor: { key: 'seed-meta:economic:imf-labor', maxStaleMin: 100800 }, // monthly seed; 70d threshold matches imfMacro imfExternal: { key: 'seed-meta:economic:imf-external', maxStaleMin: 100800 }, // monthly seed; 70d threshold matches imfMacro // plan 2026-04-25-004 Phase 2: financialSystemExposure component seeders. // Bundle: scripts/seed-bundle-macro.mjs (Codex R1 #5, Option A). wbExternalDebt: { key: 'seed-meta:economic:wb-external-debt', maxStaleMin: 100800 }, // annual WB IDS publication; 70d threshold matches IMF cadence pattern bisLbs: { key: 'seed-meta:economic:bis-lbs', maxStaleMin: 14400 }, // BIS LBS quarterly publication; 10d threshold = ~1× publish lag tolerance after macro bundle daily refresh fatfListing: { key: 'seed-meta:economic:fatf-listing', maxStaleMin: 60480 }, // FATF plenary 3×/year; 42d threshold = >1 plenary cycle (catches missed-update if cron silently fails) shippingRates: { key: 'seed-meta:supply_chain:shipping', maxStaleMin: 420 }, chokepoints: { key: 'seed-meta:supply_chain:chokepoints', maxStaleMin: 60, minRecordCount: 13 }, // 13 canonical chokepoints; get-chokepoint-status writes covered-count → < 13 = upstream partial (portwatch/ArcGIS dropped some) // minerals + giving: on-demand cachedFetchJson only, no seed-meta writer — freshness checked via TTL // bisExchange + bisCredit: extras written by same BIS script via writeExtraKey, no dedicated seed-meta fxYoy: { key: 'seed-meta:economic:fx-yoy', maxStaleMin: 1500 }, // daily cron; 25h tolerance + 1h drift sharedFxRates: { key: 'seed-meta:shared:fx-rates', maxStaleMin: 3600 }, // daily seed; 60h tolerance covers a missed 25h cache refresh gpsjam: { key: 'seed-meta:intelligence:gpsjam', maxStaleMin: 1440 }, // Wingbits API (scripts/fetch-gpsjam.mjs); 1440min = 24h tolerance gives operator headroom to handle upstream outages and monthly quota exhaustion (HTTP 402 observed 2026-04-29) without dashboard noise. Seeder catch-block extends TTL on fail without refreshing fetchedAt, so STALE_SEED via age is the only alarm path. positiveGeoEvents:{ key: 'seed-meta:positive-events:geo', maxStaleMin: 60 }, riskScores: { key: 'seed-meta:intelligence:risk-scores', maxStaleMin: 30, minRecordCount: 3 }, // CII warm-ping every 8min; recordCount is realtime signal-density coverage for score-relevant conflict (ACLED or UCDP), news, and cyber families, not raw feed availability; quiet-but-fresh feeds may warn COVERAGE_PARTIAL. iranEvents: { key: 'seed-meta:conflict:iran-events', maxStaleMin: 20160 }, // manual seed from LiveUAMap; 20160 = 14d = 2× weekly cadence ucdpEvents: { key: 'seed-meta:conflict:ucdp-events', maxStaleMin: 420 }, ucdpEventsBootstrap: { key: 'seed-meta:conflict:ucdp-events-bootstrap', maxStaleMin: 420 }, // Same cron. Monitored separately because the bootstrap tier now hydrates from the compact projection, and a transform/write failure there must not hide behind a healthy canonical key (#5300). acledIntel: { key: 'seed-meta:conflict:acled-intel', maxStaleMin: 38 }, // conflict:acled:v1:all:0:0, now ACLED-or-GDELT fallback for the forecast EMA input (#5099). militaryFlights: { key: 'seed-meta:military:flights', maxStaleMin: 30 }, // cron ~10min (LIVE_TTL=600s); 30min = 3x interval, militaryCii: { key: 'seed-meta:intelligence:military-cii', maxStaleMin: 45 }, // seed-military-cii cron ~10min; 45 = generous grace (relay-dependent; preserve-last-good runs still refresh meta) defensePatents: { key: 'seed-meta:military:defense-patents', maxStaleMin: 25200 }, satellites: { key: 'seed-meta:intelligence:satellites', maxStaleMin: 240 }, // CelesTrak every 120min; 240min = absorbs one missed cycle temporalAnomalies:{ key: 'seed-meta:temporal:anomalies', maxStaleMin: 45 }, // request-driven producer kept warm by seed-infra; data TTL is 60min so health reaches STALE_SEED before EMPTY weatherAlerts: { key: 'seed-meta:weather:alerts', maxStaleMin: 45 }, // relay loop every 15min; 45 = 3× interval (was 30 = 2×, too tight on relay hiccup) spending: { key: 'seed-meta:economic:spending', maxStaleMin: 120 }, globalTenders: { key: 'seed-meta:economic:global-tenders', maxStaleMin: 180 }, globalTendersSam: { key: 'seed-meta:economic:global-tenders:sam', maxStaleMin: 240 }, // 150min request pacing + hourly member gate yields ~180min publishes; 240min leaves one gate of scheduling jitter without raising the 10/day SAM budget. globalTendersTed: { key: 'seed-meta:economic:global-tenders:ted', maxStaleMin: 180 }, globalTendersContractsFinder: { key: 'seed-meta:economic:global-tenders:contracts-finder', maxStaleMin: 180 }, globalTendersCanadaBuys: { key: 'seed-meta:economic:global-tenders:canada-buys', maxStaleMin: 180 }, globalTendersGets: { key: 'seed-meta:economic:global-tenders:gets', maxStaleMin: 180 }, globalTendersWorldBank: { key: 'seed-meta:economic:global-tenders:world-bank', maxStaleMin: 180 }, techEvents: { key: 'seed-meta:research:tech-events', maxStaleMin: 480 }, researchArxivHnTrending: { key: 'seed-meta:research:arxiv-hn-trending', maxStaleMin: 150 }, gdeltIntel: { key: 'seed-meta:intelligence:gdelt-intel', maxStaleMin: 45 }, // 15min bulk materializer; 45min = 3× cadence and expires before the 24h canonical key. telegramFeed: { key: 'seed-meta:intelligence:telegram-feed:v1', maxStaleMin: 10 }, // 60s poll interval; 10min grace catches poll failures before they go stale in the panel digestNotifications: { key: 'seed-meta:digest:last-run', maxStaleMin: 90 }, // Railway digest-notifications cron runs every 30min; 90 = 3x cadence and detects a dead cron before daily digests are missed. forecasts: { key: 'seed-meta:forecast:predictions', maxStaleMin: 90 }, forecastsBootstrap: { key: 'seed-meta:forecast:predictions-bootstrap', maxStaleMin: 90 }, // Same cron. Monitored separately: the fast tier now hydrates from the dashboard list, and a transform/write failure there must not hide behind a healthy canonical key (#5300). forecastResolutions: { key: 'seed-meta:forecast:resolutions', maxStaleMin: 2160 }, // daily Bet-2 resolver; 36h catches a missed cron without flapping on normal daily jitter forecastScorecard: { key: 'seed-meta:forecast:scorecard', maxStaleMin: 2160 }, // scorecard extra key written by seed-forecast-resolutions forecastBets: { key: 'seed-meta:forecast:bets', maxStaleMin: 2880 }, // #5233 shadow bet-engine seeder; daily cron (05:00 UTC), 48h = 2× interval forecastMarketsResolution: { key: 'seed-meta:prediction:markets-resolution', maxStaleMin: 2160 }, // #5525 market settlement feed; written every resolver run (even zero-due), so it shares the resolver's 36h window forecastFunnel: { key: 'seed-meta:forecast:funnel:health:v1', maxStaleMin: 180 }, // funnel-diversity guardrail (#5233); written by seed-forecasts afterPublish each hourly run (3× cadence). status:'error' → SEED_ERROR when the published funnel collapses (too few domains / mostly synthetic) sectors: { key: 'seed-meta:market:sectors', maxStaleMin: 30 }, techReadiness: { key: 'seed-meta:economic:worldbank-techreadiness:v1', maxStaleMin: 10080 }, progressData: { key: 'seed-meta:economic:worldbank-progress:v1', maxStaleMin: 10080 }, renewableEnergy: { key: 'seed-meta:economic:worldbank-renewable:v1', maxStaleMin: 10080 }, intlDelays: { key: 'seed-meta:aviation:intl', maxStaleMin: 90 }, // faaDelays shares seed-meta key with flightDelays — no duplicate entry needed here theaterPosture: { key: 'seed-meta:theater-posture', maxStaleMin: 60 }, correlationCards: { key: 'seed-meta:correlation:cards', maxStaleMin: 30 }, // 5min cron (seed-bundle-derived-signals); 30min = 6× interval. Was 15 (3× = gold-standard floor) — overnight UptimeRobot flips when bundle jitter spaced two consecutive runs ~9-10min apart, producing 15-19min gaps that tripped STALE_SEED briefly. See WM 2026-05-10 health:failure-log. portwatch: { key: 'seed-meta:supply_chain:portwatch', maxStaleMin: 720 }, portwatchDisruptions: { key: 'seed-meta:portwatch:disruptions', maxStaleMin: 150 }, portwatchPortActivity: { key: 'seed-meta:supply_chain:portwatch-ports', maxStaleMin: 2160, minRecordCount: 174 }, // 12h cron; 36h = 3x interval; #3613 requires full 174-country coverage before OK. corridorrisk: { key: 'seed-meta:supply_chain:corridorrisk', maxStaleMin: 120 }, chokepointTransits: { key: 'seed-meta:supply_chain:chokepoint_transits', maxStaleMin: 30 }, // relay every 10min; 30min = 3x interval, transitSummaries: { key: 'seed-meta:supply_chain:transit-summaries', maxStaleMin: 30 }, // relay every 10min; 30min = 3x interval, usniFleet: { key: 'seed-meta:military:usni-fleet', maxStaleMin: 720 }, // relay loop every 6h; 720 = 2× interval (was 480 = 1.3×, too tight) securityAdvisories: { key: 'seed-meta:intelligence:advisories', maxStaleMin: 120 }, secCikMap: { key: 'seed-meta:intelligence:sec-cik-map', maxStaleMin: 2880, minRecordCount: 5000 }, // daily bundle section; 2880min = 48h = 2x interval. minRecordCount mirrors MIN_CIK_ENTRIES in scripts/seed-sec-cik-map.mjs. sec8kStream: { key: 'seed-meta:intelligence:sec-8k-stream', maxStaleMin: 120, minRecordCount: 50 }, // 30min bundle section; 120min = 4x interval. minRecordCount mirrors MIN_STREAM_EVENTS in scripts/seed-sec-8k-stream.mjs — a drained window means Atom-parse decay, not a quiet market. customsRevenue: { key: 'seed-meta:trade:customs-revenue', maxStaleMin: 1440 }, comtradeFlows: { key: 'seed-meta:trade:comtrade-flows', maxStaleMin: 2880 }, // 24h cron; 2880min = 48h = 2x interval comtradeBilateralHs4: { key: 'seed-meta:comtrade:bilateral-hs4', maxStaleMin: 50400, minRecordCount: 110 }, // 35d health budget for monthly seed; 40d payload/meta TTL leaves a 5d stale-but-queryable warning window. minRecordCount mirrors MIN_COUNTRY_COVERAGE in scripts/seed-comtrade-bilateral-hs4.mjs (110 of 197 clusters) so a shrunken run reads COVERAGE_PARTIAL, not OK — without it 3-of-197 and 197-of-197 were indistinguishable for the full 35d window. blsSeries: { key: 'seed-meta:economic:bls-series', maxStaleMin: 2880 }, // daily seed; 2880min = 48h = 2x interval sanctionsPressure: { key: 'seed-meta:sanctions:pressure', maxStaleMin: 720 }, crossSourceSignals: { key: 'seed-meta:intelligence:cross-source-signals', maxStaleMin: 30 }, // 15min cron; 30min = 2x interval regionalSnapshots: { key: 'seed-meta:intelligence:regional-snapshots', maxStaleMin: 720 }, // 6h cron via seed-bundle-derived-signals; 720min = 12h = 2x interval regionalBriefs: { key: 'seed-meta:intelligence:regional-briefs', maxStaleMin: 20160 }, // weekly cron; 20160min = 14 days = 2x interval sanctionsEntities: { key: 'seed-meta:sanctions:entities', maxStaleMin: 1440 }, // 12h cron; 1440min = 24h = 2x interval radiationWatch: { key: 'seed-meta:radiation:observations', maxStaleMin: 30 }, groceryBasket: { key: 'seed-meta:economic:grocery-basket', maxStaleMin: 10080 }, // weekly seed; 10080 = 7 days bigmac: { key: 'seed-meta:economic:bigmac', maxStaleMin: 10080 }, // weekly seed; 10080 = 7 days fuelPrices: { key: 'seed-meta:economic:fuel-prices', maxStaleMin: 10080 }, // weekly seed; 10080 = 7 days faoFoodPriceIndex: { key: 'seed-meta:economic:fao-ffpi', maxStaleMin: 86400 }, // monthly seed; 86400 = 60 days (2x interval) thermalEscalation: { key: 'seed-meta:thermal:escalation', maxStaleMin: 360 }, // cron every 2h; 360 = 3x interval (was 240 = 2x) thermalEscalationBootstrap: { key: 'seed-meta:thermal:escalation-bootstrap', maxStaleMin: 360 }, // Same cron as above. Monitored separately because the bootstrap tier now hydrates from the compact projection, and a transform/write failure there must not hide behind a healthy canonical key (#5300). nationalDebt: { key: 'seed-meta:economic:national-debt', maxStaleMin: 86400 }, // monthly seed (seed-bundle-macro intervalMs: 30 * DAY); 60d = 2x interval absorbs one missed run. Prior 10080 (7d) was narrower than the cron interval so every cron past day 7 alarmed STALE_SEED. tariffTrendsUs: { key: 'seed-meta:trade:tariffs:v1:840:all:10', maxStaleMin: 540 }, // co-pinned to TARIFF_TTL (8h=480min) + 60min grace. Prior 900 (15h) created an 8h-15h silent window where data had expired but seed-meta was still considered fresh, masking real outages as status=EMPTY (not STALE_SEED). See scripts/seed-supply-chain-trade.mjs TARIFF_TTL. // publish.ts runs once daily (02:30 UTC); seed-meta TTL=52h — maxStaleMin must cover the full 24h cycle consumerPricesOverview: { key: 'seed-meta:consumer-prices:overview:ae', maxStaleMin: 1500 }, // 25h = 24h cadence + 1h grace consumerPricesCategories: { key: 'seed-meta:consumer-prices:categories:ae:30d', maxStaleMin: 1500 }, consumerPricesMovers: { key: 'seed-meta:consumer-prices:movers:ae:30d', maxStaleMin: 1500 }, consumerPricesSpread: { key: 'seed-meta:consumer-prices:retailer-spread:ae:essentials-ae', maxStaleMin: 1500 }, consumerPricesFreshness: { key: 'seed-meta:consumer-prices:freshness:ae', maxStaleMin: 1500 }, // defiTokens/aiTokens/otherTokens all share one seed run (seed-token-panels cron, every 30min) defiTokens: { key: 'seed-meta:market:token-panels', maxStaleMin: 90 }, aiTokens: { key: 'seed-meta:market:token-panels', maxStaleMin: 90 }, otherTokens: { key: 'seed-meta:market:token-panels', maxStaleMin: 90 }, fredBatch: { key: 'seed-meta:economic:fred:v1:FEDFUNDS:0', maxStaleMin: 1500 }, // daily cron forecastFredWalcl: { key: 'seed-meta:economic:fred:v1:WALCL:0', maxStaleMin: 1500 }, forecastFredT10y2y: { key: 'seed-meta:economic:fred:v1:T10Y2Y:0', maxStaleMin: 1500 }, forecastFredUnrate: { key: 'seed-meta:economic:fred:v1:UNRATE:0', maxStaleMin: 1500 }, forecastFredCpiaucsl: { key: 'seed-meta:economic:fred:v1:CPIAUCSL:0', maxStaleMin: 1500 }, forecastFredDgs10: { key: 'seed-meta:economic:fred:v1:DGS10:0', maxStaleMin: 1500 }, forecastFredVixcls: { key: 'seed-meta:economic:fred:v1:VIXCLS:0', maxStaleMin: 1500 }, forecastFredGdp: { key: 'seed-meta:economic:fred:v1:GDP:0', maxStaleMin: 1500 }, forecastFredM2sl: { key: 'seed-meta:economic:fred:v1:M2SL:0', maxStaleMin: 1500 }, forecastFredDcoilwtico: { key: 'seed-meta:economic:fred:v1:DCOILWTICO:0', maxStaleMin: 1500 }, ecbEstr: { key: 'seed-meta:economic:ecb-short-rates', maxStaleMin: 4320 }, // daily ECB publish; 4320min = 3d = TTL/interval ecbEuribor3m: { key: 'seed-meta:economic:ecb-short-rates', maxStaleMin: 4320 }, // shared meta key with ecbEstr ecbEuribor6m: { key: 'seed-meta:economic:ecb-short-rates', maxStaleMin: 4320 }, // shared meta key with ecbEstr ecbEuribor1y: { key: 'seed-meta:economic:ecb-short-rates', maxStaleMin: 4320 }, // shared meta key with ecbEstr gscpi: { key: 'seed-meta:economic:gscpi', maxStaleMin: 2880 }, // 24h interval; 2880min = 48h = 2x interval fearGreedIndex: { key: 'seed-meta:market:fear-greed', maxStaleMin: 720 }, // 6h cron; 720min = 12h = 2x interval breadthHistory: { key: 'seed-meta:market:breadth-history', maxStaleMin: 5760 }, // cron at 02:00 UTC, Tue-Sat (captures Mon-Fri market close); max gap Sat→Tue = 72h + 24h miss buffer = 96h = 5760min. 48h was wrong — alarmed every Monday morning when Sun+Mon are intentionally skipped. hormuzTracker: { key: 'seed-meta:supply_chain:hormuz_tracker', maxStaleMin: 2880 }, // daily cron; 2880min = 48h = 2x interval earningsCalendar: { key: 'seed-meta:market:earnings-calendar', maxStaleMin: 1440 }, // 12h cron; 1440min = 24h = 2x interval econCalendar: { key: 'seed-meta:economic:econ-calendar', maxStaleMin: 1440 }, // 12h cron; 1440min = 24h = 2x interval cotPositioning: { key: 'seed-meta:market:cot', maxStaleMin: 14400 }, // weekly CFTC release; 14400min = 10d = 1.4x interval (weekend + delay buffer) hyperliquidFlow: { key: 'seed-meta:market:hyperliquid-flow', maxStaleMin: 30 }, // 5min cron (bundled in seed-bundle-market-backup); 30min = 6× interval. Was 15 (3× = gold-standard floor with zero safety margin) — same exposure as correlationCards. Pre-fix comment said "Railway cron" but it's bundle-driven, so subject to the 80% skip-gate that produced 9-19min jitter under load. crudeInventories: { key: 'seed-meta:economic:crude-inventories', maxStaleMin: 20160 }, // weekly EIA data; 20160min = 14 days = 2x weekly cadence natGasStorage: { key: 'seed-meta:economic:nat-gas-storage', maxStaleMin: 20160 }, // weekly EIA data; 20160min = 14 days = 2x weekly cadence spr: { key: 'seed-meta:economic:spr', maxStaleMin: 20160 }, // weekly EIA data; 20160min = 14 days = 2x weekly cadence refineryInputs: { key: 'seed-meta:economic:refinery-inputs', maxStaleMin: 20160 }, // weekly EIA data; 20160min = 14 days = 2x weekly cadence ecbFxRates: { key: 'seed-meta:economic:ecb-fx-rates', maxStaleMin: 5760 }, // daily seed (weekdays + holidays); 5760min = 96h = covers Wed→Mon Easter gap eurostatCountryData: { key: 'seed-meta:economic:eurostat-country-data', maxStaleMin: 4320 }, // daily seed; 4320min = 3 days = 3x interval eurostatHousePrices: { key: 'seed-meta:economic:eurostat-house-prices', maxStaleMin: 60 * 24 * 50 }, // weekly cron, annual data; 50d threshold = 35d TTL + 15d buffer eurostatGovDebtQ: { key: 'seed-meta:economic:eurostat-gov-debt-q', maxStaleMin: 60 * 24 * 14 }, // 2d cron, quarterly data; 14d threshold matches TTL + quarterly release drift eurostatIndProd: { key: 'seed-meta:economic:eurostat-industrial-production', maxStaleMin: 60 * 24 * 5 }, // daily cron, monthly data; 5d threshold matches TTL euGasStorage: { key: 'seed-meta:economic:eu-gas-storage', maxStaleMin: 2880 }, // daily seed (T+1); 2880min = 48h = 2x interval euYieldCurve: { key: 'seed-meta:economic:yield-curve-eu', maxStaleMin: 4320 }, // daily seed (weekdays only); 4320min = 72h = covers Fri→Mon gap euFsi: { key: 'seed-meta:economic:fsi-eu', maxStaleMin: 5760 }, // daily seed (weekdays + holidays); 5760min = 96h = covers Wed→Mon Easter gap. Data freshness is tracked separately via content-age (STALE_CONTENT) — see seed-fsi-eu.mjs. newsThreatSummary: { key: 'seed-meta:news:threat-summary', maxStaleMin: 60 }, // relay classify every ~20min; 60min = 3x interval shippingStress: { key: 'seed-meta:supply_chain:shipping_stress', maxStaleMin: 45 }, // relay loop every 15min; 45 = 3x interval (was 30 = 2×, too tight on relay hiccup) diseaseOutbreaks: { key: 'seed-meta:health:disease-outbreaks', maxStaleMin: 2880 }, // daily seed; 2880 = 48h = 2x interval healthAirQuality: { key: 'seed-meta:health:air-quality', maxStaleMin: 180 }, // hourly cron; 180 = 3x interval for shared health/climate seed socialVelocity: { key: 'seed-meta:intelligence:social-reddit', maxStaleMin: 540 }, // relay loop every 180min (3h; was 60min, dropped now that ScrapeCreators handles Reddit); 540 = 3x interval. Co-pinned with SOCIAL_VELOCITY_TTL=43200 (ais-relay.cjs): the data-key TTL must STRICTLY exceed this (720min > 540min) so a dead relay shows STALE_SEED before the key expires to EMPTY. wsbTickers: { key: 'seed-meta:intelligence:wsb-tickers', maxStaleMin: 540 }, // relay loop every 180min (3h); 540 = 3x interval. Co-pinned with WSB_TICKERS_TTL=43200 (ais-relay.cjs); TTL strictly > maxStaleMin (see socialVelocity note). pizzint: { key: 'seed-meta:intelligence:pizzint', maxStaleMin: 30 }, // relay loop every 10min; 30 = 3x interval productCatalog: { key: 'seed-meta:product-catalog', maxStaleMin: 1080 }, // relay loop every 6h; 1080 = 18h = 3x interval vpdTrackerRealtime: { key: 'seed-meta:health:vpd-tracker', maxStaleMin: 2880 }, // daily seed (0 2 * * *); 2880min = 48h = 2x interval vpdTrackerHistorical: { key: 'seed-meta:health:vpd-tracker', maxStaleMin: 2880 }, // shares seed-meta key with vpdTrackerRealtime (same run) resilienceStaticIndex: { key: 'seed-meta:resilience:static', maxStaleMin: 576000 }, // annual October snapshot; 400d threshold matches TTL and preserves prior-year data on source outages resilienceStaticFao: { key: 'seed-meta:resilience:static', maxStaleMin: 576000 }, // same seeder + same heartbeat as resilienceStaticIndex; required so EMPTY_DATA_OK + missing data degrades to STALE_SEED instead of silent OK resilienceRanking: { key: 'seed-meta:resilience:ranking', maxStaleMin: 840 }, // RPC cache (12h TTL, refreshed every 6h by seed-resilience-scores cron); 14h budget tolerates 1 missed tick (12h gap) + ~2h jitter for in-flight deploys that preempt a scheduled tick; alerts at 2 missed ticks (18h gap). Bumped from 720 — see comment below. resilienceIntervals: { key: 'seed-meta:resilience:intervals', maxStaleMin: 840 }, // bundled into seed-bundle-resilience, written by the Resilience-Scores section. Real Railway cron is `0 */6 * * *` (every 6h on the hour, UTC) — empirically verified 2026-04-28 via Railway logs showing 6h gaps between successful runs (the prior `intervalMs=2h with hourly fires` claim did not match what's deployed; either the bundle interval gate or the Railway service schedule makes the effective cadence 6h). 840 = 14h staleness ≈ 2.33× cadence: tolerates 1 missed tick (12h gap) + ~2h jitter for in-flight deploys; alerts at 2 missed ticks (18h gap). Matches resilienceRanking above (same Resilience-Scores section writes both). Prior values: 20160 (14d, 168× — silent on real outage), 1080 (18h, 3× — over-permissive: masks a 12h outage), 720 (12h, 2× — exact floor; flipped UptimeRobot WARNING for ~1min on every Railway-deploy-preempted tick: 2026-05-10 incident at 18:02 UTC, seedAgeMin=722 vs maxStale=720 after the 12:00 UTC tick was skipped during an in-flight deploy), 360 (1× — false-positive on routine jitter, 2026-04-28: seedAgeMin=367 vs maxStale=360). Re-tighten ONLY if/when the actual Railway cron schedule is verified sub-6h. energyExposure: { key: 'seed-meta:economic:owid-energy-mix', maxStaleMin: 50400 }, // monthly cron on 1st; 50400min = 35d = TTL matches cron cadence + 5d buffer energyMixAll: { key: 'seed-meta:economic:owid-energy-mix', maxStaleMin: 50400 }, // same seed run as energyExposure; shares seed-meta key regulatoryActions: { key: 'seed-meta:regulatory:actions', maxStaleMin: 360 }, // 2h cron; 360min = 3x interval energySpineCountries: { key: 'seed-meta:energy:spine', maxStaleMin: 2880 }, // daily cron (06:00 UTC); 2880min = 48h = 2x interval electricityPrices: { key: 'seed-meta:energy:electricity-prices', maxStaleMin: 2880 }, // daily cron (14:00 UTC); 2880min = 48h = 2x interval gasStorageCountries: { key: 'seed-meta:energy:gas-storage-countries', maxStaleMin: 2880 }, // daily cron at 10:30 UTC; 2880min = 48h = 2x interval energyIntelligence: { key: 'seed-meta:energy:intelligence', maxStaleMin: 720 }, // 6h cron; 720min = 2x interval jodiOil: { key: 'seed-meta:energy:jodi-oil', maxStaleMin: 60 * 24 * 40 }, // monthly cron on 25th; 40d threshold matches 35d TTL + 5d buffer ieaOilStocks: { key: 'seed-meta:energy:iea-oil-stocks', maxStaleMin: 60 * 24 * 40 }, // monthly cron on 15th; 40d threshold = TTL_SECONDS oilStocksAnalysis: { key: 'seed-meta:energy:oil-stocks-analysis', maxStaleMin: 60 * 24 * 50 }, // afterPublish of ieaOilStocks; 50d = matches seed-meta TTL (exceeds 40d data TTL) eiaPetroleum: { key: 'seed-meta:energy:eia-petroleum', maxStaleMin: 4320 }, // daily bundle cron (seed-bundle-energy-sources); 72h = 3× interval, well under 7d data TTL jodiGas: { key: 'seed-meta:energy:jodi-gas', maxStaleMin: 60 * 24 * 40 }, // monthly cron on 25th; 40d threshold matches 35d TTL + 5d buffer lngVulnerability: { key: 'seed-meta:energy:jodi-gas', maxStaleMin: 60 * 24 * 40 }, // written by jodi-gas seeder afterPublish; shares seed-meta key chokepointBaselines: { key: 'seed-meta:energy:chokepoint-baselines', maxStaleMin: 60 * 24 * 400 }, // 400 days sprPolicies: { key: 'seed-meta:energy:spr-policies', maxStaleMin: 60 * 24 * 400 }, // 400 days; static registry, same cadence as chokepoint baselines pipelinesGas: { key: 'seed-meta:energy:pipelines-gas', maxStaleMin: 20_160 }, // 14d — weekly cron (7d) × 2 headroom pipelinesOil: { key: 'seed-meta:energy:pipelines-oil', maxStaleMin: 20_160 }, // 14d — same seed-pipelines.mjs publishes both keys storageFacilities: { key: 'seed-meta:energy:storage-facilities', maxStaleMin: 20_160 }, // 14d — weekly cron (7d) × 2 headroom fuelShortages: { key: 'seed-meta:energy:fuel-shortages', maxStaleMin: 2880 }, // 2d — daily cron × 2 headroom (classifier-driven post-launch) energyDisruptions: { key: 'seed-meta:energy:disruptions', maxStaleMin: 20_160 }, // 14d — weekly cron × 2 headroom energyCrisisPolicies: { key: 'seed-meta:energy:crisis-policies', maxStaleMin: 60 * 24 * 400 }, // static data, ~400d TTL matches seeder aaiiSentiment: { key: 'seed-meta:market:aaii-sentiment', maxStaleMin: 20160 }, // weekly cron; 20160min = 14 days = 2x weekly cadence portwatchChokepointsRef: { key: 'seed-meta:portwatch:chokepoints-ref', maxStaleMin: 60 * 24 * 14 }, // seed-bundle-portwatch runs this at WEEK cadence; 14d = 2× interval chokepointFlows: { key: 'seed-meta:energy:chokepoint-flows', maxStaleMin: 720 }, // 6h cron; 720min = 2x interval // Relay-side heartbeat written by ais-relay.cjs on successful child exit. // Detects "relay loop fires but child dies at import/runtime" failures // (e.g. ERR_MODULE_NOT_FOUND from a missing Dockerfile COPY) 4h earlier // than the 720min seed-meta threshold above. TTL is 18h on the writer. chokepointFlowsRelayHeartbeat: { key: 'relay:heartbeat:chokepoint-flows', maxStaleMin: 480 }, // 6h loop; 8h alarm climateNewsRelayHeartbeat: { key: 'relay:heartbeat:climate-news', maxStaleMin: 60 }, // 30m loop; 60m alarm emberElectricity: { key: 'seed-meta:energy:ember', maxStaleMin: 2880 }, // daily cron (08:00 UTC); 2880min = 48h = 2x interval cryptoSectors: { key: 'seed-meta:market:crypto-sectors', maxStaleMin: 120 }, // relay loop every ~30min; 120min = 2h = 4x interval ddosAttacks: { key: 'seed-meta:cf:radar:ddos', maxStaleMin: 60 }, // written by seed-internet-outages afterPublish; outages cron ~15min; 60 = 4x interval economicStress: { key: 'seed-meta:economic:stress-index', maxStaleMin: 180 }, // computed in seed-economy afterPublish; cron ~1h; 180min = 3x interval marketImplications: { key: 'seed-meta:intelligence:market-implications', maxStaleMin: 120 }, // LLM-generated in seed-forecasts; cron ~1h; 120min = 2x interval trafficAnomalies: { key: 'seed-meta:cf:radar:traffic-anomalies', maxStaleMin: 60 }, // written by seed-internet-outages afterPublish; outages cron ~15min; 60 = 4x interval chokepointExposure: { key: 'seed-meta:supply_chain:chokepoint-exposure', maxStaleMin: 2880 }, // daily cron; 2880min = 48h = 2x interval recoveryFiscalSpace: { key: 'seed-meta:resilience:recovery:fiscal-space', maxStaleMin: 129600 }, // monthly cron; 129600min = 90d = 3x interval (bumped from 86400/60d = 2x in PR #3669 for month-2 hiccup margin) recoveryReserveAdequacy: { key: 'seed-meta:resilience:recovery:reserve-adequacy', maxStaleMin: 86400 }, // monthly cron; 86400min = 60d = 2x interval recoveryExternalDebt: { key: 'seed-meta:resilience:recovery:external-debt', maxStaleMin: 86400 }, // monthly cron; 86400min = 60d = 2x interval recoveryImportHhi: { key: 'seed-meta:resilience:recovery:import-hhi', maxStaleMin: 50400 }, // monthly cron; 50400min = 35d catches missed import-HHI runs before stale country coverage lingers for 46d+ // recoveryFuelStocks: probe removed (PR #3764). See STANDALONE_KEYS comment. recoveryReexportShare: { key: 'seed-meta:resilience:recovery:reexport-share', maxStaleMin: 86400 }, // monthly cron; 86400min = 60d = 2x interval recoverySovereignWealth: { key: 'seed-meta:resilience:recovery:sovereign-wealth', maxStaleMin: 86400 }, // monthly cron; 86400min = 60d = 2x interval // PR 1 v2 energy seeds — weekly cron (8d * 1440 = 11520min = 2x interval). // STRICT SEED_META (not ON_DEMAND): plan 2026-04-24-001 made /api/health // CRIT on absent/stale so operators see the Railway-bundle gap before // the flag flips. See the ON_DEMAND_KEYS "do not add back" note below. lowCarbonGeneration: { key: 'seed-meta:resilience:low-carbon-generation', maxStaleMin: 11520 }, fossilElectricityShare: { key: 'seed-meta:resilience:fossil-electricity-share', maxStaleMin: 11520 }, powerLosses: { key: 'seed-meta:resilience:power-losses', maxStaleMin: 11520 }, webcams: { key: 'seed-meta:webcam:cameras:geo', maxStaleMin: 1440 }, // seed-webcams writes 24h geo/meta keys plus a 30h active pointer; stale at 24h before the layer goes blank. // #5736 — history-ingest freshness per collector. `fetchedAt` here is the // last HEALTHY append (success, or a correctly-detected unconfigured run), // NEVER the last attempt, so a relay that rejects every chunk ages this out // while the collector's own seed-meta stays fresh. Budgets mirror each // collector's canonical maxStaleMin above — the hook runs once per successful // publish, so a tighter budget would alarm on the parent's cadence, not on // history. Keep in step with api/seed-health.js (intervalMin = maxStaleMin/2). intelHistoryIngestConflictAcled: { key: 'seed-meta:intel-history:conflict:acled-intel', maxStaleMin: 38 }, // mirrors acledIntel intelHistoryIngestMilitaryCrossStrait: { key: 'seed-meta:intel-history:military:cross-strait-activity', maxStaleMin: 720 }, // mirrors crossStraitActivity intelHistoryIngestEnergyIntelligence: { key: 'seed-meta:intel-history:energy:intelligence', maxStaleMin: 720 }, // mirrors energyIntelligence }; // Iran-events sunset: when disabled (default), drop it from all health // classification so the deliberately-dormant manual seed can't raise // STALE_SEED (present-but-stale) or EMPTY (absent). Re-enabling restores both. if (!IRAN_EVENTS_ENABLED) { delete BOOTSTRAP_KEYS.iranEvents; delete SEED_META.iranEvents; } // Standalone keys that are populated on-demand by RPC handlers (not seeds). // Empty = WARN not CRIT since they only exist after first request. // // POLICY (2026-05-01): If the seed-meta key feeds a panel that renders on the // DEFAULT homepage layout (`enabled: true, priority: 1` in src/config/panels.ts // or per-variant equivalents), it MUST NOT be in this set. ON_DEMAND softens // EMPTY to WARN, which is correct ONLY when data is genuinely populated lazily // after a user action (premium RPC caches that warm on click, intermediate // seed-to-seed pipeline keys, relay heartbeats, click-warmed lookups). For a // homepage panel, chronic absence is a real outage and deserves CRIT — softening // it masks production breakage behind an OK summary. // // Specific incident that motivated this policy: marketImplications (homepage // panel, default-enabled in panels.ts:114) sat at age=988 max=120 (8.2× the // staleness budget) for 16+ hours while the LLM provider returned HTTP 402 on // every cron run. /api/health stayed onDemandWarn=1 instead of crit, so the // chronic outage went undetected until a user noticed the panel was stuck on // "Loading...". Removed marketImplications below. const ON_DEMAND_KEYS = new Set([ // Deployment-order bridge: Vercel can ship this reader before Railway's // hourly bundle publishes the first summary. The seeder writes a durable // activation marker after its first successful publish; after that, a // missing/stale summary is strict forever. 'chinaCoverage', 'riskScoresLive', 'usniFleetStale', 'positiveEventsLive', 'bisPolicy', 'bisExchange', 'bisCredit', // bisDsr/bisPropertyResidential/bisPropertyCommercial have dedicated SEED_META // entries (seed-bis-extended.mjs), so they are not on-demand. // shippingRates is a scheduled 6h producer and FAST bootstrap key; // absence must be EMPTY/CRIT rather than on-demand. 'macroSignals', 'chokepoints', 'minerals', 'giving', 'cyberThreatsRpc', 'militaryBases', 'displacement', 'corridorrisk', // intermediate key; data flows through transit-summaries:v1 'serviceStatuses', // RPC-populated; seed-meta written on fresh fetch only, goes stale between visits 'militaryForecastInputs', // intermediate seed-to-seed pipeline key; only populated after seed-military-flights runs // marketImplications removed 2026-05-01 — see policy block above. Homepage panel, // chronic LLM-provider failures must surface as CRIT. 'simulationPackageLatest', // written by writeSimulationPackage after deep forecast runs; only present after first successful deep run 'simulationOutcomeLatest', // written by writeSimulationOutcome after simulation runs; only present after first successful simulation // #4927 review P1: activation-gated on the operator adding UPSTASH_* as GH // Actions secrets — the publishers skip silently without them, so a // never-activated key must read as soft EMPTY_ON_DEMAND, not EMPTY/CRIT, // while the workflow stays green. On-demand softening is REVOKED once the // durable activation marker exists (see ACTIVATION_MARKERS): after the // first publish, missing data/meta is EMPTY/STALE_SEED like any other key. 'newsFeedHealth', 'newsRecallBenchmark', 'newsThreatSummary', // relay classify loop — only written when mergedByCountry has entries; absent on quiet news periods 'resilienceRanking', // on-demand RPC cache populated after ranking requests; missing before first Pro use is expected 'recoveryFiscalSpace', 'recoveryReserveAdequacy', 'recoveryExternalDebt', // NOTE (2026-04-24, plan 2026-04-24-001): the PR 1 v2 energy seeds // (`lowCarbonGeneration`, `fossilElectricityShare`, `powerLosses`) // are INTENTIONALLY NOT listed in ON_DEMAND_KEYS. They stay strict // SEED_META so `/api/health` returns CRIT (not WARN) when they are // absent — which is the alarm a future operator needs before flipping // `RESILIENCE_ENERGY_V2_ENABLED=true`. The scorer fails closed via // ResilienceConfigurationError if the flag flips before the seeds // populate (server/worldmonitor/resilience/v1/_dimension-scorers.ts // #scoreEnergy). Do NOT add these labels back to ON_DEMAND_KEYS // without revisiting that plan. 'displacementPrev', // covered by cascade onto current-year displacement; empty most of the year 'fxYoy', // TRANSITIONAL (PR #3071): seed-fx-yoy Railway cron deployed manually after merge — // gate as on-demand so a deploy-order race or first-cron-run failure doesn't // fire a CRIT health alarm. Remove from this set after ~7 days of clean // production cron runs (verify via `seed-meta:economic:fx-yoy.fetchedAt`). 'hyperliquidFlow', // TRANSITIONAL: seed-hyperliquid-flow runs inside seed-bundle-market-backup on // Railway; gate as on-demand so initial deploy-order race or first cold-start // snapshot doesn't CRIT. Remove after ~7 days of clean production cron runs. 'chokepointFlowsRelayHeartbeat', // TRANSITIONAL (PR #3133): ais-relay.cjs writes this on the // first successful child exit after a deploy. Vercel deploys // api/health.js instantly, but Railway rebuild + 6h initial // loop interval means the key is absent for up to ~6h post-merge. // Gate as on-demand so the deploy window doesn't CRIT. Remove // after ~7 days of clean production runs (verify via // `relay:heartbeat:chokepoint-flows.fetchedAt`). 'climateNewsRelayHeartbeat', // TRANSITIONAL (PR #3133): same deploy-order rationale. // 30min initial loop, so window is shorter but still present. // Remove after ~7 days alongside the chokepoint-flows entry. 'digestNotifications', // TRANSITIONAL (PR #4253): seed-digest-notifications.mjs writes // `digest:last-run` on the first cron run after deploy. Vercel // can publish this health registry before Railway's 30min cron // ticks, so gate only the first absent-key window as WARN. Remove // after ~7 days of clean `seed-meta:digest:last-run` writes. 'eiaPetroleum', // TRANSITIONAL: gold-standard migration of /api/eia/petroleum // from live Vercel fetch to Redis-reader (seed-bundle-energy-sources // daily cron). SEED_META entry above enforces 72h staleness — this // ON_DEMAND gate only softens the absent-on-deploy case (Vercel // deploys instantly; Railway EIA_API_KEY + first daily tick ~24h // behind). STALE_SEED still fires if data goes stale after first seed. // Remove from this set after ~7 days of clean cron runs so // never-provisioned Railway promotes EMPTY_ON_DEMAND → EMPTY (CRIT). // #5736 deployment-order bridge, activation-gated like chinaCoverage: Vercel // ships this registry the moment the PR merges, but the record only exists // after the collector's next Railway tick (up to 6h for energy/intelligence). // Softening is revoked the instant the durable marker appears — after the // first report, an absent record is EMPTY and a stalled one is STALE_SEED. 'intelHistoryIngestConflictAcled', 'intelHistoryIngestMilitaryCrossStrait', 'intelHistoryIngestEnergyIntelligence', ]); // Legacy broad empty-data exemptions. classifyKey uses this set in both the // missing-key and zero-record branches, so entries here can be OK while fresh // even when the data key has strlen=0. For producer-specific cases where the // payload must exist but recordCount=0 is valid, add to ZERO_RECORD_DATA_OK_KEYS // only. // #4927 re-review P1: "has ever published" must survive the 7d seed-meta // TTL — inferring activation from meta existence meant a publisher that ran // once and died read as healthy pending-activation again after the meta // expired. Publishers SET these markers with NO TTL on every successful // publish; when the marker exists the key leaves ON_DEMAND softening and // normal EMPTY/STALE_SEED rules apply. const ACTIVATION_MARKERS = { chinaCoverage: 'seed-activated:health:china-coverage', newsFeedHealth: 'seed-activated:news:feed-health', newsRecallBenchmark: 'seed-activated:news:recall-benchmark', // Written by scripts/_seed-history.mjs on every ingest-health report, // including an `unconfigured` one — "the hook ran" is the activation signal, // not "the relay answered". See historyIngestActivationKey there. intelHistoryIngestConflictAcled: 'seed-activated:intel-history:conflict:acled-intel', intelHistoryIngestMilitaryCrossStrait: 'seed-activated:intel-history:military:cross-strait-activity', intelHistoryIngestEnergyIntelligence: 'seed-activated:intel-history:energy:intelligence', }; const EMPTY_DATA_OK_KEYS = new Set([ 'notamClosures', 'faaDelays', 'intlDelays', 'gpsjam', 'positiveGeoEvents', 'weatherAlerts', 'earningsCalendar', 'econCalendar', 'cotPositioning', 'usniFleet', // usniFleetStale covers the fallback; relay outages → WARN not CRIT 'newsThreatSummary', // only written when classify produces country matches; quiet news periods = 0 countries, no write 'recoveryFiscalSpace', 'ddosAttacks', 'trafficAnomalies', // zero events during quiet periods is valid, not critical 'resilienceStaticFao', // empty aggregate = no IPC Phase 3+ countries this year (possible in theory); the key must exist but count=0 is fine 'cableHealth', // `cables: {}` = no active subsea cable disruptions per NGA NAVAREA warnings — all cables implicitly healthy. Also covers NGA-upstream-down windows where get-cable-health writes back the fallback response (empty cables); without this, those would alarm EMPTY_DATA. 'forecastBets', // #5233 shadow bet-engine stream; absent before the cron ships it and empty on weeks the energy feed yields no bet — tolerate as STALE_SEED (warn), not EMPTY (crit). 'forecastFunnel', // #5233 funnel guardrail is a new afterPublish side-write; before the first seed-forecasts run ships it the key is absent — tolerate as STALE_SEED (warn), not EMPTY (crit). A COLLAPSED funnel still surfaces via seed-meta status:'error' → SEED_ERROR, which classifyKey checks before this branch. // Compact projections stay STALE_SEED before their first producer tick or // after metadata turns stale. Fresh metadata plus a missing payload is // deliberately strict: MISSING_DATA_IS_FAILURE_KEYS reports EMPTY (crit). 'thermalEscalationBootstrap', 'ucdpEventsBootstrap', 'forecastsBootstrap', 'wildfiresBootstrap', 'crossStraitActivityBootstrap', ]); // These compact projections must leave a payload on every successful publish. // This is deliberately narrower than EMPTY_DATA_OK_KEYS: DDoS, traffic, and // weather refresh only their seed metadata during quiet periods, so an absent // payload is valid for those sources. Every entry here must also be in // EMPTY_DATA_OK_KEYS so a pre-first-publish absence remains STALE_SEED rather // than a false-critical EMPTY; tests/health-empty-data-ok.test.mjs enforces it. const MISSING_DATA_IS_FAILURE_KEYS = new Set([ // These sparse feeds publish an explicit canonical payload on every // successful cycle, including valid zero-record cycles. Fresh metadata // therefore cannot excuse a vanished data key. 'cableHealth', 'notamClosures', 'thermalEscalationBootstrap', 'ucdpEventsBootstrap', 'wildfiresBootstrap', 'forecastsBootstrap', 'positiveGeoEvents', 'crossStraitActivityBootstrap', ]); // Keys where a present payload with meta recordCount=0 is valid, but the data // key itself must still exist. Do not use this set in the missing-key branch. const ZERO_RECORD_DATA_OK_KEYS = new Set([ ...EMPTY_DATA_OK_KEYS, 'globalTendersSam', 'globalTendersTed', 'globalTendersContractsFinder', 'globalTendersCanadaBuys', 'globalTendersGets', 'globalTendersWorldBank', // retailer-spread is SUPPRESSED to an explicit 0 by the aggregate job when a // market's retailers share < MIN_SPREAD_ITEMS (4) common basket items — // consumer-prices-core/src/jobs/aggregate.ts writes `retailer_spread_pct: 0` // ("prevent stale noisy value persisting") and logs "spread suppressed // (N/4 common items)". That is a valid data-coverage state, not a pipeline // outage: the AE basket's cross-retailer overlap shrank 2/4 → 1/4 → 0/4 over // late-May 2026 while the seeder kept running fresh (seedAgeMin well inside // maxStaleMin) and the sibling keys (overview/categories/movers/basket-series) // published normally. Treat 0 records as OK while fresh; STALE_SEED still // fires if the publish job itself stops. 'consumerPricesSpread', // CF Radar curated outage annotations (seed-internet-outages, zeroIsValid) // are sparse — most 28d windows publish an empty {outages:[]} envelope with // recordCount=0 (hasData=true). NARROW set, not EMPTY_DATA_OK_KEYS: the // seeder always publishes the array, so a MISSING canonical key is a real // publish failure → still EMPTY (crit). Siblings ddosAttacks/trafficAnomalies // sit in the broad set because their data key can be wholly absent on quiet // (writeSeedMeta-only path). 'outages', // Official disclosure categories are sparse. The canonical snapshot always // exists after a successful query, but a quiet 90-day window can validly // contain zero owned-category events. 'chinaCorporateDisclosures', // #5736 ingest-health records. `recordCount` is the volume the relay accepted // on the last successful append, and zero is legitimate — a run whose records // were all deduped, or one with no noteworthy records at all. The record key // itself must still exist, so this is the NARROW set: a vanished record is a // real gap, not a quiet period. 'intelHistoryIngestConflictAcled', 'intelHistoryIngestMilitaryCrossStrait', 'intelHistoryIngestEnergyIntelligence', ]); // Cascade groups: if any key in the group has data, all empty siblings are OK. // Theater posture uses live → stale → backup fallback chain. const CASCADE_GROUPS = { theaterPosture: ['theaterPosture', 'theaterPostureLive', 'theaterPostureBackup'], theaterPostureLive: ['theaterPosture', 'theaterPostureLive', 'theaterPostureBackup'], theaterPostureBackup: ['theaterPosture', 'theaterPostureLive', 'theaterPostureBackup'], militaryFlights: ['militaryFlights', 'militaryFlightsStale'], militaryFlightsStale: ['militaryFlights', 'militaryFlightsStale'], // Displacement key embeds UTC year — on Jan 1 the new-year key may be empty // for hours until the seed runs. Cascade onto the previous-year snapshot. displacement: ['displacement', 'displacementPrev'], displacementPrev: ['displacement', 'displacementPrev'], }; const NEG_SENTINEL = '__WM_NEG__'; function parseRedisValue(raw) { if (!raw || raw === NEG_SENTINEL) return null; try { return JSON.parse(raw); } catch { return raw; } } // Real data is always >0 bytes. The negative-cache sentinel is exactly // NEG_SENTINEL.length bytes (10), so any strlen > 0 that is NOT exactly that // length counts as data. The previous `> 10` heuristic misclassified // legitimately small payloads (`{}`, `[]`, `0`) as missing. function strlenIsData(strlen) { return strlen > 0 && strlen !== NEG_SENTINEL.length; } // Data keys whose Redis value is a LIST rather than a string. STRLEN against a // list returns WRONGTYPE, which lands in keyErrors and pins the key at // REDIS_PARTIAL forever even though its seeder is healthy — measure these with // LLEN instead. Presence is then simply `len > 0`: the NEG_SENTINEL // byte-length rule above is a STRING concept, and applying it to a list would // declare a 10-ELEMENT history empty. const LIST_DATA_KEYS = new Set([ STANDALONE_KEYS.forecastBets, // LPUSH/LTRIM — scripts/seed-forecast-bets.mjs ]); function dataLenCommand(redisKey) { return [LIST_DATA_KEYS.has(redisKey) ? 'LLEN' : 'STRLEN', redisKey]; } function keyHasData(redisKey, len) { return LIST_DATA_KEYS.has(redisKey) ? len > 0 : strlenIsData(len); } function readSeedMeta(seedCfg, keyMetaValues, keyMetaErrors, now) { if (!seedCfg) { return { hasMeta: false, seedAge: null, seedStale: null, seedError: false, sourceUnavailable: false, sourceBlocked: false, metaReadFailed: false, metaCount: null, contentAge: null }; } // Per-command Redis errors on the GET seed-meta half of the pipeline must // not silently fall through to STALE_SEED — promote to REDIS_PARTIAL. if (keyMetaErrors.get(seedCfg.key)) { return { hasMeta: false, seedAge: null, seedStale: null, seedError: false, sourceUnavailable: false, sourceBlocked: false, metaReadFailed: true, metaCount: null, contentAge: null }; } // Unwrap through the envelope helper. Legacy seed-meta is a bare // `{ fetchedAt, recordCount, sourceVersion, status? }` object with no `_seed` // wrapper, so `unwrapEnvelope` returns it as `.data` unchanged. PR 2 wires // true envelope reads at the canonical-key layer; this import establishes // the dependency so behavior stays byte-identical in PR 1. const meta = unwrapEnvelope(parseRedisValue(keyMetaValues.get(seedCfg.key))).data; const metaCount = meta?.count ?? meta?.recordCount ?? null; const errorCode = typeof meta?.errorCode === 'string' && /^[A-Z0-9_]{1,64}$/.test(meta.errorCode) ? meta.errorCode : null; if (meta?.status === 'error') { return { hasMeta: true, seedAge: null, seedStale: true, seedError: true, sourceUnavailable: false, sourceBlocked: false, metaReadFailed: false, metaCount, contentAge: null, errorCode, }; } let seedAge = null; let seedStale = true; const fetchedAt = Number(meta?.fetchedAt); if (Number.isFinite(fetchedAt) && fetchedAt > 0) { seedAge = Math.round((now - fetchedAt) / 60_000); seedStale = seedAge > seedCfg.maxStaleMin; } // `unavailable` is the one sourceState that means "this deployment never opted // into the adapter" (the producer had no credential to try with), as opposed to // "the adapter was tried and is broken" (`stale`/`error`). Grading the two the // same makes an optional, deliberately-unconfigured source an eternal warn that // no amount of operator work can clear — so it is classified separately below. const sourceUnavailable = meta?.sourceState === 'unavailable'; // A current, bounded upstream classification can prove that every admitted // transport path is externally blocked. Keep that state visible without // treating it as a broken producer that can be repaired by another retry. const sourceBlocked = meta?.sourceState === 'blocked'; // Source-specific producers can preserve usable last-good records while a // current upstream attempt is degraded. Surface that state immediately as a // warning without discarding the retained record count from health output. const sourceDegraded = typeof meta?.sourceState === 'string' && meta.sourceState !== 'ok' && !sourceUnavailable && !sourceBlocked; // Content-age trio (2026-05-04 health-readiness plan). Presence of // maxContentAgeMin is the opt-in signal — legacy seeders without it // get contentAge: null and skip the STALE_CONTENT branch in classifyKey. // newestItemAt may be explicit null when seeder's contentMeta returned null // (no usable item timestamps); classifier reads that as STALE_CONTENT. let contentAge = null; if (meta && typeof meta.maxContentAgeMin === 'number') { const newestItemAt = (typeof meta.newestItemAt === 'number') ? meta.newestItemAt : null; const contentAgeMin = newestItemAt == null ? null : Math.round((now - newestItemAt) / 60_000); // Future-dated newestItemAt (contentAgeMin < 0) is suspicious data, not // fresh data: an upstream that publishes timestamps in the future is // either confusing forecasts with observations, mishandling timezones, // or running on a skewed clock. Treat as STALE so the signal surfaces // — without this, `contentAgeMin > maxContentAgeMin` is false for any // negative number and the staleness check silently passes. The // negative `contentAgeMin` is preserved on the wire so operators can // see HOW far in the future the timestamp was (a -10-minute drift is // a clock-skew nit; -8760 minutes is a year-from-now corruption). const isFutureDated = contentAgeMin != null && contentAgeMin < 0; contentAge = { newestItemAt, oldestItemAt: (typeof meta.oldestItemAt === 'number') ? meta.oldestItemAt : null, maxContentAgeMin: meta.maxContentAgeMin, contentAgeMin, contentStale: contentAgeMin == null || isFutureDated || contentAgeMin > meta.maxContentAgeMin, }; } return { hasMeta: meta != null, seedAge, seedStale, seedError: sourceDegraded, sourceUnavailable, sourceBlocked, metaReadFailed: false, metaCount, contentAge, errorCode, }; } function isCascadeCovered(name, hasData, keyStrens, keyErrors) { const siblings = CASCADE_GROUPS[name]; if (!siblings || hasData) return false; for (const sibling of siblings) { if (sibling === name) continue; const sibKey = STANDALONE_KEYS[sibling] ?? BOOTSTRAP_KEYS[sibling]; if (!sibKey) continue; if (keyErrors.get(sibKey)) continue; if (keyHasData(sibKey, keyStrens.get(sibKey) ?? 0)) return true; } return false; } function classifyKey(name, redisKey, opts, ctx) { const { keyStrens, keyErrors, keyMetaValues, keyMetaErrors, now } = ctx; const seedCfg = SEED_META[name]; const isOnDemand = !!opts.allowOnDemand && ON_DEMAND_KEYS.has(name) && !(ctx.activatedNames && ctx.activatedNames.has(name)); const meta = readSeedMeta(seedCfg, keyMetaValues, keyMetaErrors, now); // Per-command Redis errors (data STRLEN or seed-meta GET) propagate as their // own bucket — don't conflate with "key missing", since ops needs to know if // the read itself failed. if (keyErrors.get(redisKey) || meta.metaReadFailed) { const entry = { status: 'REDIS_PARTIAL', records: null }; if (seedCfg) entry.maxStaleMin = seedCfg.maxStaleMin; return entry; } const strlen = keyStrens.get(redisKey) ?? 0; const hasData = keyHasData(redisKey, strlen); const { hasMeta, seedAge, seedStale, seedError, sourceUnavailable, sourceBlocked, metaCount, contentAge, errorCode, } = meta; // When the data key is gone the meta count is meaningless; force records=0 // so we never display the contradictory "EMPTY records=N>0" pair (item 1). const records = hasData ? (metaCount ?? 1) : 0; const cascadeCovered = isCascadeCovered(name, hasData, keyStrens, keyErrors); let status; // Precedes every fault branch: an adapter this deployment never configured has // nothing to be stale, empty, or degraded ABOUT. The producer still writes the // key each run (recordCount 0) so operators can see the adapter is dormant, and // the moment the credential lands the next run reports sourceState 'ok' and this // flips to OK on its own — no health-config change needed. if (sourceUnavailable) status = 'NOT_CONFIGURED'; else if (sourceBlocked && name !== 'crossStraitActivityJapanMod') { // Keep the fleet-wide escape hatch narrow: only the Japan MOD adapter has // a reviewed-data contract and explicit two-path proof for this state. status = 'SEED_ERROR'; } else if (sourceBlocked && hasData && records > 0 && seedStale !== true) status = 'SOURCE_BLOCKED'; else if (seedError) status = 'SEED_ERROR'; else if (!hasData) { if (cascadeCovered) status = 'OK_CASCADE'; else if (MISSING_DATA_IS_FAILURE_KEYS.has(name) && hasMeta && seedStale !== true) status = 'EMPTY'; else if (EMPTY_DATA_OK_KEYS.has(name)) status = seedStale === true ? 'STALE_SEED' : 'OK'; else if (isOnDemand) status = 'EMPTY_ON_DEMAND'; else status = 'EMPTY'; } else if (records === 0) { // hasData is true in this branch, so cascade can never apply (isCascadeCovered // short-circuits when hasData=true). Cascade only shields wholly absent keys. if (ZERO_RECORD_DATA_OK_KEYS.has(name)) status = seedStale === true ? 'STALE_SEED' : 'OK'; else if (isOnDemand) status = 'EMPTY_ON_DEMAND'; else status = 'EMPTY_DATA'; } else if (seedStale === true) status = 'STALE_SEED'; // Coverage threshold: producers that know their canonical shape size can // declare minRecordCount. When the writer reports a count below threshold // (e.g., 10/13 chokepoints because portwatch dropped some), this degrades // to COVERAGE_PARTIAL (warn) instead of reporting OK. Producer must write // seed-meta.recordCount using the *covered* count, not the shape size. else if (seedCfg?.minRecordCount != null && records < seedCfg.minRecordCount) status = 'COVERAGE_PARTIAL'; // Content-age check (opt-in via runSeed contentMeta + maxContentAgeMin). // Fires AFTER all earlier failure paths so STALE_SEED, COVERAGE_PARTIAL, // EMPTY_*, etc. take precedence — STALE_CONTENT is "the seeder is healthy // and the data set is sized correctly, but the content itself is older than // the seeder's content-age budget" (e.g. WHO Disease Outbreak News hasn't // published in >9 days for the disease-outbreaks pilot). // The opt-in signal is contentAge being non-null in seed-meta (presence of // meta.maxContentAgeMin); legacy seeders without it skip this branch. // 2026-05-04 health-readiness plan, Sprint 1. else if (contentAge && contentAge.contentStale) status = 'STALE_CONTENT'; else status = 'OK'; const entry = { status, records }; // Source-level on-demand marker: "this key is RPC-populated or awaiting its // first producer run", NOT "any failure here is acceptable". The status suffix // (`EMPTY_ON_DEMAND`) cannot carry that, because it only covers the // absent/zero-record branches — an on-demand key that HAS data and goes stale // falls through to plain STALE_SEED. Emitted for every status so a consumer // can combine source and status; deciding WHICH statuses the marker excuses is // the consumer's call, and scripts/check-seed-freshness.mjs deliberately // excuses only the empty/absent ones (see ON_DEMAND_SOFT_STATUSES there — // softening SEED_ERROR/STALE_SEED is the marketImplications blind spot the // ON_DEMAND_KEYS policy block above exists to prevent). if (isOnDemand) entry.onDemand = true; if (seedAge !== null) entry.seedAgeMin = seedAge; if (seedCfg) entry.maxStaleMin = seedCfg.maxStaleMin; if (seedCfg?.minRecordCount != null) entry.minRecordCount = seedCfg.minRecordCount; if (status === 'SEED_ERROR' && errorCode) entry.errorCode = errorCode; // Surface content-age fields when seeder opted in (presence of // meta.maxContentAgeMin). Operators can distinguish "stale content" from // "stale seeder run" at a glance. if (contentAge) { entry.contentAgeMin = contentAge.contentAgeMin; // null when contentMeta returned null entry.maxContentAgeMin = contentAge.maxContentAgeMin; } return entry; } const STATUS_COUNTS = { OK: 'ok', OK_CASCADE: 'ok', // An optional source adapter this deployment never supplied a credential for // (producer wrote sourceState:'unavailable'). Buckets to `ok` because there is // no fault and no operator action that would clear it other than opting in. // Must stay registered here: the summary does `STATUS_COUNTS[status] ?? 'warn'`, // so an unlisted status would silently re-become the warn this exists to stop. NOT_CONFIGURED: 'ok', // The producer ran and classified an external transport refusal with a // documented reason. Retained reviewed data remains usable, and retrying the // same bounded paths cannot clear it, so this is informational rather than a // fleet-health warning. SOURCE_BLOCKED: 'ok', STALE_SEED: 'warn', SEED_ERROR: 'warn', EMPTY_ON_DEMAND: 'warn', REDIS_PARTIAL: 'warn', COVERAGE_PARTIAL: 'warn', // Content-age signal — seeder is healthy but upstream stopped publishing. // Operator can't fix upstream cadence, so de-rank vs. STALE_SEED in alerting // (both bucket to 'warn' — overall status is `degraded`, not `critical`). // 2026-05-04 health-readiness plan, Sprint 1. STALE_CONTENT: 'warn', CHINA_DEGRADED: 'warn', CHINA_UNAVAILABLE: 'crit', EMPTY: 'crit', EMPTY_DATA: 'crit', }; function isValidChinaCoverageSummary(candidate) { if ( !candidate || typeof candidate !== 'object' || candidate.schemaVersion !== 1 || candidate.countryCode !== 'CN' || !['healthy', 'degraded', 'unavailable'].includes(candidate.status) || typeof candidate.evaluatedAt !== 'string' || !Number.isFinite(Date.parse(candidate.evaluatedAt)) || !Array.isArray(candidate.entries) || candidate.entries.length === 0 || candidate.entries.length > 100 || !candidate.counts || typeof candidate.counts !== 'object' ) { return false; } const ids = new Set(); const actual = { total: candidate.entries.length, launched: 0, planned: 0, blocked: 0, healthy: 0, degraded: 0, unavailable: 0, }; for (const entry of candidate.entries) { if ( !entry || typeof entry !== 'object' || typeof entry.id !== 'string' || entry.id.length === 0 || entry.id.length > 100 || ids.has(entry.id) || !['launched', 'planned', 'blocked'].includes(entry.launchStatus) || !Array.isArray(entry.reasonCodes) || entry.reasonCodes.length > 16 || entry.reasonCodes.some((reason) => typeof reason !== 'string' || reason.length > 64) ) { return false; } ids.add(entry.id); actual[entry.launchStatus]++; if (entry.launchStatus === 'launched') { if (!['healthy', 'degraded', 'unavailable'].includes(entry.status)) return false; actual[entry.status]++; } else if (entry.status !== entry.launchStatus) { return false; } } if (actual.launched === 0) return false; for (const [name, value] of Object.entries(actual)) { if (candidate.counts[name] !== value) return false; } const expectedStatus = actual.unavailable === actual.launched ? 'unavailable' : actual.degraded > 0 || actual.unavailable > 0 ? 'degraded' : 'healthy'; return candidate.status === expectedStatus; } function projectChinaCoverageStatus(raw, readError = false) { if (readError) { return { status: 'REDIS_PARTIAL', chinaStatus: null, reason: 'SUMMARY_READ_FAILED' }; } const candidate = typeof raw === 'string' ? unwrapEnvelope(parseRedisValue(raw)).data : unwrapEnvelope(raw).data; if (!isValidChinaCoverageSummary(candidate)) { return { status: 'CHINA_UNAVAILABLE', chinaStatus: 'unavailable', reason: 'SUMMARY_INVALID' }; } const status = { healthy: 'OK', degraded: 'CHINA_DEGRADED', unavailable: 'CHINA_UNAVAILABLE', }[candidate.status] ?? 'CHINA_UNAVAILABLE'; const problems = Array.isArray(candidate.entries) ? candidate.entries .filter((entry) => entry?.launchStatus === 'launched' && entry?.status !== 'healthy') .map((entry) => ({ id: entry.id, status: entry.status, reasonCodes: entry.reasonCodes ?? [] })) : []; return { status, chinaStatus: candidate.status, evaluatedAt: candidate.evaluatedAt ?? null, counts: candidate.counts ?? null, ...(problems.length > 0 ? { problems } : {}), }; } function composeChinaCoverageStatus(entry, raw, readError = false) { if (!entry || !['OK', 'STALE_SEED', 'SEED_ERROR'].includes(entry.status)) return entry; const seedStatus = entry.status; const projected = projectChinaCoverageStatus(raw, readError); if (seedStatus === 'OK') return { ...entry, ...projected }; // Preserve writer-health failures when the last summary was healthy, but do // not let a stale/error seed downgrade a known China content outage from // critical to warning. For degraded/unavailable summaries, surface the // content verdict and retain the independent writer signal as seedStatus. if (projected.status === 'OK') { return { ...entry, ...projected, status: seedStatus, seedStatus }; } return { ...entry, ...projected, seedStatus }; } function parseHealthVerdictSnapshot(raw, now, { requireChecks = true } = {}) { if (typeof raw !== 'string') return null; const snapshot = parseRedisValue(raw); if (!snapshot || typeof snapshot !== 'object') return null; if (typeof snapshot.status !== 'string' || typeof snapshot.checkedAt !== 'string') return null; if (!snapshot.summary || typeof snapshot.summary !== 'object') return null; // The compact snapshot deliberately has no `checks` map — that omission IS the // 20 KB saving. Only the full snapshot must carry one. if (requireChecks && (!snapshot.checks || typeof snapshot.checks !== 'object')) return null; const checkedAtMs = Date.parse(snapshot.checkedAt); const ageMs = now - checkedAtMs; // Redis expiry is the primary bound; this is a defensive guard against a // malformed/manual snapshot write or a future TTL regression. if (!Number.isFinite(checkedAtMs) || ageMs < 0 || ageMs > HEALTH_VERDICT_SNAPSHOT_TTL_MS) return null; return snapshot; } async function releaseHealthVerdictRefreshLock(lockToken) { if (!lockToken) return; await redisPipeline([[ 'EVAL', HEALTH_VERDICT_RELEASE_LOCK_SCRIPT, '1', HEALTH_VERDICT_REFRESH_LOCK_KEY, lockToken, ]], 4_000).catch(() => null); } // Single source of truth for "is this check a problem?", shared by the compact // `problems` map and the failure log. Derived from STATUS_COUNTS rather than a // hardcoded status list: any status that buckets to `ok` is by definition not a // problem, so adding a new ok-bucket status (NOT_CONFIGURED) can never again // leave one problem surface silently reporting it. An unregistered status is // treated as a problem, matching the summary's `STATUS_COUNTS[s] ?? 'warn'`. function isProblemStatus(status) { return STATUS_COUNTS[status] !== 'ok'; } // Failure-log / ?history=1 problem set. Distinct from the compact `problems` map // in exactly one way: EMPTY_ON_DEMAND is suppressed here. It is warn-level for // visibility only (realWarnCount subtracts it, it never flips `overall`), so an // unrequested on-demand key must not pollute the incident signature and cause a // spurious failure-log append. That single exception is the ONLY divergence — // everything else defers to isProblemStatus. function collectFailureLogProblems(checks) { const entries = Object.entries(checks) .filter(([, c]) => isProblemStatus(c.status) && c.status !== 'EMPTY_ON_DEMAND'); return { problemKeys: entries.map(([k, c]) => `${k}:${c.status}${c.seedAgeMin != null ? `(${c.seedAgeMin}min)` : ''}`), // The dedupe signature uses only key:status (no age) so a long STALE_SEED // window doesn't produce a new log entry on every poll. sigKeys: entries.map(([k, c]) => `${k}:${c.status}`).sort(), }; } function healthResponseBody(snapshot, compact) { const body = { status: snapshot.status, summary: snapshot.summary, checkedAt: snapshot.checkedAt, }; if (!compact) { body.checks = snapshot.checks; return body; } // Two shapes reach here. A freshly-swept verdict (and the full cached snapshot) // carries `checks`, so derive `problems` from it. The compact snapshot key stores // `problems` already computed — that is why a browser poll reads ~1 KB instead of // the full 20 KB check map. Passing a compact snapshot back through here is a // no-op, which is what makes buildCompactVerdictSnapshot() below safe. const problems = snapshot.checks ? Object.fromEntries(Object.entries(snapshot.checks).filter( ([name, check]) => name !== 'chinaDecisionSignals' && isProblemStatus(check.status), )) : { ...(snapshot.problems ?? {}) }; // Older compact snapshots may predate the operator-only China health // projection. Strip it again at the response boundary so a cached value // cannot leak source freshness details to anonymous status readers. delete problems.chinaDecisionSignals; if (Object.keys(problems).length > 0) body.problems = problems; return body; } /** * The compact snapshot stored in Redis is byte-for-byte the body a `?compact=1` * request returns. Deriving it from the same function that renders the response * means the cached form can never drift from the live form. */ function buildCompactVerdictSnapshot(snapshot) { return healthResponseBody(snapshot, true); } function healthResponse(snapshot, compact, headers) { let responseHeaders = headers; if (compact) { const { 'CF-Cache-Status': _bypassMarker, ...base } = headers; responseHeaders = { ...base, 'Cache-Control': 'no-store, max-age=0', 'CDN-Cache-Control': 'no-store', }; } return new Response(JSON.stringify(healthResponseBody(snapshot, compact), null, compact ? 0 : 2), { status: 200, headers: responseHeaders, }); } export default async function handler(req, ctx) { if (isDisallowedOrigin(req)) { return new Response('Forbidden', { status: 403 }); } const cors = getCorsHeaders(req, 'GET, OPTIONS'); const headers = { 'Content-Type': 'application/json', 'Cache-Control': 'private, no-store, max-age=0', 'CDN-Cache-Control': 'no-store', 'CF-Cache-Status': 'BYPASS', ...cors, }; if (req.method === 'OPTIONS') { return new Response(null, { status: 204, headers }); } const url = new URL(req.url); const compact = url.searchParams.get('compact') === '1'; const wantsHistory = url.searchParams.get('history') === '1'; if (!compact || wantsHistory) { const keyCheck = await validateApiKey(req, { forceKey: true }); if (keyCheck.required && !keyCheck.valid) { const error = keyCheck.error === USER_API_KEY_GATEWAY_VALIDATION_ERROR ? 'Invalid API key' : keyCheck.error; // RFC 7235 §3.1 requires WWW-Authenticate on every 401. The challenge // must reflect what this gate actually accepts: validateApiKey reads the // X-WorldMonitor-Key / X-Api-Key headers (or tester cookies) — never // Authorization: Bearer — so advertising a Bearer/OAuth challenge here // sent agents down a flow that cannot succeed (post-#4867 review). The // hint exists because this URL is what old copies of the api-catalog // linkset advertised as the public status endpoint (#4856) — a keyless // caller landing here should learn the public form, not dead-end. return jsonResponse( { error, hint: 'Detailed health requires an operator/enterprise API key. Public status: /api/health?compact=1', }, 401, { ...headers, 'WWW-Authenticate': 'ApiKey realm="worldmonitor-health", header="X-WorldMonitor-Key"', } ); } } // ?history=1 — fast read of the failure-log persisted by previous /api/health // probes. Skips the full freshness check (which is expensive — fetches every // bootstrap key) and returns only the last-failure snapshot + the deduped // incident log. Use to answer "what's been flipping?" without needing // Upstash creds. The classifier writes these keys on every non-OK probe: // health:last-failure — single most recent unhealthy snapshot (24h TTL) // health:failure-log — last 50 deduped incidents (7-day TTL) // See WM 2026-05-10 — added after a night of UptimeRobot flips that needed // direct Upstash inspection to diagnose. if (wantsHistory) { const results = await redisPipeline( [ ['GET', 'health:last-failure'], ['LRANGE', 'health:failure-log', '0', '-1'], ], 4_000, ); const parseJson = (raw) => { if (typeof raw !== 'string') return null; try { return JSON.parse(raw); } catch { return null; } }; const lastFailureRaw = results?.[0]?.result; const failureLogRaw = results?.[1]?.result; const body = { lastFailure: parseJson(lastFailureRaw), failureLog: Array.isArray(failureLogRaw) ? failureLogRaw.map(parseJson).filter((e) => e !== null) : [], checkedAt: new Date().toISOString(), }; return new Response(JSON.stringify(body, null, 2), { status: 200, headers }); } const now = Date.now(); // A snapshot hit is one Redis command instead of the ~390-command registry // sweep below. A failed snapshot read is a real Redis outage, not a cache // miss: returning 503 preserves UptimeRobot's hard-down signal. let refreshLockToken = null; let ownsSnapshotRefreshLock = false; try { if (!getRedisCredentials()) throw new Error('Redis not configured'); // Read the snapshot this request will actually render. `?compact=1` — the // browser poll, ~115k/day — reads the ~1 KB compact key instead of dragging the // full ~20 KB check map out of Redis to show a tenth of it (#5300). const snapshotKey = compact ? HEALTH_VERDICT_COMPACT_SNAPSHOT_KEY : HEALTH_VERDICT_SNAPSHOT_KEY; const snapshotResult = await redisPipeline([['GET', snapshotKey]], 4_000); if (!snapshotResult) throw new Error('Redis request failed'); if (snapshotResult[0]?.error) throw new Error('Redis snapshot read failed'); const cachedSnapshot = parseHealthVerdictSnapshot(snapshotResult[0]?.result, Date.now(), { requireChecks: !compact }); if (cachedSnapshot) return healthResponse(cachedSnapshot, compact, headers); refreshLockToken = `${now}:${crypto.randomUUID()}`; let lockResult = await redisPipeline([[ 'SET', HEALTH_VERDICT_REFRESH_LOCK_KEY, refreshLockToken, 'EX', String(HEALTH_VERDICT_REFRESH_LOCK_TTL_SECONDS), 'NX', ]], 4_000); if (!lockResult || lockResult[0]?.error) throw new Error('Redis snapshot lock failed'); ownsSnapshotRefreshLock = lockResult[0]?.result === 'OK'; if (!ownsSnapshotRefreshLock) { // Another edge invocation is already refreshing. Wait briefly for its // snapshot instead of multiplying the ~390-command sweep during a cold // burst. If the holder dies, retry SET NX until its lease expires; only // a lock owner may proceed to the sweep. const waitDeadline = Date.now() + HEALTH_VERDICT_REFRESH_WAIT_MS; for ( let attempt = 0; attempt < HEALTH_VERDICT_REFRESH_WAIT_ATTEMPTS && Date.now() < waitDeadline; attempt++ ) { const backoffMs = Math.min(100 * (2 ** attempt), 1_000); const jitterMs = Math.floor(Math.random() * 50); const sleepMs = Math.min(backoffMs + jitterMs, Math.max(0, waitDeadline - Date.now())); await new Promise((resolve) => setTimeout(resolve, sleepMs)); const remainingMs = waitDeadline - Date.now(); if (remainingMs < HEALTH_VERDICT_MIN_REDIS_TIMEOUT_MS) break; const redisTimeoutMs = Math.min(4_000, remainingMs); const refreshedResult = await redisPipeline([['GET', snapshotKey]], redisTimeoutMs); if (!refreshedResult || refreshedResult[0]?.error) throw new Error('Redis snapshot wait failed'); const refreshedSnapshot = parseHealthVerdictSnapshot(refreshedResult[0]?.result, Date.now(), { requireChecks: !compact }); if (refreshedSnapshot) return healthResponse(refreshedSnapshot, compact, headers); lockResult = await redisPipeline([[ 'SET', HEALTH_VERDICT_REFRESH_LOCK_KEY, refreshLockToken, 'EX', String(HEALTH_VERDICT_REFRESH_LOCK_TTL_SECONDS), 'NX', ]], redisTimeoutMs); if (!lockResult || lockResult[0]?.error) throw new Error('Redis snapshot lock retry failed'); ownsSnapshotRefreshLock = lockResult[0]?.result === 'OK'; if (ownsSnapshotRefreshLock) break; } // Redis stayed reachable but another refresher held the lock through our // request budget. Fall back to one direct sweep rather than mislabeling // healthy Redis as REDIS_DOWN. This path is bounded and should be rare; // the normal cold-burst path still permits only the elected owner. } } catch (err) { if (ownsSnapshotRefreshLock) await releaseHealthVerdictRefreshLock(refreshLockToken); return jsonResponse({ status: 'REDIS_DOWN', error: err.message, checkedAt: new Date(now).toISOString(), }, 503, headers); } const allDataKeys = [ ...Object.values(BOOTSTRAP_KEYS), ...Object.values(STANDALONE_KEYS), ]; const allMetaKeys = Object.values(SEED_META).map(s => s.key); const activationEntries = Object.entries(ACTIVATION_MARKERS); // STRLEN for data keys avoids loading large blobs into memory (OOM prevention). // NEG_SENTINEL ('__WM_NEG__') is 10 bytes — strlenIsData() rejects exactly // that length while accepting any other non-zero strlen as data. List-typed // keys (LIST_DATA_KEYS) take LLEN instead; STRLEN would WRONGTYPE on them. let results; try { const commands = [ ...allDataKeys.map(dataLenCommand), ...allMetaKeys.map(k => ['GET', k]), ...activationEntries.map(([, marker]) => ['EXISTS', marker]), ['GET', CHINA_COVERAGE_SUMMARY_KEY], ]; if (!getRedisCredentials()) throw new Error('Redis not configured'); results = await redisPipeline(commands, 8_000); if (!results) throw new Error('Redis request failed'); } catch (err) { if (ownsSnapshotRefreshLock) await releaseHealthVerdictRefreshLock(refreshLockToken); // REDIS_DOWN is the one hard-down state that returns 503: with Redis // unreachable the endpoint can assess nothing, so a plain HTTP-status // monitor (UptimeRobot, k8s probe, LB) must see a failure. DEGRADED/ // UNHEALTHY/WARNING stay 200 (verdict in body) so warn-level seed jitter // doesn't flap HTTP monitors — see #2699 and the always-200 main response. return jsonResponse({ status: 'REDIS_DOWN', error: err.message, checkedAt: new Date(now).toISOString(), }, 503, headers); } // keyStrens: byte length per data key (0 = missing/empty/sentinel) // keyErrors: per-command Redis errors so we can surface REDIS_PARTIAL const keyStrens = new Map(); const keyErrors = new Map(); for (let i = 0; i < allDataKeys.length; i++) { const r = results[i]; if (r?.error) keyErrors.set(allDataKeys[i], r.error); keyStrens.set(allDataKeys[i], r?.result ?? 0); } // keyMetaValues: parsed seed-meta objects (GET, small payloads) // keyMetaErrors: per-command errors so a single GET failure surfaces as // REDIS_PARTIAL instead of silently degrading to STALE_SEED. const keyMetaValues = new Map(); const keyMetaErrors = new Map(); for (let i = 0; i < allMetaKeys.length; i++) { const r = results[allDataKeys.length + i]; if (r?.error) keyMetaErrors.set(allMetaKeys[i], r.error); keyMetaValues.set(allMetaKeys[i], r?.result ?? null); } // activatedNames: keys whose durable activation marker exists — these // leave ON_DEMAND softening permanently (#4927 re-review P1). A marker // read error degrades to not-activated (soft), never to a false alarm. const activatedNames = new Set(); for (let i = 0; i < activationEntries.length; i++) { const r = results[allDataKeys.length + allMetaKeys.length + i]; if (!r?.error && Number(r?.result) === 1) activatedNames.add(activationEntries[i][0]); } const chinaCoverageResult = results[allDataKeys.length + allMetaKeys.length + activationEntries.length]; const chinaCoverageRaw = chinaCoverageResult?.error ? null : chinaCoverageResult?.result; const classifyCtx = { keyStrens, keyErrors, keyMetaValues, keyMetaErrors, activatedNames, now }; const checks = {}; const counts = { ok: 0, warn: 0, onDemandWarn: 0, staleContent: 0, crit: 0 }; let totalChecks = 0; const sources = [ [BOOTSTRAP_KEYS, { allowOnDemand: false }], [STANDALONE_KEYS, { allowOnDemand: true }], ]; for (const [registry, opts] of sources) { for (const [name, redisKey] of Object.entries(registry)) { totalChecks++; let entry = classifyKey(name, redisKey, opts, classifyCtx); if (name === 'chinaCoverage') { entry = composeChinaCoverageStatus(entry, chinaCoverageRaw, Boolean(chinaCoverageResult?.error)); } checks[name] = entry; const bucket = STATUS_COUNTS[entry.status] ?? 'warn'; counts[bucket]++; if (entry.status === 'EMPTY_ON_DEMAND') counts.onDemandWarn++; // STALE_CONTENT = "seeder is fresh but the upstream DATA stopped // advancing" (a frozen feed — see issue #3845). It still buckets to // `warn`; this sub-count surfaces it explicitly so operators can tell a // frozen feed apart from an ordinary stale-seeder warn at a glance. if (entry.status === 'STALE_CONTENT') counts.staleContent++; } } // On-demand keys that simply haven't been requested yet should not flip // overall to WARNING — they're warn-level only for visibility. const realWarnCount = counts.warn - counts.onDemandWarn; const critCount = counts.crit; let overall; if (critCount === 0 && realWarnCount === 0) overall = 'HEALTHY'; else if (critCount === 0) overall = 'WARNING'; // Degraded threshold scales with registry size so adding keys doesn't // silently raise the page-out bar. ~3% of total keys (was hardcoded 3). else if (critCount / totalChecks <= 0.03) overall = 'DEGRADED'; else overall = 'UNHEALTHY'; if (overall !== 'HEALTHY') { // problemKeys includes seedAgeMin for the snapshot (useful for post-mortem), // but the dedupe signature uses only key:status (no age) so a long STALE_SEED // window doesn't produce a new log entry on every poll. const { problemKeys, sigKeys } = collectFailureLogProblems(checks); console.log('[health] %s problems=[%s]', overall, problemKeys.join(', ')); const failureLogEntry = { at: new Date(now).toISOString(), status: overall, critCount, warnCount: realWarnCount, problems: problemKeys, }; // Dedupe: only LPUSH when the incident signature (status + problem set, // excluding seedAgeMin) changes. Read the previous sig first, then write // everything (last-failure + sig + LPUSH) in one atomic pipeline so the // sig only advances when the LPUSH succeeds. If the pipeline fails, the // sig stays stale and the next poll retries the append. const sig = `${overall}|${sigKeys.join(',')}`; const prevSigResult = await redisPipeline([['GET', 'health:failure-log-sig']], 4_000).catch(() => null); const prevSig = prevSigResult?.[0]?.result ?? ''; const persistCmds = [ ['SET', 'health:last-failure', JSON.stringify(failureLogEntry), 'EX', 86400], ]; if (sig !== prevSig) { persistCmds.push( ['LPUSH', 'health:failure-log', JSON.stringify(failureLogEntry)], ['LTRIM', 'health:failure-log', 0, 49], ['EXPIRE', 'health:failure-log', 86400 * 7], ['SET', 'health:failure-log-sig', sig, 'EX', 86400], ); } const persist = redisPipeline(persistCmds, 4_000).catch(() => {}); if (ctx && typeof ctx.waitUntil === 'function') ctx.waitUntil(persist); } else { // Clear the sig on recovery so a recurrence of the same problem set // after a healthy gap is logged as a new incident, not deduped against // the previous one. const clear = redisPipeline([['DEL', 'health:failure-log-sig']], 4_000).catch(() => {}); if (ctx && typeof ctx.waitUntil === 'function') ctx.waitUntil(clear); } const verdictSnapshot = { status: overall, summary: { total: totalChecks, ok: counts.ok, // `warn` excludes on-demand-empty (cosmetic warns); `onDemandWarn` is // surfaced separately so readers can reconcile against `overall`. warn: realWarnCount, onDemandWarn: counts.onDemandWarn, // `staleContent` is a SUBSET of `warn` (fresh seeder, frozen upstream // data — issue #3845). Surfaced so a frozen feed is visible without // walking every check entry. staleContent: counts.staleContent, crit: critCount, }, checkedAt: new Date(now).toISOString(), checks, }; // Await the write so the next request cannot race an unstarted background // SET and repeat the full sweep. A write failure does not invalidate the // live verdict just computed; the next request will retry by sweeping. // Both snapshots are written by the SAME sweep, in one pipeline, so the compact // form can never disagree with the full one or outlive it. const snapshotWriteResult = await redisPipeline([ [ 'SET', HEALTH_VERDICT_SNAPSHOT_KEY, JSON.stringify(verdictSnapshot), 'EX', String(HEALTH_VERDICT_SNAPSHOT_TTL_SECONDS), ], [ 'SET', HEALTH_VERDICT_COMPACT_SNAPSHOT_KEY, JSON.stringify(buildCompactVerdictSnapshot(verdictSnapshot)), 'EX', String(HEALTH_VERDICT_SNAPSHOT_TTL_SECONDS), ], ], 4_000).catch(() => null); const snapshotWriteFailed = !snapshotWriteResult || snapshotWriteResult.length !== 2 || snapshotWriteResult.some((entry) => entry?.error); if (ownsSnapshotRefreshLock) await releaseHealthVerdictRefreshLock(refreshLockToken); // A failed cache write does not invalidate the live verdict. Releasing only // this request's token lets the next caller retry immediately without ever // deleting a successor's lock after a slow sweep outlives its lease. if (snapshotWriteFailed) console.warn('[health] verdict snapshot write failed'); // Compact is the public keyless form polled by external uptime MONITORS, so it // must NEVER be edge-cached: the prior `s-maxage=60` (#4907, to collapse // dashboard-tab polling) let a shared CDN entry pin a stale WARNING that kept // UptimeRobot reading "down" long AFTER the seed recovered (2026-07-07 — the // literal `?cb=*cachebuster*` was a constant, so it never busted the entry). // no-store guarantees that Redis reachability is checked on every request; // only the verdict payload is reused, for at most 60 seconds. All other // responses already carry the no-store defaults from `headers` (a cached // 401 pins an auth failure; a cached 503 masks REDIS_DOWN recovery). return healthResponse(verdictSnapshot, compact, headers); } // Test-only exports. Not part of the public edge handler surface — Vercel's // runtime invokes only `default export`. These let scoped unit tests exercise // the classifier without standing up the full bootstrap-keys + Redis pipeline. // 2026-05-04 health-readiness plan, Sprint 1 test plan (Codex round 2 P1). export const __testing__ = { readSeedMeta, classifyKey, healthResponseBody, collectFailureLogProblems, ACTIVATION_MARKERS, STATUS_COUNTS, // List-typed data keys + the command builder that measures them with LLEN // instead of STRLEN (tests/health-list-data-keys.test.mjs). LIST_DATA_KEYS, dataLenCommand, HEALTH_VERDICT_SNAPSHOT_KEY, HEALTH_VERDICT_COMPACT_SNAPSHOT_KEY, buildCompactVerdictSnapshot, HEALTH_VERDICT_SNAPSHOT_TTL_SECONDS, HEALTH_VERDICT_REFRESH_LOCK_KEY, HEALTH_VERDICT_REFRESH_WAIT_MS, CHINA_COVERAGE_SUMMARY_KEY, projectChinaCoverageStatus, composeChinaCoverageStatus, healthVerdictRedisKey, parseHealthVerdictSnapshot, // U7 (Tier 3 parity test): exposed for tests/mcp-bootstrap-parity.test.mjs // to walk the canonical seeded-data inventory. Both consts are unexported // at module scope by design — this is the test-only escape hatch. BOOTSTRAP_KEYS, STANDALONE_KEYS, SEED_META, EMPTY_DATA_OK_KEYS, MISSING_DATA_IS_FAILURE_KEYS, ZERO_RECORD_DATA_OK_KEYS, // Exposed so a registration test can prove a key's deployment-order softening // is actually wired — membership here is what keeps a not-yet-published key // out of CRIT, and nothing could assert it before (#5736). ON_DEMAND_KEYS, };