// @ts-check // Source freshness registry. Mirrors the table in // docs/internal/pro-regional-intelligence-appendix-scoring.md "Source Freshness Registry". // // Each entry maps a Redis key (or key prefix) to its expected max-age. // The snapshot writer marks inputs as stale or missing based on this table // and feeds those flags into SnapshotMeta.snapshot_confidence. import { CII_RISK_SCORE_CACHE_KEYS } from '../_cii-risk-cache-keys.mjs'; /** * @typedef {object} SourceFreshnessSpec * @property {string} key - Redis key (literal, no template variables) * @property {number} maxAgeMin - Maximum acceptable age in minutes * @property {string[]} feedsAxes - Which balance axes / sections this input drives * @property {string=} metaKey - Optional companion seed-meta key carrying * {fetchedAt, recordCount}. Used when the primary payload has no * top-level timestamp field. classifyInputs() will prefer the meta key's * timestamp when both are available so a stalled seeder can be detected * even if the data payload is still served from a previous write. */ /** * Only keys that compute modules actually consume via sources['...']. * Keys must be added here in lockstep with new compute consumers, never * speculatively. Drift between this list and the consumers is an alerting * blind spot (a missing key drags down snapshot_confidence and a present * key with no consumer wastes a Redis read). * * @type {SourceFreshnessSpec[]} */ export const FRESHNESS_REGISTRY = [ { key: CII_RISK_SCORE_CACHE_KEYS.stale, maxAgeMin: 30, feedsAxes: ['domestic_fragility', 'coercive_pressure'], metaKey: 'seed-meta:intelligence:risk-scores' }, { key: 'forecast:predictions:v2', maxAgeMin: 180, feedsAxes: ['scenarios', 'actors'] }, { key: 'supply_chain:chokepoints:v4', maxAgeMin: 30, feedsAxes: ['maritime_access', 'corridors'] }, { key: 'supply_chain:transit-summaries:v1', maxAgeMin: 30, feedsAxes: ['maritime_access'], metaKey: 'seed-meta:supply_chain:transit-summaries' }, { key: 'intelligence:cross-source-signals:v1', maxAgeMin: 45, feedsAxes: ['coercive_pressure', 'evidence'], metaKey: 'seed-meta:intelligence:cross-source-signals' }, { key: 'relay:oref:history:v1', maxAgeMin: 15, feedsAxes: ['coercive_pressure', 'triggers'], metaKey: 'seed-meta:relay:oref:history' }, { key: 'economic:macro-signals:v1', maxAgeMin: 60, feedsAxes: ['capital_stress'] }, { key: 'economic:national-debt:v1', maxAgeMin: 86400, feedsAxes: ['capital_stress'], metaKey: 'seed-meta:economic:national-debt' }, // monthly seed (30d cron), 60d window absorbs one missed run — mirrors api/health.js nationalDebt. metaKey is the primary freshness source (payload's seededAt is also recognized by extractTimestamp as a fallback). { key: 'economic:stress-index:v1', maxAgeMin: 120, feedsAxes: ['capital_stress'] }, { key: 'energy:mix:v1:_all', maxAgeMin: 50400, feedsAxes: ['energy_vulnerability'], metaKey: 'seed-meta:economic:owid-energy-mix' }, { key: 'economic:eu-gas-storage:v1', maxAgeMin: 2880, feedsAxes: ['energy_vulnerability'], metaKey: 'seed-meta:economic:eu-gas-storage' }, // runSeed writes seed-meta with numeric fetchedAt; metaKey takes priority over payload fields, defense-in-depth against a future refactor that strips fetchedAt/seededAt from the payload (regression risk that produced #3728). { key: 'economic:spr:v1', maxAgeMin: 10080, feedsAxes: ['energy_buffer'] }, // Mobility v1 (Phase 2 PR2) — feed the MobilityState block via mobility.mjs. // maxAgeMin matches each seeder's cron interval + safety buffer. // The aviation/gpsjam payloads have no top-level timestamp field, so they // rely on companion seed-meta:* keys (written by the seeders via // writeFreshnessMetadata / upstashSet) for stale detection. Without these // metaKey hints, classifyInputs would fall back to "undated = fresh" and // miss stalled seeders entirely. { key: 'aviation:delays:faa:v1', maxAgeMin: 60, feedsAxes: ['mobility'], metaKey: 'seed-meta:aviation:faa' }, { key: 'aviation:delays:intl:v3', maxAgeMin: 90, feedsAxes: ['mobility'], metaKey: 'seed-meta:aviation:intl' }, { key: 'aviation:notam:closures:v2', maxAgeMin: 120, feedsAxes: ['mobility'], metaKey: 'seed-meta:aviation:notam' }, { key: 'intelligence:gpsjam:v2', maxAgeMin: 1440, feedsAxes: ['mobility', 'airspace'], metaKey: 'seed-meta:intelligence:gpsjam' }, // gpsjam.org is a DAILY source (restored from Wingbits, PR #4987); 1440min matches api/health.js gpsjam.maxStaleMin so snapshots don't mark it stale hours after a healthy daily seed. // military:flights:v1 already carries top-level fetchedAt, no metaKey needed. { key: 'military:flights:v1', maxAgeMin: 30, feedsAxes: ['mobility', 'reroute_intensity'] }, ]; /** Every metaKey referenced by FRESHNESS_REGISTRY, for pre-fetching. */ export const ALL_META_KEYS = FRESHNESS_REGISTRY .map((s) => s.metaKey) .filter((k) => typeof k === 'string' && k.length > 0); export const ALL_INPUT_KEYS = FRESHNESS_REGISTRY.map((s) => s.key); /** * Classify each input as fresh, stale, or missing. * * Timestamp resolution order per input: * 1. If the spec has a `metaKey`, use metaPayloads[metaKey].fetchedAt. * This is the canonical signal for sources whose data payload lacks * a top-level timestamp (FAA alerts, AviationStack, NOTAM, GPS jam). * 2. Otherwise, pull a timestamp from the primary payload via * extractTimestamp (fetchedAt, generatedAt, timestamp, updatedAt, * lastUpdate, seededAt). * 3. If neither yields a timestamp, classify as stale. A present-but-undated * payload cannot be proven fresh — defaulting to fresh let stalled * seeders silently inflate snapshot_confidence (#3728). Forcing stale * pressures upstream to emit a timestamp. * * @param {Record} payloads - Map of key -> raw value (or null) * @param {Record} [metaPayloads] - Map of metaKey -> raw value (or null) * @returns {{ fresh: string[]; stale: string[]; missing: string[] }} */ export function classifyInputs(payloads, metaPayloads = {}) { const fresh = []; const stale = []; const missing = []; const now = Date.now(); for (const spec of FRESHNESS_REGISTRY) { const payload = payloads[spec.key]; if (payload === null || payload === undefined) { missing.push(spec.key); continue; } // Prefer the companion seed-meta:*.fetchedAt when the spec declares one. // This is the only way to detect a stalled seeder for payloads that // don't carry a top-level timestamp of their own. let ts = null; if (spec.metaKey) { const meta = metaPayloads[spec.metaKey]; ts = extractTimestamp(meta); } if (ts === null) ts = extractTimestamp(payload); if (ts === null) { // Present but undated — classify as stale. We cannot prove freshness, // and the previous "default to fresh" behavior fabricated // snapshot_confidence whenever a seeder stalled with no timestamp. stale.push(spec.key); continue; } const ageMin = (now - ts) / 60_000; if (ageMin > spec.maxAgeMin) { stale.push(spec.key); } else { fresh.push(spec.key); } } return { fresh, stale, missing }; } /** * Resolve the effective timestamp for an input, preferring the metaKey * (when declared) over the payload's own timestamp. Returns null if neither * source carries a parseable timestamp. * * Exported for snapshot-meta.mjs, which needs the per-input timestamp to * derive valid_until from the registry's maxAgeMin. * * @param {SourceFreshnessSpec} spec * @param {unknown} payload * @param {Record} metaPayloads * @returns {number | null} */ export function resolveInputTimestamp(spec, payload, metaPayloads) { let ts = null; if (spec.metaKey) { const meta = metaPayloads[spec.metaKey]; ts = extractTimestamp(meta); } if (ts === null) ts = extractTimestamp(payload); return ts; } /** Pull a timestamp out of common payload shapes; null if none found. */ function extractTimestamp(payload) { if (typeof payload !== 'object' || payload === null) return null; const obj = payload; // `seededAt` is the convention used by runSeed-based seeders that wrap // data in a { ...data, seededAt: ISOString } shape (seed-national-debt, // seed-iea-oil-stocks, seed-eurostat-country-data, etc.). Without it here, // those seeds got classified "present but undated" → always fresh, // silently masking stalled crons. for (const field of ['fetchedAt', 'generatedAt', 'timestamp', 'updatedAt', 'lastUpdate', 'seededAt']) { if (typeof obj[field] === 'number') return obj[field]; if (typeof obj[field] === 'string') { const parsed = Date.parse(obj[field]); if (Number.isFinite(parsed)) return parsed; } } return null; }