Download persistent.js from Okyanus/pomona-sensor-data-checker: direct link, hf CLI and curl.
- Browser
- Download file 31 kB
-
https://huggingface.co/spaces/Okyanus/pomona-sensor-data-checker/resolve/main/persistent.js
- Command line
-
hf download hf://spaces/Okyanus/pomona-sensor-data-checker/persistent.js
-
curl -L -o persistent.js https://huggingface.co/spaces/Okyanus/pomona-sensor-data-checker/resolve/main/persistent.js
31 kB
| /* | |
| * Pomona persistent sensor checks, ported to JavaScript: Core's trusted per-stream reference and the | |
| * checks that judge a reading against it. | |
| * | |
| * Source of truth: | |
| * services/core/app/trusted_baseline.py stream_baseline (the one-pass reference; the incremental | |
| * StreamBaseline tracker is byte-identical to it and not ported) | |
| * services/model-router/app/sensor_quality.py apply_persistent_checks and what it calls (adaptive pH/EC | |
| * stuck, step check, noise check, level drift, persistent stuck limits) | |
| * tests/persistent.test.mjs checks this file against the Python code (cases from | |
| * tests/make_persistent_cases.py); run it after changing either side. | |
| * | |
| * rules.js (the stateless checks over the last 11 packets) is untouched and stays an exact copy of the one in | |
| * the greenhouse demo Space. This file sits next to it: `derive` is deriveSensorQuality plus the persistent step. | |
| * | |
| * Python things reproduced on purpose: round(x, n) is half-to-even on the exact binary value, `:g` / `:.1f` / | |
| * `:.0%` format the exact value half-to-even (toPrecision/toFixed round half up), timedelta arithmetic is exact | |
| * in microseconds, and total_seconds() is a correctly rounded int / 10**6. | |
| */ | |
| (function (root, factory) { | |
| if (typeof module === "object" && module.exports) module.exports = factory(require("./rules.js")); | |
| else root.PomonaPersistent = factory(root.PomonaRules); | |
| })(typeof self !== "undefined" ? self : this, function (R) { | |
| "use strict"; | |
| const US = R.US_PER_S; | |
| const numeric = R.numeric; | |
| const has = (obj, key) => Object.prototype.hasOwnProperty.call(obj, key); | |
| const isPlainObject = (v) => v !== null && typeof v === "object" && !Array.isArray(v); | |
| const get = (obj, key) => (has(obj, key) ? obj[key] : undefined); | |
| const isNone = (v) => v === undefined || v === null; | |
| const addUnique = (items, value) => { if (!items.includes(value)) items.push(value); }; | |
| const cmp = (a, b) => (a < b ? -1 : a > b ? 1 : 0); | |
| // ---- constants (trusted_baseline.py) ------------------------------------------------------------ | |
| const DEFAULT_WINDOW_S = 2 * 3600; | |
| const DEFAULT_MIN_N = 6; | |
| const DEFAULT_SCAN_LIMIT = 5000; | |
| const IDENTICAL_EPSILON = 1e-9; | |
| const NORM_STEPS = 200; | |
| const NORM_FIELDS = ["ph", "ec_ms_cm"]; | |
| const SPREAD_LAG_S = 1800; | |
| const BASELINE_FIELDS = ["ph", "ec_ms_cm", "air_temperature_c", "humidity_pct", "soil_moisture_pct", | |
| "substrate_moisture_pct", "substrate_temperature_c"]; | |
| const MEASUREMENT_FIELDS = { | |
| ph: ["ph"], ec_ms_cm: ["ec_ms_cm"], air_temperature_c: ["air_temperature_c"], humidity_pct: ["humidity_pct"], | |
| soil_moisture_pct: ["soil_moisture_pct", "substrate_moisture_pct"], | |
| }; | |
| const ALIASES = { soil_moisture_pct: "substrate_moisture_pct", substrate_moisture_pct: "soil_moisture_pct" }; | |
| // ---- constants (sensor_quality.py) -------------------------------------------------------------- | |
| const STUCK_EPSILON = 1e-9; | |
| const BASELINE_DRIFT_THRESHOLDS = { ph: 0.35, ec_ms_cm: 0.4 }; | |
| const PLAUSIBLE_RANGES = { ph: [3.0, 11.0], ec_ms_cm: [0.0, 12.0] }; | |
| const MINUTE_US = 60n * US; | |
| const HOUR_US = 3600n * US; | |
| const PERSISTENT_STUCK_LIMITS_US = { air_temperature_c: 45n * MINUTE_US, humidity_pct: 30n * MINUTE_US, | |
| substrate_temperature_c: 6n * HOUR_US, water_temperature_c: 3n * HOUR_US }; | |
| const ADAPTIVE_STUCK_FIELDS = ["ph", "ec_ms_cm"]; | |
| const ADAPTIVE_STUCK_FLOOR_US = 5n * MINUTE_US; | |
| const ADAPTIVE_STUCK_RUN_FACTOR = 4.0; | |
| const ADAPTIVE_STUCK_MIN_STEPS = 15; | |
| const ADAPTIVE_STUCK_MIN_CHANGE_SHARE = 0.7; | |
| const ADAPTIVE_STUCK_MIN_RUN = 3; | |
| // field: [smallest allowance, speed per minute, unit, instrument] | |
| const STEP_LIMITS = { | |
| air_temperature_c: [3.0, 1.0, "C", "a second thermometer"], | |
| humidity_pct: [12.0, 4.0, "%RH", "a second hygrometer"], | |
| soil_moisture_pct: [8.0, 2.0, "%", "a hand-held moisture meter"], | |
| substrate_moisture_pct: [8.0, 2.0, "%", "a hand-held moisture meter"], | |
| substrate_temperature_c: [1.5, 0.3, "C", "a second thermometer"], | |
| }; | |
| const STEP_REWET_BELOW_PCT = { soil_moisture_pct: 15.0, substrate_moisture_pct: 15.0 }; | |
| const NOISE_STEP_LIMIT_5MIN = { humidity_pct: 10.0, air_temperature_c: 2.0, ph: 0.5, | |
| soil_moisture_pct: 1.0, substrate_moisture_pct: 1.0, substrate_temperature_c: 0.4 }; | |
| const NOISE_STEP_FLOOR = { humidity_pct: 3.0, air_temperature_c: 0.4, ph: 0.1, | |
| soil_moisture_pct: 2.5, substrate_moisture_pct: 2.5, substrate_temperature_c: 0.5 }; | |
| // field: [limit as it is, in the field's unit; largest median sample interval of the window it is judged at, in seconds]. | |
| // EC is judged only on a fast stream (90 s or less): 3 months of real 10-second tank EC (four probes, about 1.2 mS/cm) | |
| // never give a 12-reading window with a median step above 0.143 mS/cm at any rate from 10 to 90 s, so 0.20 flags | |
| // none of about 0.92 million windows per probe. A slower stream has no real data behind a limit (5-minute drain EC | |
| // really jumps) and is not judged. The limit is used as it is, without the square-root scaling of the other fields. | |
| const NOISE_FIXED_LIMIT = { ec_ms_cm: [0.20, 90.0] }; | |
| const NOISE_REFERENCE_INTERVAL_S = 300.0; | |
| const NOISE_WINDOW = 12; | |
| const NOISE_MAX_NET_SHARE = 0.5; | |
| const NOISE_MIN_SAMPLES = 6; | |
| const DRIFT_SPREAD_FACTOR = 3.0; | |
| const LAGGED_SPREAD_FIELDS = ["ph"]; | |
| const CALIBRATION_HINT = "if a meter agrees with the reading, it is real: record a calibration for this probe to accept it"; | |
| const ROUTINE_CHECK = "continue routine monitoring"; | |
| // ---- exact number work: Python's float formatting, round() and timedelta ---------------------------- | |
| const F64 = new DataView(new ArrayBuffer(8)); | |
| // |x| = m * 2^e exactly, for a finite double. | |
| function decompose(x) { | |
| F64.setFloat64(0, x); | |
| const hi = F64.getUint32(0), lo = F64.getUint32(4); | |
| const exp = (hi >>> 20) & 0x7ff; | |
| let m = (BigInt(hi & 0xfffff) << 32n) | BigInt(lo); | |
| if (exp === 0) return { m, e: -1074 }; | |
| return { m: m | (1n << 52n), e: exp - 1075 }; | |
| } | |
| const pow10 = (k) => 10n ** BigInt(k); | |
| // round-half-even(|x| * 10^k) as a BigInt, from the exact binary value (k may be negative). | |
| function roundScaled(x, k) { | |
| const { m, e } = decompose(Math.abs(x)); | |
| let num = m, den = 1n; | |
| if (k >= 0) num *= pow10(k); else den *= pow10(-k); | |
| if (e >= 0) num <<= BigInt(e); else den <<= BigInt(-e); | |
| const q = num / den, r2 = 2n * (num % den); | |
| return r2 > den || (r2 === den && (q & 1n) === 1n) ? q + 1n : q; | |
| } | |
| const isNeg = (x) => x < 0 || Object.is(x, -0); | |
| // Python round(x, ndigits) for ndigits >= 0 (half-to-even on the exact value, then the nearest double). | |
| function pyRound(x, ndigits) { | |
| if (!Number.isFinite(x) || x === 0) return x; | |
| const r = Number(`${roundScaled(x, ndigits)}e-${ndigits}`); | |
| return x < 0 ? -r : r; | |
| } | |
| // format(x, f"{'+' if signed}.{digits}f") | |
| function formatFixed(x, digits, signed = false) { | |
| const n = roundScaled(x, digits).toString().padStart(digits + 1, "0"); | |
| const body = digits ? `${n.slice(0, -digits)}.${n.slice(-digits)}` : n; | |
| return (isNeg(x) ? "-" : signed ? "+" : "") + body; | |
| } | |
| // format(x, ".0%") | |
| const formatPercent0 = (x) => (Number.isFinite(x * 100) ? `${formatFixed(x * 100, 0)}%` : `${x < 0 ? "-" : ""}inf%`); | |
| // format(x, "g") (precision 6) | |
| function formatG(x, precision = 6) { | |
| if (x === 0) return Object.is(x, -0) ? "-0" : "0"; | |
| const ax = Math.abs(x); | |
| let exp = Math.floor(Math.log10(ax)); | |
| let n = roundScaled(ax, precision - 1 - exp); | |
| const top = pow10(precision); | |
| if (n >= top) { exp += 1; n = roundScaled(ax, precision - 1 - exp); } | |
| else if (n < top / 10n) { exp -= 1; n = roundScaled(ax, precision - 1 - exp); if (n >= top) { exp += 1; n = roundScaled(ax, precision - 1 - exp); } } | |
| const digits = n.toString(); | |
| let body; | |
| if (exp >= -4 && exp < precision) { | |
| if (exp >= 0) { | |
| const frac = digits.slice(exp + 1).replace(/0+$/, ""); | |
| body = digits.slice(0, exp + 1) + (frac ? `.${frac}` : ""); | |
| } else { | |
| body = `0.${"0".repeat(-exp - 1)}${digits.replace(/0+$/, "")}`; | |
| } | |
| } else { | |
| const frac = digits.slice(1).replace(/0+$/, ""); | |
| body = `${digits[0]}${frac ? `.${frac}` : ""}e${exp < 0 ? "-" : "+"}${String(Math.abs(exp)).padStart(2, "0")}`; | |
| } | |
| return (x < 0 ? "-" : "") + body; | |
| } | |
| // num / den as Python's int true division gives it (correctly rounded). | |
| const TWO_53 = 2n ** 53n; | |
| function ratioToDouble(num, den) { | |
| if (num >= -TWO_53 && num <= TWO_53 && den <= TWO_53) return Number(num) / Number(den); | |
| const neg = num < 0n; | |
| const a = neg ? -num : num; | |
| const bits = (v) => v.toString(2).length; | |
| const shift = Math.max(0, 64 + bits(den) - bits(a)); | |
| let q = (a << BigInt(shift)) / den; | |
| if (((a << BigInt(shift)) % den) !== 0n) q |= 1n; // sticky bit: no double rounding | |
| const r = Number(q) * 2 ** -shift; | |
| return neg ? -r : r; | |
| } | |
| const totalSeconds = (us) => ratioToDouble(us, US); | |
| // Sample times travel as plain Numbers (microseconds) while they sit within +-2^52 of the epoch (years 1828 to 2112), where | |
| // sums and differences are exact; anything else stays a BigInt. Comparing the two kinds is exact. | |
| const SAFE_US = 2 ** 52; | |
| const toSafe = (us) => { | |
| if (typeof us !== "bigint") return us; | |
| const n = Number(us); // rounded beyond 2^53, but then it is far outside the safe range anyway | |
| return n >= -SAFE_US && n <= SAFE_US ? n : us; | |
| }; | |
| const diffSeconds = (a, b) => (typeof a === "number" && typeof b === "number" ? (a - b) / 1e6 : totalSeconds(BigInt(a) - BigInt(b))); | |
| const floorDiv = (a, b) => { const q = a / b; return (a % b !== 0n && (a < 0n) !== (b < 0n)) ? q - 1n : q; }; | |
| // C round(): half away from zero. | |
| function cRound(v) { | |
| const t = Math.trunc(v); | |
| return Math.abs(v - t) >= 0.5 ? t + Math.sign(v) : t; | |
| } | |
| // timedelta(seconds=x) in microseconds, as CPython's delta_new builds it from a float. | |
| function timedeltaUs(seconds) { | |
| const whole = Math.trunc(seconds); | |
| let us = BigInt(whole) * US; | |
| const frac = seconds - whole; | |
| if (frac === 0) return us; | |
| const scaled = 1e6 * frac; | |
| const intPart = Math.trunc(scaled); | |
| us += BigInt(intPart); | |
| const leftover = scaled - intPart; | |
| if (leftover !== 0) { | |
| let wholeUs = cRound(leftover); | |
| if (Math.abs(wholeUs - leftover) === 0.5) { | |
| const odd = us & 1n ? 1 : 0; | |
| wholeUs = 2.0 * cRound((leftover + odd) * 0.5) - odd; | |
| } | |
| us += BigInt(wholeUs); | |
| } | |
| return us; | |
| } | |
| // ---- time ---------------------------------------------------------------------------------------- | |
| // trusted_baseline.parse_time as microseconds since the epoch (BigInt) or null: an ISO string (Z or an | |
| // offset; no zone means UTC) or an already parsed BigInt. rules.js (which mirrors sensor_quality.parse_timestamp) | |
| // rejects a zoneless time, so the common zoneless forms (date, date and time) are completed to UTC first; an | |
| // exotic zoneless spelling (week dates, basic format with a time) reads as unusable here, where Python assumes UTC. | |
| function parseTime(value) { | |
| if (typeof value === "bigint") return value; | |
| if (typeof value !== "string") return null; | |
| const text = value.trim().split("Z").join("+00:00"); | |
| const aware = R.parseTimestampUs(text); | |
| if (aware !== null) return aware; | |
| if (/^\d{4}-\d{2}-\d{2}$/.test(text) || /^\d{8}$/.test(text)) return R.parseTimestampUs(`${text}T00:00:00+00:00`); | |
| if (/^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(:\d{2}([.,]\d{1,6})?)?$/.test(text)) return R.parseTimestampUs(`${text}+00:00`); | |
| return null; | |
| } | |
| // datetime.isoformat() of an aware UTC time given as microseconds since the epoch. | |
| function isoformatUs(microseconds) { | |
| const us = BigInt(microseconds); | |
| const dayUs = 86400n * US; | |
| const days = floorDiv(us, dayUs); | |
| const rem = us - days * dayUs; | |
| // civil date from days since 1970-01-01 (proleptic Gregorian) | |
| const z = Number(days) + 719468; | |
| const era = Math.floor(z / 146097); | |
| const doe = z - era * 146097; | |
| const yoe = Math.floor((doe - Math.floor(doe / 1460) + Math.floor(doe / 36524) - Math.floor(doe / 146096)) / 365); | |
| const doy = doe - (365 * yoe + Math.floor(yoe / 4) - Math.floor(yoe / 100)); | |
| const mp = Math.floor((5 * doy + 2) / 153); | |
| const day = doy - Math.floor((153 * mp + 2) / 5) + 1; | |
| const month = mp < 10 ? mp + 3 : mp - 9; | |
| const year = yoe + era * 400 + (month <= 2 ? 1 : 0); | |
| const secs = Number(rem / US), micro = Number(rem % US); | |
| const p = (n, w = 2) => String(n).padStart(w, "0"); | |
| return `${p(year, 4)}-${p(month)}-${p(day)}T${p(Math.floor(secs / 3600))}:${p(Math.floor(secs / 60) % 60)}:${p(secs % 60)}` + | |
| `${micro ? `.${p(micro, 6)}` : ""}+00:00`; | |
| } | |
| // ---- trusted_baseline.py -------------------------------------------------------------------------- | |
| const readNumber = (value) => (typeof value === "number" && Number.isFinite(value) ? value : null); | |
| // The values sorted ascending (a typed array when it is safe to use one: its order puts -0 before 0, where Python's stable sort keeps arrival order). | |
| function sortedCopy(values) { | |
| for (const v of values) if (v === 0 && 1 / v < 0) return [...values].sort(cmp); | |
| return Float64Array.from(values).sort(); | |
| } | |
| // The 10th to 90th percentile range of values SORTED (at least one). | |
| function percentileRange(ordered) { | |
| const count = ordered.length; | |
| return ordered[Math.trunc(0.9 * (count - 1))] - ordered[Math.trunc(0.1 * (count - 1))]; | |
| } | |
| // before: [stamp, value] rows, newest first. | |
| function norm(before) { | |
| if (before.length < 2) return {}; | |
| let changes = 0, longest = 0.0; | |
| let groupNewest = before[0][0]; | |
| let [lastStamp, lastValue] = before[0]; | |
| for (let i = 1; i < before.length; i++) { | |
| const [stamp, value] = before[i]; | |
| if (Math.abs(value - lastValue) > IDENTICAL_EPSILON) { | |
| changes += 1; | |
| longest = Math.max(longest, diffSeconds(groupNewest, lastStamp)); | |
| groupNewest = stamp; | |
| } | |
| lastStamp = stamp; lastValue = value; | |
| } | |
| longest = Math.max(longest, diffSeconds(groupNewest, lastStamp)); | |
| const steps = before.length - 1; | |
| return { change_share: pyRound(changes / steps, 4), change_steps: steps, longest_run_s: longest }; | |
| } | |
| const lagged = (old, minN) => (old.length && old.length >= minN ? { spread_lag: pyRound(percentileRange(old), 6) } : {}); | |
| function summary(ordered, windowS, changedAt, lastValue, anchor, before, lag) { | |
| const count = ordered.length; | |
| const middle = Math.floor(count / 2); | |
| return { median: count % 2 ? ordered[middle] : (ordered[middle - 1] + ordered[middle]) / 2, n: count, window_s: windowS, | |
| spread: pyRound(percentileRange(ordered), 6), last_change_at: isoformatUs(changedAt), last_value: lastValue, | |
| last_at: isoformatUs(anchor), ...norm(before), ...lag }; | |
| } | |
| // One field's newest-first walk over its (sample time, value, vetoed) rows, one push per row. | |
| class Walk { | |
| constructor(field, windowS, startAt, normSteps = 0) { | |
| this.field = field; | |
| this.alias = has(ALIASES, field) ? ALIASES[field] : null; | |
| this.windowS = windowS; | |
| this.startAt = startAt; | |
| this.normSteps = normSteps; | |
| this.newest = null; this.changedAt = null; this.anchor = null; | |
| this.runOpen = true; this.windowOpen = true; this.open = true; | |
| this.normOpen = normSteps > 0; | |
| this.values = []; this.stamps = []; this.before = []; | |
| } | |
| push(stamp, value, vetoed) { | |
| const startAt = this.startAt; | |
| if (this.newest === null) { this.newest = value; this.changedAt = stamp; } | |
| else if (this.runOpen) { | |
| if (Math.abs(value - this.newest) <= IDENTICAL_EPSILON) this.changedAt = stamp; | |
| else this.runOpen = false; | |
| } | |
| if (this.windowOpen) { | |
| if (startAt !== null && stamp < startAt) this.windowOpen = false; | |
| else if (!vetoed) { | |
| if (this.anchor === null) this.anchor = stamp; | |
| else if (diffSeconds(this.anchor, stamp) > this.windowS) this.windowOpen = false; | |
| if (this.windowOpen) { | |
| this.values.push(value); | |
| if (this.normSteps > 0) this.stamps.push(stamp); | |
| } | |
| } | |
| } | |
| if (this.normOpen && !this.runOpen) { | |
| if (startAt !== null && stamp < startAt) this.normOpen = false; | |
| else if (!vetoed) { | |
| this.before.push([stamp, value]); | |
| this.normOpen = this.before.length <= this.normSteps; | |
| } | |
| } | |
| this.open = this.runOpen || this.windowOpen || this.normOpen; | |
| return this.open; | |
| } | |
| stats(minN, now = null, maxAgeS = null) { | |
| if (this.newest === null) return null; | |
| const values = this.values, anchor = this.anchor; | |
| const startAt = this.startAt; | |
| const changedAt = startAt === null ? this.changedAt : (this.changedAt > startAt ? this.changedAt : startAt); | |
| if (values.length < minN || (maxAgeS !== null && now !== null && diffSeconds(now, anchor) > maxAgeS)) return null; | |
| const old = []; | |
| values.forEach((v, i) => { if (i < this.stamps.length && diffSeconds(anchor, this.stamps[i]) > SPREAD_LAG_S) old.push(v); }); | |
| return summary(sortedCopy(values), this.windowS, changedAt, values[0], anchor, this.before, lagged(sortedCopy(old), minN)); | |
| } | |
| } | |
| function latestCalibrations(calibrations, limit) { | |
| const latest = {}; | |
| for (const item of calibrations) { | |
| const stamp = toSafe(parseTime(get(item, "performed_at"))); | |
| if (stamp === null || (limit !== null && stamp > limit)) continue; | |
| const measurement = get(item, "measurement"); | |
| const fields = typeof measurement === "string" && has(MEASUREMENT_FIELDS, measurement) ? MEASUREMENT_FIELDS[measurement] : []; | |
| for (const field of fields) if (!has(latest, field) || stamp > latest[field]) latest[field] = stamp; | |
| } | |
| return latest; | |
| } | |
| /** | |
| * stream_baseline(prior, calibrations, now, window_s, min_n, max_age_s): {field: stats} for every field that has stats. | |
| * prior: the stream's readings sampled before the current one, oldest first, each {timestamp, <fields>, vetoed_fields} | |
| * (timestamp: ISO text or microseconds as BigInt; field values are numbers). now: BigInt microseconds or null. | |
| */ | |
| function streamBaseline(prior, calibrations, now, windowS = DEFAULT_WINDOW_S, minN = DEFAULT_MIN_N, maxAgeS = null) { | |
| const limit = now === undefined || now === null ? null : toSafe(now); | |
| const latest = latestCalibrations(calibrations || [], limit); | |
| const walks = BASELINE_FIELDS.map((field) => new Walk(field, windowS, has(latest, field) ? latest[field] : null, | |
| NORM_FIELDS.includes(field) ? NORM_STEPS : 0)); | |
| let openWalks = walks; | |
| for (let p = prior.length - 1; p >= 0; p--) { | |
| const packet = prior[p]; | |
| let stamp = null, vetoed = null, closed = false; | |
| for (const walk of openWalks) { | |
| const value = readNumber(packet[walk.field]); // readings are plain objects: no inherited fields | |
| if (value === null) continue; | |
| if (stamp === null) { | |
| stamp = toSafe(parseTime(get(packet, "timestamp"))); | |
| if (stamp === null || (limit !== null && stamp > limit)) stamp = false; | |
| else vetoed = get(packet, "vetoed_fields") || []; | |
| } | |
| if (stamp === false) break; // an unreadable or future sample time is no row for any field | |
| if (!walk.push(stamp, value, vetoed.includes(walk.field) || (walk.alias !== null && vetoed.includes(walk.alias)))) closed = true; | |
| } | |
| if (closed) { | |
| openWalks = openWalks.filter((walk) => walk.open); | |
| if (!openWalks.length) break; | |
| } | |
| } | |
| const out = {}; | |
| for (const walk of walks) { | |
| const stats = walk.stats(minN, limit, maxAgeS); | |
| if (stats !== null) out[walk.field] = stats; | |
| } | |
| return out; | |
| } | |
| // ---- sensor_quality.py ---------------------------------------------------------------------------- | |
| function seriesPoints(history, sensor, field) { | |
| const points = []; | |
| for (const packet of [...history, sensor]) { | |
| if (!isPlainObject(packet)) continue; | |
| const parsed = numeric(get(packet, field)); | |
| if (parsed !== null) points.push([parsed, packet]); | |
| } | |
| return points; | |
| } | |
| function trailingRun(values) { | |
| let run = 0; | |
| for (let i = values.length - 1; i >= 0; i--) { | |
| if (run && Math.abs(values[i] - values[values.length - 1]) > STUCK_EPSILON) break; | |
| run++; | |
| } | |
| return run; | |
| } | |
| function median(values) { | |
| const o = [...values].sort(cmp); | |
| const m = Math.floor(o.length / 2); | |
| return o.length % 2 ? o[m] : (o[m - 1] + o[m]) / 2; | |
| } | |
| // The stream changed in at least 70 % of its last trusted steps (and has 15 of them). | |
| function isSteadyStream(stats) { | |
| const share = numeric(get(stats, "change_share")); | |
| const steps = numeric(get(stats, "change_steps")); | |
| return share !== null && steps !== null && steps >= ADAPTIVE_STUCK_MIN_STEPS && share >= ADAPTIVE_STUCK_MIN_CHANGE_SHARE; | |
| } | |
| // Microseconds, or null when the stream is not judged. | |
| function adaptiveStuckLimitUs(stats) { | |
| const share = numeric(get(stats, "change_share")); | |
| const steps = numeric(get(stats, "change_steps")); | |
| const longest = numeric(get(stats, "longest_run_s")); | |
| if (share === null || steps === null || longest === null || longest < 0 || | |
| steps < ADAPTIVE_STUCK_MIN_STEPS || share < ADAPTIVE_STUCK_MIN_CHANGE_SHARE) return null; | |
| const run = timedeltaUs(ADAPTIVE_STUCK_RUN_FACTOR * longest); | |
| return run > ADAPTIVE_STUCK_FLOOR_US ? run : ADAPTIVE_STUCK_FLOOR_US; | |
| } | |
| function applyStepCheck(field, current, sampledUs, stats, labels, suspect, checks) { | |
| const lastValue = numeric(get(stats, "last_value")); | |
| const lastAt = R.parseTimestampUs(get(stats, "last_at")); | |
| if (lastValue === null || lastAt === null || sampledUs === null) return; | |
| if (current > lastValue && lastValue < (has(STEP_REWET_BELOW_PCT, field) ? STEP_REWET_BELOW_PCT[field] : -Infinity)) return; | |
| const [smallest, speed, unit, instrument] = STEP_LIMITS[field]; | |
| const minutes = Math.max(totalSeconds(sampledUs - lastAt), 0.0) / 60; | |
| const allowed = Math.max(smallest, speed * minutes); | |
| if (Math.abs(current - lastValue) > allowed) { | |
| addUnique(labels, "sensor_drift_possible"); | |
| addUnique(suspect, field); | |
| checks.push(`compare ${field} with ${instrument}: it moved ${formatFixed(current - lastValue, 1, true)} ${unit} in ` + | |
| `${formatFixed(minutes, 1)} minutes, faster than it can change; ${CALIBRATION_HINT}`); | |
| } | |
| } | |
| function applyNoiseCheck(field, sensor, history, labels, suspect, checks) { | |
| const points = seriesPoints(history, sensor, field).slice(-NOISE_WINDOW); | |
| if (points.length < NOISE_MIN_SAMPLES) return; | |
| const stamps = points.map(([, packet]) => R.parseTimestampUs(get(packet, "timestamp"))); | |
| const gaps = []; | |
| for (let i = 1; i < stamps.length; i++) { | |
| if (stamps[i - 1] === null || stamps[i] === null) continue; | |
| const g = totalSeconds(stamps[i] - stamps[i - 1]); | |
| if (g > 0) gaps.push(g); | |
| } | |
| gaps.sort(cmp); | |
| if (!gaps.length) return; | |
| let limit; | |
| if (has(NOISE_FIXED_LIMIT, field)) { | |
| let longestInterval; | |
| [limit, longestInterval] = NOISE_FIXED_LIMIT[field]; | |
| if (median(gaps) > longestInterval) return; // no real data behind a limit for a slower stream | |
| } else { | |
| limit = Math.max(NOISE_STEP_FLOOR[field], | |
| NOISE_STEP_LIMIT_5MIN[field] * Math.sqrt(median(gaps) / NOISE_REFERENCE_INTERVAL_S)); | |
| } | |
| const values = points.map((p) => p[0]); | |
| const steps = values.slice(1).map((b, i) => Math.abs(b - values[i])); | |
| let travelled = 0.0; | |
| for (const step of steps) travelled += step; // left to right, as the Python loop does | |
| if (median(steps) >= limit && Math.abs(values[values.length - 1] - values[0]) <= NOISE_MAX_NET_SHARE * travelled) { | |
| addUnique(labels, "noisy_signal_possible"); | |
| addUnique(suspect, field); | |
| checks.push(`inspect ${field}: readings jump between samples; check probe shielding, wiring and averaging`); | |
| } | |
| } | |
| /** apply_persistent_checks: judge the reading against Core's trusted reference of its stream (mutates labels, suspect, checks). */ | |
| function applyPersistentChecks(sensor, history, streamBaselineStats, labels, suspect, checks) { | |
| if (!isPlainObject(streamBaselineStats)) return; | |
| const sampled = R.parseTimestampUs(get(sensor, "timestamp")); | |
| let previousPacket = {}; | |
| for (let i = history.length - 1; i >= 0; i--) if (isPlainObject(history[i])) { previousPacket = history[i]; break; } | |
| for (const field of Object.keys(streamBaselineStats)) { | |
| const stats = streamBaselineStats[field]; | |
| const current = numeric(get(sensor, field)); | |
| if (!isPlainObject(stats) || current === null) continue; | |
| const med = numeric(get(stats, "median")); | |
| let threshold = has(BASELINE_DRIFT_THRESHOLDS, field) ? BASELINE_DRIFT_THRESHOLDS[field] : null; | |
| const [low, high] = has(PLAUSIBLE_RANGES, field) ? PLAUSIBLE_RANGES[field] : [-Infinity, Infinity]; | |
| let spread = numeric(get(stats, "spread")) || 0.0; | |
| const laggedSpread = numeric(get(stats, "spread_lag")); | |
| if (LAGGED_SPREAD_FIELDS.includes(field) && laggedSpread !== null && laggedSpread >= 0 && isSteadyStream(stats)) spread = laggedSpread; | |
| if (threshold !== null) threshold = Math.max(threshold, DRIFT_SPREAD_FACTOR * spread); | |
| if (med !== null && threshold !== null && low <= current && current <= high && Math.abs(current - med) >= threshold) { | |
| addUnique(labels, "baseline_drift_possible"); | |
| addUnique(suspect, field); | |
| checks.push(`verify ${field} with a meter: ${formatG(current)} differs from the trusted level ${formatG(med)}; ${CALIBRATION_HINT}`); | |
| } | |
| let limit = has(PERSISTENT_STUCK_LIMITS_US, field) ? PERSISTENT_STUCK_LIMITS_US[field] : null; | |
| let habit = ""; | |
| const adaptive = limit === null && ADAPTIVE_STUCK_FIELDS.includes(field); | |
| if (adaptive) { | |
| limit = adaptiveStuckLimitUs(stats); | |
| habit = `, though this probe normally changes on ${formatPercent0(numeric(get(stats, "change_share")) || 0)} of its readings`; | |
| } | |
| const changedAt = R.parseTimestampUs(get(stats, "last_change_at")); | |
| const previous = numeric(get(previousPacket, field)); | |
| if (limit !== null && sampled !== null && changedAt !== null && previous !== null && | |
| Math.abs(current - previous) <= STUCK_EPSILON && sampled - changedAt > limit && | |
| (!adaptive || trailingRun(seriesPoints(history, sensor, field).map((p) => p[0])) >= ADAPTIVE_STUCK_MIN_RUN)) { | |
| addUnique(labels, "stuck_value"); | |
| addUnique(suspect, field); | |
| checks.push(`inspect ${field}: the same value for ${floorDiv(sampled - changedAt, MINUTE_US)} minutes${habit}; ${CALIBRATION_HINT}`); | |
| } | |
| if (has(STEP_LIMITS, field)) applyStepCheck(field, current, sampled, stats, labels, suspect, checks); | |
| if (has(NOISE_STEP_LIMIT_5MIN, field) || has(NOISE_FIXED_LIMIT, field)) applyNoiseCheck(field, sensor, history, labels, suspect, checks); | |
| } | |
| } | |
| // derive_sensor_quality's inputs are valid (otherwise it returns before any check runs). | |
| function validInputs(farmContext, sensor, expectedFields, history) { | |
| return isPlainObject(farmContext) && isPlainObject(sensor) && Array.isArray(expectedFields) && | |
| expectedFields.every((f) => typeof f === "string") && | |
| (history === null || (Array.isArray(history) && history.every((h) => h === null || isPlainObject(h)))); | |
| } | |
| /** | |
| * derive_sensor_quality(..., history=..., stream_baseline=...): deriveSensorQuality (rules.js) and then the persistent | |
| * checks. In Python they run inside it, before the default "continue routine monitoring" check and the review | |
| * flag are set; this redoes those two from the final lists. | |
| */ | |
| function derive(farmContext, sensor, expectedFields, nowUs = null, history = null, streamBaselineStats = null) { | |
| const out = R.deriveSensorQuality(farmContext, sensor, expectedFields, nowUs, history); | |
| if (!isPlainObject(streamBaselineStats) || !validInputs(farmContext, sensor, expectedFields, history)) return out; | |
| const labels = [...out.data_quality_labels], suspect = [...out.suspect_fields]; | |
| const checks = out.safe_next_checks.length === 1 && out.safe_next_checks[0] === ROUTINE_CHECK ? [] : [...out.safe_next_checks]; | |
| applyPersistentChecks(sensor, history || [], streamBaselineStats, labels, suspect, checks); | |
| if (!checks.length) checks.push(ROUTINE_CHECK); | |
| return { | |
| data_quality_labels: labels, missing_fields: out.missing_fields, suspect_fields: suspect, safe_next_checks: checks, | |
| human_review_required: Boolean(labels.length || out.missing_fields.length || suspect.length), | |
| rationale: labels.length ? "Sensor packet needs verification before downstream risk or action reasoning." | |
| : "Critical sensor readings are present and inside plausible ranges.", | |
| }; | |
| } | |
| /** | |
| * The trusted reference of one stream that is checked reading by reading, as Core builds it: the stream's earlier | |
| * readings (at most `scanLimit` of them, newest by arrival), those sampled before the current one, in sample-time | |
| * order. `add` each reading after it was checked, with the fields the pipeline vetoed. | |
| */ | |
| class StreamReference { | |
| constructor(scanLimit = DEFAULT_SCAN_LIMIT) { | |
| this.scanLimit = scanLimit; | |
| this.sorted = []; // entries by sample time, equal times in arrival order | |
| this.arrival = []; // the same entries by arrival | |
| } | |
| baseline(stampUs, calibrations = []) { | |
| const sorted = this.sorted; | |
| let lo = 0, hi = sorted.length; // entries sampled before stampUs | |
| while (lo < hi) { const mid = (lo + hi) >> 1; if (sorted[mid].timestamp < stampUs) lo = mid + 1; else hi = mid; } | |
| return streamBaseline(lo === sorted.length ? sorted : sorted.slice(0, lo), calibrations, stampUs); | |
| } | |
| add(packet, stampUs, vetoedFields) { | |
| const entry = { timestamp: stampUs, vetoed_fields: vetoedFields }; | |
| for (const field of BASELINE_FIELDS) entry[field] = numeric(get(packet, field)); | |
| const sorted = this.sorted; | |
| let lo = 0, hi = sorted.length; // after the entries sampled at the same time | |
| while (lo < hi) { const mid = (lo + hi) >> 1; if (sorted[mid].timestamp <= stampUs) lo = mid + 1; else hi = mid; } | |
| sorted.splice(lo, 0, entry); | |
| this.arrival.push(entry); | |
| if (this.arrival.length > this.scanLimit) { | |
| const oldest = this.arrival.shift(); | |
| sorted.splice(sorted.indexOf(oldest), 1); | |
| } | |
| } | |
| } | |
| return { derive, applyPersistentChecks, streamBaseline, StreamReference, parseTime, isoformatUs, | |
| pyRound, formatG, formatFixed, formatPercent0, totalSeconds, timedeltaUs, adaptiveStuckLimitUs, | |
| BASELINE_FIELDS, DEFAULT_SCAN_LIMIT }; | |
| }); | |