File size: 31,025 Bytes
ba35560 f0907bb ba35560 f0907bb ba35560 f0907bb ba35560 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 | /*
* Pomona persistent sensor checks, ported to JavaScript: Core's trusted per-stream reference and the
* checks that judge a reading against it.
*
* Source of truth:
* services/core/app/trusted_baseline.py stream_baseline (the one-pass reference; the incremental
* StreamBaseline tracker is byte-identical to it and not ported)
* services/model-router/app/sensor_quality.py apply_persistent_checks and what it calls (adaptive pH/EC
* stuck, step check, noise check, level drift, persistent stuck limits)
* tests/persistent.test.mjs checks this file against the Python code (cases from
* tests/make_persistent_cases.py); run it after changing either side.
*
* rules.js (the stateless checks over the last 11 packets) is untouched and stays an exact copy of the one in
* the greenhouse demo Space. This file sits next to it: `derive` is deriveSensorQuality plus the persistent step.
*
* Python things reproduced on purpose: round(x, n) is half-to-even on the exact binary value, `:g` / `:.1f` /
* `:.0%` format the exact value half-to-even (toPrecision/toFixed round half up), timedelta arithmetic is exact
* in microseconds, and total_seconds() is a correctly rounded int / 10**6.
*/
(function (root, factory) {
if (typeof module === "object" && module.exports) module.exports = factory(require("./rules.js"));
else root.PomonaPersistent = factory(root.PomonaRules);
})(typeof self !== "undefined" ? self : this, function (R) {
"use strict";
const US = R.US_PER_S;
const numeric = R.numeric;
const has = (obj, key) => Object.prototype.hasOwnProperty.call(obj, key);
const isPlainObject = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
const get = (obj, key) => (has(obj, key) ? obj[key] : undefined);
const isNone = (v) => v === undefined || v === null;
const addUnique = (items, value) => { if (!items.includes(value)) items.push(value); };
const cmp = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
// ---- constants (trusted_baseline.py) ------------------------------------------------------------
const DEFAULT_WINDOW_S = 2 * 3600;
const DEFAULT_MIN_N = 6;
const DEFAULT_SCAN_LIMIT = 5000;
const IDENTICAL_EPSILON = 1e-9;
const NORM_STEPS = 200;
const NORM_FIELDS = ["ph", "ec_ms_cm"];
const SPREAD_LAG_S = 1800;
const BASELINE_FIELDS = ["ph", "ec_ms_cm", "air_temperature_c", "humidity_pct", "soil_moisture_pct",
"substrate_moisture_pct", "substrate_temperature_c"];
const MEASUREMENT_FIELDS = {
ph: ["ph"], ec_ms_cm: ["ec_ms_cm"], air_temperature_c: ["air_temperature_c"], humidity_pct: ["humidity_pct"],
soil_moisture_pct: ["soil_moisture_pct", "substrate_moisture_pct"],
};
const ALIASES = { soil_moisture_pct: "substrate_moisture_pct", substrate_moisture_pct: "soil_moisture_pct" };
// ---- constants (sensor_quality.py) --------------------------------------------------------------
const STUCK_EPSILON = 1e-9;
const BASELINE_DRIFT_THRESHOLDS = { ph: 0.35, ec_ms_cm: 0.4 };
const PLAUSIBLE_RANGES = { ph: [3.0, 11.0], ec_ms_cm: [0.0, 12.0] };
const MINUTE_US = 60n * US;
const HOUR_US = 3600n * US;
const PERSISTENT_STUCK_LIMITS_US = { air_temperature_c: 45n * MINUTE_US, humidity_pct: 30n * MINUTE_US,
substrate_temperature_c: 6n * HOUR_US, water_temperature_c: 3n * HOUR_US };
const ADAPTIVE_STUCK_FIELDS = ["ph", "ec_ms_cm"];
const ADAPTIVE_STUCK_FLOOR_US = 5n * MINUTE_US;
const ADAPTIVE_STUCK_RUN_FACTOR = 4.0;
const ADAPTIVE_STUCK_MIN_STEPS = 15;
const ADAPTIVE_STUCK_MIN_CHANGE_SHARE = 0.7;
const ADAPTIVE_STUCK_MIN_RUN = 3;
// field: [smallest allowance, speed per minute, unit, instrument]
const STEP_LIMITS = {
air_temperature_c: [3.0, 1.0, "C", "a second thermometer"],
humidity_pct: [12.0, 4.0, "%RH", "a second hygrometer"],
soil_moisture_pct: [8.0, 2.0, "%", "a hand-held moisture meter"],
substrate_moisture_pct: [8.0, 2.0, "%", "a hand-held moisture meter"],
substrate_temperature_c: [1.5, 0.3, "C", "a second thermometer"],
};
const STEP_REWET_BELOW_PCT = { soil_moisture_pct: 15.0, substrate_moisture_pct: 15.0 };
const NOISE_STEP_LIMIT_5MIN = { humidity_pct: 10.0, air_temperature_c: 2.0, ph: 0.5,
soil_moisture_pct: 1.0, substrate_moisture_pct: 1.0, substrate_temperature_c: 0.4 };
const NOISE_STEP_FLOOR = { humidity_pct: 3.0, air_temperature_c: 0.4, ph: 0.1,
soil_moisture_pct: 2.5, substrate_moisture_pct: 2.5, substrate_temperature_c: 0.5 };
// field: [limit as it is, in the field's unit; largest median sample interval of the window it is judged at, in seconds].
// EC is judged only on a fast stream (90 s or less): 3 months of real 10-second tank EC (four probes, about 1.2 mS/cm)
// never give a 12-reading window with a median step above 0.143 mS/cm at any rate from 10 to 90 s, so 0.20 flags
// none of about 0.92 million windows per probe. A slower stream has no real data behind a limit (5-minute drain EC
// really jumps) and is not judged. The limit is used as it is, without the square-root scaling of the other fields.
const NOISE_FIXED_LIMIT = { ec_ms_cm: [0.20, 90.0] };
const NOISE_REFERENCE_INTERVAL_S = 300.0;
const NOISE_WINDOW = 12;
const NOISE_MAX_NET_SHARE = 0.5;
const NOISE_MIN_SAMPLES = 6;
const DRIFT_SPREAD_FACTOR = 3.0;
const LAGGED_SPREAD_FIELDS = ["ph"];
const CALIBRATION_HINT = "if a meter agrees with the reading, it is real: record a calibration for this probe to accept it";
const ROUTINE_CHECK = "continue routine monitoring";
// ---- exact number work: Python's float formatting, round() and timedelta ----------------------------
const F64 = new DataView(new ArrayBuffer(8));
// |x| = m * 2^e exactly, for a finite double.
function decompose(x) {
F64.setFloat64(0, x);
const hi = F64.getUint32(0), lo = F64.getUint32(4);
const exp = (hi >>> 20) & 0x7ff;
let m = (BigInt(hi & 0xfffff) << 32n) | BigInt(lo);
if (exp === 0) return { m, e: -1074 };
return { m: m | (1n << 52n), e: exp - 1075 };
}
const pow10 = (k) => 10n ** BigInt(k);
// round-half-even(|x| * 10^k) as a BigInt, from the exact binary value (k may be negative).
function roundScaled(x, k) {
const { m, e } = decompose(Math.abs(x));
let num = m, den = 1n;
if (k >= 0) num *= pow10(k); else den *= pow10(-k);
if (e >= 0) num <<= BigInt(e); else den <<= BigInt(-e);
const q = num / den, r2 = 2n * (num % den);
return r2 > den || (r2 === den && (q & 1n) === 1n) ? q + 1n : q;
}
const isNeg = (x) => x < 0 || Object.is(x, -0);
// Python round(x, ndigits) for ndigits >= 0 (half-to-even on the exact value, then the nearest double).
function pyRound(x, ndigits) {
if (!Number.isFinite(x) || x === 0) return x;
const r = Number(`${roundScaled(x, ndigits)}e-${ndigits}`);
return x < 0 ? -r : r;
}
// format(x, f"{'+' if signed}.{digits}f")
function formatFixed(x, digits, signed = false) {
const n = roundScaled(x, digits).toString().padStart(digits + 1, "0");
const body = digits ? `${n.slice(0, -digits)}.${n.slice(-digits)}` : n;
return (isNeg(x) ? "-" : signed ? "+" : "") + body;
}
// format(x, ".0%")
const formatPercent0 = (x) => (Number.isFinite(x * 100) ? `${formatFixed(x * 100, 0)}%` : `${x < 0 ? "-" : ""}inf%`);
// format(x, "g") (precision 6)
function formatG(x, precision = 6) {
if (x === 0) return Object.is(x, -0) ? "-0" : "0";
const ax = Math.abs(x);
let exp = Math.floor(Math.log10(ax));
let n = roundScaled(ax, precision - 1 - exp);
const top = pow10(precision);
if (n >= top) { exp += 1; n = roundScaled(ax, precision - 1 - exp); }
else if (n < top / 10n) { exp -= 1; n = roundScaled(ax, precision - 1 - exp); if (n >= top) { exp += 1; n = roundScaled(ax, precision - 1 - exp); } }
const digits = n.toString();
let body;
if (exp >= -4 && exp < precision) {
if (exp >= 0) {
const frac = digits.slice(exp + 1).replace(/0+$/, "");
body = digits.slice(0, exp + 1) + (frac ? `.${frac}` : "");
} else {
body = `0.${"0".repeat(-exp - 1)}${digits.replace(/0+$/, "")}`;
}
} else {
const frac = digits.slice(1).replace(/0+$/, "");
body = `${digits[0]}${frac ? `.${frac}` : ""}e${exp < 0 ? "-" : "+"}${String(Math.abs(exp)).padStart(2, "0")}`;
}
return (x < 0 ? "-" : "") + body;
}
// num / den as Python's int true division gives it (correctly rounded).
const TWO_53 = 2n ** 53n;
function ratioToDouble(num, den) {
if (num >= -TWO_53 && num <= TWO_53 && den <= TWO_53) return Number(num) / Number(den);
const neg = num < 0n;
const a = neg ? -num : num;
const bits = (v) => v.toString(2).length;
const shift = Math.max(0, 64 + bits(den) - bits(a));
let q = (a << BigInt(shift)) / den;
if (((a << BigInt(shift)) % den) !== 0n) q |= 1n; // sticky bit: no double rounding
const r = Number(q) * 2 ** -shift;
return neg ? -r : r;
}
const totalSeconds = (us) => ratioToDouble(us, US);
// Sample times travel as plain Numbers (microseconds) while they sit within +-2^52 of the epoch (years 1828 to 2112), where
// sums and differences are exact; anything else stays a BigInt. Comparing the two kinds is exact.
const SAFE_US = 2 ** 52;
const toSafe = (us) => {
if (typeof us !== "bigint") return us;
const n = Number(us); // rounded beyond 2^53, but then it is far outside the safe range anyway
return n >= -SAFE_US && n <= SAFE_US ? n : us;
};
const diffSeconds = (a, b) => (typeof a === "number" && typeof b === "number" ? (a - b) / 1e6 : totalSeconds(BigInt(a) - BigInt(b)));
const floorDiv = (a, b) => { const q = a / b; return (a % b !== 0n && (a < 0n) !== (b < 0n)) ? q - 1n : q; };
// C round(): half away from zero.
function cRound(v) {
const t = Math.trunc(v);
return Math.abs(v - t) >= 0.5 ? t + Math.sign(v) : t;
}
// timedelta(seconds=x) in microseconds, as CPython's delta_new builds it from a float.
function timedeltaUs(seconds) {
const whole = Math.trunc(seconds);
let us = BigInt(whole) * US;
const frac = seconds - whole;
if (frac === 0) return us;
const scaled = 1e6 * frac;
const intPart = Math.trunc(scaled);
us += BigInt(intPart);
const leftover = scaled - intPart;
if (leftover !== 0) {
let wholeUs = cRound(leftover);
if (Math.abs(wholeUs - leftover) === 0.5) {
const odd = us & 1n ? 1 : 0;
wholeUs = 2.0 * cRound((leftover + odd) * 0.5) - odd;
}
us += BigInt(wholeUs);
}
return us;
}
// ---- time ----------------------------------------------------------------------------------------
// trusted_baseline.parse_time as microseconds since the epoch (BigInt) or null: an ISO string (Z or an
// offset; no zone means UTC) or an already parsed BigInt. rules.js (which mirrors sensor_quality.parse_timestamp)
// rejects a zoneless time, so the common zoneless forms (date, date and time) are completed to UTC first; an
// exotic zoneless spelling (week dates, basic format with a time) reads as unusable here, where Python assumes UTC.
function parseTime(value) {
if (typeof value === "bigint") return value;
if (typeof value !== "string") return null;
const text = value.trim().split("Z").join("+00:00");
const aware = R.parseTimestampUs(text);
if (aware !== null) return aware;
if (/^\d{4}-\d{2}-\d{2}$/.test(text) || /^\d{8}$/.test(text)) return R.parseTimestampUs(`${text}T00:00:00+00:00`);
if (/^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(:\d{2}([.,]\d{1,6})?)?$/.test(text)) return R.parseTimestampUs(`${text}+00:00`);
return null;
}
// datetime.isoformat() of an aware UTC time given as microseconds since the epoch.
function isoformatUs(microseconds) {
const us = BigInt(microseconds);
const dayUs = 86400n * US;
const days = floorDiv(us, dayUs);
const rem = us - days * dayUs;
// civil date from days since 1970-01-01 (proleptic Gregorian)
const z = Number(days) + 719468;
const era = Math.floor(z / 146097);
const doe = z - era * 146097;
const yoe = Math.floor((doe - Math.floor(doe / 1460) + Math.floor(doe / 36524) - Math.floor(doe / 146096)) / 365);
const doy = doe - (365 * yoe + Math.floor(yoe / 4) - Math.floor(yoe / 100));
const mp = Math.floor((5 * doy + 2) / 153);
const day = doy - Math.floor((153 * mp + 2) / 5) + 1;
const month = mp < 10 ? mp + 3 : mp - 9;
const year = yoe + era * 400 + (month <= 2 ? 1 : 0);
const secs = Number(rem / US), micro = Number(rem % US);
const p = (n, w = 2) => String(n).padStart(w, "0");
return `${p(year, 4)}-${p(month)}-${p(day)}T${p(Math.floor(secs / 3600))}:${p(Math.floor(secs / 60) % 60)}:${p(secs % 60)}` +
`${micro ? `.${p(micro, 6)}` : ""}+00:00`;
}
// ---- trusted_baseline.py --------------------------------------------------------------------------
const readNumber = (value) => (typeof value === "number" && Number.isFinite(value) ? value : null);
// The values sorted ascending (a typed array when it is safe to use one: its order puts -0 before 0, where Python's stable sort keeps arrival order).
function sortedCopy(values) {
for (const v of values) if (v === 0 && 1 / v < 0) return [...values].sort(cmp);
return Float64Array.from(values).sort();
}
// The 10th to 90th percentile range of values SORTED (at least one).
function percentileRange(ordered) {
const count = ordered.length;
return ordered[Math.trunc(0.9 * (count - 1))] - ordered[Math.trunc(0.1 * (count - 1))];
}
// before: [stamp, value] rows, newest first.
function norm(before) {
if (before.length < 2) return {};
let changes = 0, longest = 0.0;
let groupNewest = before[0][0];
let [lastStamp, lastValue] = before[0];
for (let i = 1; i < before.length; i++) {
const [stamp, value] = before[i];
if (Math.abs(value - lastValue) > IDENTICAL_EPSILON) {
changes += 1;
longest = Math.max(longest, diffSeconds(groupNewest, lastStamp));
groupNewest = stamp;
}
lastStamp = stamp; lastValue = value;
}
longest = Math.max(longest, diffSeconds(groupNewest, lastStamp));
const steps = before.length - 1;
return { change_share: pyRound(changes / steps, 4), change_steps: steps, longest_run_s: longest };
}
const lagged = (old, minN) => (old.length && old.length >= minN ? { spread_lag: pyRound(percentileRange(old), 6) } : {});
function summary(ordered, windowS, changedAt, lastValue, anchor, before, lag) {
const count = ordered.length;
const middle = Math.floor(count / 2);
return { median: count % 2 ? ordered[middle] : (ordered[middle - 1] + ordered[middle]) / 2, n: count, window_s: windowS,
spread: pyRound(percentileRange(ordered), 6), last_change_at: isoformatUs(changedAt), last_value: lastValue,
last_at: isoformatUs(anchor), ...norm(before), ...lag };
}
// One field's newest-first walk over its (sample time, value, vetoed) rows, one push per row.
class Walk {
constructor(field, windowS, startAt, normSteps = 0) {
this.field = field;
this.alias = has(ALIASES, field) ? ALIASES[field] : null;
this.windowS = windowS;
this.startAt = startAt;
this.normSteps = normSteps;
this.newest = null; this.changedAt = null; this.anchor = null;
this.runOpen = true; this.windowOpen = true; this.open = true;
this.normOpen = normSteps > 0;
this.values = []; this.stamps = []; this.before = [];
}
push(stamp, value, vetoed) {
const startAt = this.startAt;
if (this.newest === null) { this.newest = value; this.changedAt = stamp; }
else if (this.runOpen) {
if (Math.abs(value - this.newest) <= IDENTICAL_EPSILON) this.changedAt = stamp;
else this.runOpen = false;
}
if (this.windowOpen) {
if (startAt !== null && stamp < startAt) this.windowOpen = false;
else if (!vetoed) {
if (this.anchor === null) this.anchor = stamp;
else if (diffSeconds(this.anchor, stamp) > this.windowS) this.windowOpen = false;
if (this.windowOpen) {
this.values.push(value);
if (this.normSteps > 0) this.stamps.push(stamp);
}
}
}
if (this.normOpen && !this.runOpen) {
if (startAt !== null && stamp < startAt) this.normOpen = false;
else if (!vetoed) {
this.before.push([stamp, value]);
this.normOpen = this.before.length <= this.normSteps;
}
}
this.open = this.runOpen || this.windowOpen || this.normOpen;
return this.open;
}
stats(minN, now = null, maxAgeS = null) {
if (this.newest === null) return null;
const values = this.values, anchor = this.anchor;
const startAt = this.startAt;
const changedAt = startAt === null ? this.changedAt : (this.changedAt > startAt ? this.changedAt : startAt);
if (values.length < minN || (maxAgeS !== null && now !== null && diffSeconds(now, anchor) > maxAgeS)) return null;
const old = [];
values.forEach((v, i) => { if (i < this.stamps.length && diffSeconds(anchor, this.stamps[i]) > SPREAD_LAG_S) old.push(v); });
return summary(sortedCopy(values), this.windowS, changedAt, values[0], anchor, this.before, lagged(sortedCopy(old), minN));
}
}
function latestCalibrations(calibrations, limit) {
const latest = {};
for (const item of calibrations) {
const stamp = toSafe(parseTime(get(item, "performed_at")));
if (stamp === null || (limit !== null && stamp > limit)) continue;
const measurement = get(item, "measurement");
const fields = typeof measurement === "string" && has(MEASUREMENT_FIELDS, measurement) ? MEASUREMENT_FIELDS[measurement] : [];
for (const field of fields) if (!has(latest, field) || stamp > latest[field]) latest[field] = stamp;
}
return latest;
}
/**
* stream_baseline(prior, calibrations, now, window_s, min_n, max_age_s): {field: stats} for every field that has stats.
* prior: the stream's readings sampled before the current one, oldest first, each {timestamp, <fields>, vetoed_fields}
* (timestamp: ISO text or microseconds as BigInt; field values are numbers). now: BigInt microseconds or null.
*/
function streamBaseline(prior, calibrations, now, windowS = DEFAULT_WINDOW_S, minN = DEFAULT_MIN_N, maxAgeS = null) {
const limit = now === undefined || now === null ? null : toSafe(now);
const latest = latestCalibrations(calibrations || [], limit);
const walks = BASELINE_FIELDS.map((field) => new Walk(field, windowS, has(latest, field) ? latest[field] : null,
NORM_FIELDS.includes(field) ? NORM_STEPS : 0));
let openWalks = walks;
for (let p = prior.length - 1; p >= 0; p--) {
const packet = prior[p];
let stamp = null, vetoed = null, closed = false;
for (const walk of openWalks) {
const value = readNumber(packet[walk.field]); // readings are plain objects: no inherited fields
if (value === null) continue;
if (stamp === null) {
stamp = toSafe(parseTime(get(packet, "timestamp")));
if (stamp === null || (limit !== null && stamp > limit)) stamp = false;
else vetoed = get(packet, "vetoed_fields") || [];
}
if (stamp === false) break; // an unreadable or future sample time is no row for any field
if (!walk.push(stamp, value, vetoed.includes(walk.field) || (walk.alias !== null && vetoed.includes(walk.alias)))) closed = true;
}
if (closed) {
openWalks = openWalks.filter((walk) => walk.open);
if (!openWalks.length) break;
}
}
const out = {};
for (const walk of walks) {
const stats = walk.stats(minN, limit, maxAgeS);
if (stats !== null) out[walk.field] = stats;
}
return out;
}
// ---- sensor_quality.py ----------------------------------------------------------------------------
function seriesPoints(history, sensor, field) {
const points = [];
for (const packet of [...history, sensor]) {
if (!isPlainObject(packet)) continue;
const parsed = numeric(get(packet, field));
if (parsed !== null) points.push([parsed, packet]);
}
return points;
}
function trailingRun(values) {
let run = 0;
for (let i = values.length - 1; i >= 0; i--) {
if (run && Math.abs(values[i] - values[values.length - 1]) > STUCK_EPSILON) break;
run++;
}
return run;
}
function median(values) {
const o = [...values].sort(cmp);
const m = Math.floor(o.length / 2);
return o.length % 2 ? o[m] : (o[m - 1] + o[m]) / 2;
}
// The stream changed in at least 70 % of its last trusted steps (and has 15 of them).
function isSteadyStream(stats) {
const share = numeric(get(stats, "change_share"));
const steps = numeric(get(stats, "change_steps"));
return share !== null && steps !== null && steps >= ADAPTIVE_STUCK_MIN_STEPS && share >= ADAPTIVE_STUCK_MIN_CHANGE_SHARE;
}
// Microseconds, or null when the stream is not judged.
function adaptiveStuckLimitUs(stats) {
const share = numeric(get(stats, "change_share"));
const steps = numeric(get(stats, "change_steps"));
const longest = numeric(get(stats, "longest_run_s"));
if (share === null || steps === null || longest === null || longest < 0 ||
steps < ADAPTIVE_STUCK_MIN_STEPS || share < ADAPTIVE_STUCK_MIN_CHANGE_SHARE) return null;
const run = timedeltaUs(ADAPTIVE_STUCK_RUN_FACTOR * longest);
return run > ADAPTIVE_STUCK_FLOOR_US ? run : ADAPTIVE_STUCK_FLOOR_US;
}
function applyStepCheck(field, current, sampledUs, stats, labels, suspect, checks) {
const lastValue = numeric(get(stats, "last_value"));
const lastAt = R.parseTimestampUs(get(stats, "last_at"));
if (lastValue === null || lastAt === null || sampledUs === null) return;
if (current > lastValue && lastValue < (has(STEP_REWET_BELOW_PCT, field) ? STEP_REWET_BELOW_PCT[field] : -Infinity)) return;
const [smallest, speed, unit, instrument] = STEP_LIMITS[field];
const minutes = Math.max(totalSeconds(sampledUs - lastAt), 0.0) / 60;
const allowed = Math.max(smallest, speed * minutes);
if (Math.abs(current - lastValue) > allowed) {
addUnique(labels, "sensor_drift_possible");
addUnique(suspect, field);
checks.push(`compare ${field} with ${instrument}: it moved ${formatFixed(current - lastValue, 1, true)} ${unit} in ` +
`${formatFixed(minutes, 1)} minutes, faster than it can change; ${CALIBRATION_HINT}`);
}
}
function applyNoiseCheck(field, sensor, history, labels, suspect, checks) {
const points = seriesPoints(history, sensor, field).slice(-NOISE_WINDOW);
if (points.length < NOISE_MIN_SAMPLES) return;
const stamps = points.map(([, packet]) => R.parseTimestampUs(get(packet, "timestamp")));
const gaps = [];
for (let i = 1; i < stamps.length; i++) {
if (stamps[i - 1] === null || stamps[i] === null) continue;
const g = totalSeconds(stamps[i] - stamps[i - 1]);
if (g > 0) gaps.push(g);
}
gaps.sort(cmp);
if (!gaps.length) return;
let limit;
if (has(NOISE_FIXED_LIMIT, field)) {
let longestInterval;
[limit, longestInterval] = NOISE_FIXED_LIMIT[field];
if (median(gaps) > longestInterval) return; // no real data behind a limit for a slower stream
} else {
limit = Math.max(NOISE_STEP_FLOOR[field],
NOISE_STEP_LIMIT_5MIN[field] * Math.sqrt(median(gaps) / NOISE_REFERENCE_INTERVAL_S));
}
const values = points.map((p) => p[0]);
const steps = values.slice(1).map((b, i) => Math.abs(b - values[i]));
let travelled = 0.0;
for (const step of steps) travelled += step; // left to right, as the Python loop does
if (median(steps) >= limit && Math.abs(values[values.length - 1] - values[0]) <= NOISE_MAX_NET_SHARE * travelled) {
addUnique(labels, "noisy_signal_possible");
addUnique(suspect, field);
checks.push(`inspect ${field}: readings jump between samples; check probe shielding, wiring and averaging`);
}
}
/** apply_persistent_checks: judge the reading against Core's trusted reference of its stream (mutates labels, suspect, checks). */
function applyPersistentChecks(sensor, history, streamBaselineStats, labels, suspect, checks) {
if (!isPlainObject(streamBaselineStats)) return;
const sampled = R.parseTimestampUs(get(sensor, "timestamp"));
let previousPacket = {};
for (let i = history.length - 1; i >= 0; i--) if (isPlainObject(history[i])) { previousPacket = history[i]; break; }
for (const field of Object.keys(streamBaselineStats)) {
const stats = streamBaselineStats[field];
const current = numeric(get(sensor, field));
if (!isPlainObject(stats) || current === null) continue;
const med = numeric(get(stats, "median"));
let threshold = has(BASELINE_DRIFT_THRESHOLDS, field) ? BASELINE_DRIFT_THRESHOLDS[field] : null;
const [low, high] = has(PLAUSIBLE_RANGES, field) ? PLAUSIBLE_RANGES[field] : [-Infinity, Infinity];
let spread = numeric(get(stats, "spread")) || 0.0;
const laggedSpread = numeric(get(stats, "spread_lag"));
if (LAGGED_SPREAD_FIELDS.includes(field) && laggedSpread !== null && laggedSpread >= 0 && isSteadyStream(stats)) spread = laggedSpread;
if (threshold !== null) threshold = Math.max(threshold, DRIFT_SPREAD_FACTOR * spread);
if (med !== null && threshold !== null && low <= current && current <= high && Math.abs(current - med) >= threshold) {
addUnique(labels, "baseline_drift_possible");
addUnique(suspect, field);
checks.push(`verify ${field} with a meter: ${formatG(current)} differs from the trusted level ${formatG(med)}; ${CALIBRATION_HINT}`);
}
let limit = has(PERSISTENT_STUCK_LIMITS_US, field) ? PERSISTENT_STUCK_LIMITS_US[field] : null;
let habit = "";
const adaptive = limit === null && ADAPTIVE_STUCK_FIELDS.includes(field);
if (adaptive) {
limit = adaptiveStuckLimitUs(stats);
habit = `, though this probe normally changes on ${formatPercent0(numeric(get(stats, "change_share")) || 0)} of its readings`;
}
const changedAt = R.parseTimestampUs(get(stats, "last_change_at"));
const previous = numeric(get(previousPacket, field));
if (limit !== null && sampled !== null && changedAt !== null && previous !== null &&
Math.abs(current - previous) <= STUCK_EPSILON && sampled - changedAt > limit &&
(!adaptive || trailingRun(seriesPoints(history, sensor, field).map((p) => p[0])) >= ADAPTIVE_STUCK_MIN_RUN)) {
addUnique(labels, "stuck_value");
addUnique(suspect, field);
checks.push(`inspect ${field}: the same value for ${floorDiv(sampled - changedAt, MINUTE_US)} minutes${habit}; ${CALIBRATION_HINT}`);
}
if (has(STEP_LIMITS, field)) applyStepCheck(field, current, sampled, stats, labels, suspect, checks);
if (has(NOISE_STEP_LIMIT_5MIN, field) || has(NOISE_FIXED_LIMIT, field)) applyNoiseCheck(field, sensor, history, labels, suspect, checks);
}
}
// derive_sensor_quality's inputs are valid (otherwise it returns before any check runs).
function validInputs(farmContext, sensor, expectedFields, history) {
return isPlainObject(farmContext) && isPlainObject(sensor) && Array.isArray(expectedFields) &&
expectedFields.every((f) => typeof f === "string") &&
(history === null || (Array.isArray(history) && history.every((h) => h === null || isPlainObject(h))));
}
/**
* derive_sensor_quality(..., history=..., stream_baseline=...): deriveSensorQuality (rules.js) and then the persistent
* checks. In Python they run inside it, before the default "continue routine monitoring" check and the review
* flag are set; this redoes those two from the final lists.
*/
function derive(farmContext, sensor, expectedFields, nowUs = null, history = null, streamBaselineStats = null) {
const out = R.deriveSensorQuality(farmContext, sensor, expectedFields, nowUs, history);
if (!isPlainObject(streamBaselineStats) || !validInputs(farmContext, sensor, expectedFields, history)) return out;
const labels = [...out.data_quality_labels], suspect = [...out.suspect_fields];
const checks = out.safe_next_checks.length === 1 && out.safe_next_checks[0] === ROUTINE_CHECK ? [] : [...out.safe_next_checks];
applyPersistentChecks(sensor, history || [], streamBaselineStats, labels, suspect, checks);
if (!checks.length) checks.push(ROUTINE_CHECK);
return {
data_quality_labels: labels, missing_fields: out.missing_fields, suspect_fields: suspect, safe_next_checks: checks,
human_review_required: Boolean(labels.length || out.missing_fields.length || suspect.length),
rationale: labels.length ? "Sensor packet needs verification before downstream risk or action reasoning."
: "Critical sensor readings are present and inside plausible ranges.",
};
}
/**
* The trusted reference of one stream that is checked reading by reading, as Core builds it: the stream's earlier
* readings (at most `scanLimit` of them, newest by arrival), those sampled before the current one, in sample-time
* order. `add` each reading after it was checked, with the fields the pipeline vetoed.
*/
class StreamReference {
constructor(scanLimit = DEFAULT_SCAN_LIMIT) {
this.scanLimit = scanLimit;
this.sorted = []; // entries by sample time, equal times in arrival order
this.arrival = []; // the same entries by arrival
}
baseline(stampUs, calibrations = []) {
const sorted = this.sorted;
let lo = 0, hi = sorted.length; // entries sampled before stampUs
while (lo < hi) { const mid = (lo + hi) >> 1; if (sorted[mid].timestamp < stampUs) lo = mid + 1; else hi = mid; }
return streamBaseline(lo === sorted.length ? sorted : sorted.slice(0, lo), calibrations, stampUs);
}
add(packet, stampUs, vetoedFields) {
const entry = { timestamp: stampUs, vetoed_fields: vetoedFields };
for (const field of BASELINE_FIELDS) entry[field] = numeric(get(packet, field));
const sorted = this.sorted;
let lo = 0, hi = sorted.length; // after the entries sampled at the same time
while (lo < hi) { const mid = (lo + hi) >> 1; if (sorted[mid].timestamp <= stampUs) lo = mid + 1; else hi = mid; }
sorted.splice(lo, 0, entry);
this.arrival.push(entry);
if (this.arrival.length > this.scanLimit) {
const oldest = this.arrival.shift();
sorted.splice(sorted.indexOf(oldest), 1);
}
}
}
return { derive, applyPersistentChecks, streamBaseline, StreamReference, parseTime, isoformatUs,
pyRound, formatG, formatFixed, formatPercent0, totalSeconds, timedeltaUs, adaptiveStuckLimitUs,
BASELINE_FIELDS, DEFAULT_SCAN_LIMIT };
});
|