Wire all brain honesty signals into health rollup

#4
Files changed (2) hide show
  1. szl_brainhealth.py +99 -25
  2. test/test_brainhealth_wiring.py +44 -0
szl_brainhealth.py CHANGED
@@ -127,45 +127,45 @@ COMPONENTS: list[dict] = [
127
  "key": "grounding",
128
  "title": "grounding confidence",
129
  "module": "szl_brainground",
130
- "funcs": ("compute_grounding", "grounding_confidence", "brainground",
131
- "compute", "assess", "evaluate", "for_query", "health"),
132
- "value_keys": ("grounding_confidence", "confidence", "grounding", "score", "value"),
133
  "adverse": ("ungrounded", "no-grounding", "no grounding"),
134
  },
135
  {
136
  "key": "freshness",
137
  "title": "memory freshness",
138
  "module": "szl_brainmemory",
139
- "funcs": ("compute_freshness", "freshness", "brainmemory",
140
- "compute", "assess", "evaluate", "for_query", "health"),
141
- "value_keys": ("freshness", "freshness_score", "recency", "score", "value"),
142
  "adverse": ("expired", "outdated"),
143
  },
144
  {
145
  "key": "provenance",
146
  "title": "source-lineage coverage",
147
  "module": "szl_brainprovenance",
148
- "funcs": ("compute_provenance", "provenance_coverage", "brainprovenance",
149
- "compute", "assess", "evaluate", "for_query", "health"),
150
- "value_keys": ("provenance_coverage", "coverage", "lineage_coverage", "score", "value"),
151
  "adverse": ("unprovenanced", "no-lineage", "no lineage", "unsourced"),
152
  },
153
  {
154
  "key": "contradiction",
155
  "title": "conflict flag",
156
  "module": "szl_braincontradict",
157
- "funcs": ("compute_contradiction", "contradiction", "braincontradict",
158
- "compute", "assess", "evaluate", "for_query", "health"),
159
- "value_keys": ("contradiction_score", "conflict_score", "score", "value"),
160
  "adverse": ("conflicted", "contradicted"),
161
  },
162
  {
163
  "key": "uncertainty",
164
  "title": "uncertainty / abstain",
165
  "module": "szl_brainuncertainty",
166
- "funcs": ("compute_uncertainty", "uncertainty", "brainuncertainty",
167
- "compute", "assess", "evaluate", "for_query", "health"),
168
- "value_keys": ("uncertainty", "semantic_entropy", "entropy", "score", "value"),
169
  "adverse": ("uncertain", "high-uncertainty", "high uncertainty"),
170
  },
171
  ]
@@ -196,10 +196,17 @@ def _resolve_callable(spec: dict) -> Callable | None:
196
  return None
197
 
198
 
199
- def _invoke(fn: Callable, q: str, k: int):
200
  """Call the sibling with the most specific signature it accepts, degrading through
201
  (q, k) -> (q) -> (). A TypeError only from arity is retried; anything else propagates so
202
  the caller can mark the component UNAVAILABLE honestly."""
 
 
 
 
 
 
 
203
  for args in ((q, k), (q,), ()):
204
  try:
205
  return fn(*args)
@@ -211,6 +218,65 @@ def _invoke(fn: Callable, q: str, k: int):
211
  return fn(q)
212
 
213
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
214
  def _iter_string_values(obj):
215
  """Yield every string VALUE reachable in a nested dict/list — never a key. Adverse-token
216
  scanning must inspect what a component SAYS (its values), never its field NAMES: a field
@@ -258,9 +324,10 @@ _ADVERSE_FLAGS = ("abstain", "abstained", "insufficient", "conflict", "conflict_
258
 
259
  def _detect_adverse(payload: dict, extra_tokens) -> str | None:
260
  """Return the adverse reason if the component declares one, else None. Reads the
261
- component's OWN fields VERBATIM: (1) explicit boolean adverse flags set True; (2) an
262
- adverse token in its verdict/status/signal/label string; (3) an adverse token anywhere in
263
- its serialized payload. Conservative a negated mention ('no conflict') is NOT adverse."""
 
264
  # (1) explicit boolean flags.
265
  for flag in _ADVERSE_FLAGS:
266
  v = payload.get(flag)
@@ -271,9 +338,11 @@ def _detect_adverse(payload: dict, extra_tokens) -> str | None:
271
  if isinstance(dv, bool) and dv is True:
272
  return flag
273
  tokens = tuple(_SHARED_ADVERSE) + tuple(extra_tokens or ())
274
- # (2) declared string VALUES only (never field names) verdict/status/signal/label live
275
- # here, as does any nested honesty string. A negated mention ('no conflict') is NOT adverse.
276
- for s in _iter_string_values(payload):
 
 
277
  low = s.lower()
278
  for tok in tokens:
279
  if tok in low and not _is_negated(low, tok):
@@ -312,7 +381,7 @@ def _detect_positive(payload: dict, value: float | None) -> bool:
312
  return False
313
 
314
 
315
- def _gather_component(spec: dict, q: str, k: int) -> dict:
316
  """Gather ONE component honestly. Never raises: any failure => UNAVAILABLE with a reason."""
317
  key = spec["key"]
318
  base = {
@@ -336,7 +405,7 @@ def _gather_component(spec: dict, q: str, k: int) -> dict:
336
  base["note"] = ("sibling not importable (guarded ImportError) or exposes no "
337
  "compute entrypoint; component honestly UNAVAILABLE")
338
  return base
339
- payload = _invoke(fn, q, k)
340
  except Exception as exc: # a live failure degrades THIS component honestly, never the roll-up
341
  base["note"] = f"component compute failed, reported honestly: {str(exc)[:160]}"
342
  return base
@@ -345,6 +414,7 @@ def _gather_component(spec: dict, q: str, k: int) -> dict:
345
  base["note"] = "component returned no manifest dict; honestly UNAVAILABLE"
346
  return base
347
 
 
348
  label = _read_label(payload)
349
  value = _read_value(payload, spec.get("value_keys", ()))
350
  adverse = _detect_adverse(payload, spec.get("adverse"))
@@ -359,6 +429,8 @@ def _gather_component(spec: dict, q: str, k: int) -> dict:
359
  "available": True,
360
  "label": label if label is not None else MODELED,
361
  "value": round(value, 6) if value is not None else None,
 
 
362
  "signal": signal,
363
  "adverse_reason": adverse,
364
  "note": ("component available; label read VERBATIM, never upgraded"
@@ -413,7 +485,7 @@ def _modeled_trust(components: list[dict]) -> float | None:
413
  def build_rollup(q: str = "", k: int = 12, ns: str = "a11oy") -> dict:
414
  """Gather every brain-honesty component (available ones read VERBATIM, missing ones
415
  UNAVAILABLE) and roll the AVAILABLE ones into ONE honest brain-trust verdict."""
416
- components = [_gather_component(spec, q, k) for spec in COMPONENTS]
417
  verdict, reason = _decide_verdict(components)
418
 
419
  available = [c for c in components if c["available"]]
@@ -688,6 +760,8 @@ if __name__ == "__main__":
688
  "verdict": "grounded"}
689
  _PROBE_OVERRIDES["freshness"] = lambda q, k: {"label": "SAMPLE", "freshness": 0.7,
690
  "verdict": "fresh"}
 
 
691
  r2 = build_rollup("q", k=4)
692
  # exactly two available, both OK, but 3 UNAVAILABLE -> DEGRADED (never TRUSTWORTHY w/ gaps).
693
  assert r2["verdict"] == DEGRADED, r2["verdict"]
 
127
  "key": "grounding",
128
  "title": "grounding confidence",
129
  "module": "szl_brainground",
130
+ "funcs": ("evaluate",),
131
+ "call_style": "q_k_ns",
132
+ "value_keys": ("trust_value", "grounding_confidence", "confidence", "grounding", "score", "value"),
133
  "adverse": ("ungrounded", "no-grounding", "no grounding"),
134
  },
135
  {
136
  "key": "freshness",
137
  "title": "memory freshness",
138
  "module": "szl_brainmemory",
139
+ "funcs": ("build_aggregate",),
140
+ "call_style": "ns",
141
+ "value_keys": ("trust_value", "freshness", "freshness_score", "recency", "score", "value"),
142
  "adverse": ("expired", "outdated"),
143
  },
144
  {
145
  "key": "provenance",
146
  "title": "source-lineage coverage",
147
  "module": "szl_brainprovenance",
148
+ "funcs": ("build_provenance",),
149
+ "call_style": "ns_q_k",
150
+ "value_keys": ("trust_value", "provenance_coverage", "lineage_coverage", "score", "value"),
151
  "adverse": ("unprovenanced", "no-lineage", "no lineage", "unsourced"),
152
  },
153
  {
154
  "key": "contradiction",
155
  "title": "conflict flag",
156
  "module": "szl_braincontradict",
157
+ "funcs": ("run_detection",),
158
+ "call_style": "q_k_ns",
159
+ "value_keys": ("trust_value", "contradiction_score", "conflict_score", "score", "value"),
160
  "adverse": ("conflicted", "contradicted"),
161
  },
162
  {
163
  "key": "uncertainty",
164
  "title": "uncertainty / abstain",
165
  "module": "szl_brainuncertainty",
166
+ "funcs": ("handle_uncertainty",),
167
+ "call_style": "ns_q_k",
168
+ "value_keys": ("trust_value", "semantic_entropy", "entropy", "score", "value"),
169
  "adverse": ("uncertain", "high-uncertainty", "high uncertainty"),
170
  },
171
  ]
 
196
  return None
197
 
198
 
199
+ def _invoke(fn: Callable, q: str, k: int, style: str = "generic", ns: str = "a11oy"):
200
  """Call the sibling with the most specific signature it accepts, degrading through
201
  (q, k) -> (q) -> (). A TypeError only from arity is retried; anything else propagates so
202
  the caller can mark the component UNAVAILABLE honestly."""
203
+ explicit = {
204
+ "q_k_ns": (q, k, ns),
205
+ "ns_q_k": (ns, q, k),
206
+ "ns": (ns,),
207
+ }
208
+ if style in explicit:
209
+ return fn(*explicit[style])
210
  for args in ((q, k), (q,), ()):
211
  try:
212
  return fn(*args)
 
218
  return fn(q)
219
 
220
 
221
+ def _normalize_component_payload(key: str, payload: dict) -> dict:
222
+ """Add a transparent trust-aligned value without changing the sibling's own label/verdict.
223
+
224
+ The old rollup averaged heterogeneous directions (for example, high uncertainty increased
225
+ the displayed modeled trust). Every `trust_value` below uses the same direction: 1 is the
226
+ favorable end of that component's own signal and 0 is the adverse end. Source fields remain
227
+ present and source_verdict is copied verbatim for audit/replay.
228
+ """
229
+ out = dict(payload)
230
+ out["source_verdict"] = payload.get("verdict")
231
+
232
+ def clip(value):
233
+ try:
234
+ return max(0.0, min(1.0, float(value)))
235
+ except (TypeError, ValueError):
236
+ return None
237
+
238
+ if key == "grounding":
239
+ value = clip(payload.get("grounding_confidence"))
240
+ if value is not None:
241
+ out["trust_value"] = value
242
+ if payload.get("should_abstain") is True:
243
+ out["abstain"] = True
244
+ elif key == "freshness":
245
+ counts = payload.get("verdict_counts") if isinstance(payload.get("verdict_counts"), dict) else {}
246
+ total = payload.get("node_count")
247
+ stale = counts.get("STALE")
248
+ if isinstance(total, (int, float)) and total > 0 and isinstance(stale, (int, float)):
249
+ stale_share = clip(float(stale) / float(total))
250
+ out["stale_share"] = stale_share
251
+ out["trust_value"] = round(1.0 - stale_share, 6)
252
+ out["stale_dominant"] = stale_share >= 0.5
253
+ out["verdict"] = "STALE-DOMINANT" if stale_share >= 0.5 else "FRESHNESS-MIXED"
254
+ elif key == "provenance":
255
+ coverage = payload.get("coverage") if isinstance(payload.get("coverage"), dict) else {}
256
+ value = clip(coverage.get("fraction_traceable_to_source"))
257
+ if value is not None:
258
+ out["provenance_coverage"] = value
259
+ out["trust_value"] = value
260
+ elif key == "contradiction":
261
+ summary = payload.get("summary") if isinstance(payload.get("summary"), dict) else {}
262
+ pairs = summary.get("pairs_examined")
263
+ flagged = summary.get("flagged_count")
264
+ if isinstance(pairs, (int, float)) and pairs > 0 and isinstance(flagged, (int, float)):
265
+ risk = clip(float(flagged) / float(pairs))
266
+ out["conflict_score"] = risk
267
+ out["trust_value"] = round(1.0 - risk, 6)
268
+ if str(payload.get("verdict", "")).upper() == "CONFLICT-FLAGGED":
269
+ out["contradiction_detected"] = True
270
+ elif key == "uncertainty":
271
+ uncertainty = clip(payload.get("uncertainty"))
272
+ if uncertainty is not None:
273
+ out["raw_uncertainty"] = uncertainty
274
+ out["trust_value"] = round(1.0 - uncertainty, 6)
275
+ if payload.get("abstain_recommended") is True:
276
+ out["abstain"] = True
277
+ return out
278
+
279
+
280
  def _iter_string_values(obj):
281
  """Yield every string VALUE reachable in a nested dict/list — never a key. Adverse-token
282
  scanning must inspect what a component SAYS (its values), never its field NAMES: a field
 
324
 
325
  def _detect_adverse(payload: dict, extra_tokens) -> str | None:
326
  """Return the adverse reason if the component declares one, else None. Reads the
327
+ component's OWN decision fields VERBATIM: (1) explicit boolean adverse flags set True;
328
+ (2) an adverse token in verdict/status/signal/state/answer_label. Descriptive notes,
329
+ formulas and doctrine text are intentionally excluded: a sentence explaining when to
330
+ abstain is not evidence that this request actually abstained."""
331
  # (1) explicit boolean flags.
332
  for flag in _ADVERSE_FLAGS:
333
  v = payload.get(flag)
 
338
  if isinstance(dv, bool) and dv is True:
339
  return flag
340
  tokens = tuple(_SHARED_ADVERSE) + tuple(extra_tokens or ())
341
+ # (2) declared decision strings only (never field names or descriptive prose).
342
+ for field in ("verdict", "status", "signal", "state", "answer_label"):
343
+ s = payload.get(field)
344
+ if not isinstance(s, str):
345
+ continue
346
  low = s.lower()
347
  for tok in tokens:
348
  if tok in low and not _is_negated(low, tok):
 
381
  return False
382
 
383
 
384
+ def _gather_component(spec: dict, q: str, k: int, ns: str = "a11oy") -> dict:
385
  """Gather ONE component honestly. Never raises: any failure => UNAVAILABLE with a reason."""
386
  key = spec["key"]
387
  base = {
 
405
  base["note"] = ("sibling not importable (guarded ImportError) or exposes no "
406
  "compute entrypoint; component honestly UNAVAILABLE")
407
  return base
408
+ payload = _invoke(fn, q, k, style=spec.get("call_style", "generic"), ns=ns)
409
  except Exception as exc: # a live failure degrades THIS component honestly, never the roll-up
410
  base["note"] = f"component compute failed, reported honestly: {str(exc)[:160]}"
411
  return base
 
414
  base["note"] = "component returned no manifest dict; honestly UNAVAILABLE"
415
  return base
416
 
417
+ payload = _normalize_component_payload(key, payload)
418
  label = _read_label(payload)
419
  value = _read_value(payload, spec.get("value_keys", ()))
420
  adverse = _detect_adverse(payload, spec.get("adverse"))
 
429
  "available": True,
430
  "label": label if label is not None else MODELED,
431
  "value": round(value, 6) if value is not None else None,
432
+ "value_semantics": "trust-aligned [0,1]; source direction normalized transparently",
433
+ "source_verdict": payload.get("source_verdict"),
434
  "signal": signal,
435
  "adverse_reason": adverse,
436
  "note": ("component available; label read VERBATIM, never upgraded"
 
485
  def build_rollup(q: str = "", k: int = 12, ns: str = "a11oy") -> dict:
486
  """Gather every brain-honesty component (available ones read VERBATIM, missing ones
487
  UNAVAILABLE) and roll the AVAILABLE ones into ONE honest brain-trust verdict."""
488
+ components = [_gather_component(spec, q, k, ns=ns) for spec in COMPONENTS]
489
  verdict, reason = _decide_verdict(components)
490
 
491
  available = [c for c in components if c["available"]]
 
760
  "verdict": "grounded"}
761
  _PROBE_OVERRIDES["freshness"] = lambda q, k: {"label": "SAMPLE", "freshness": 0.7,
762
  "verdict": "fresh"}
763
+ for _missing_key in ("provenance", "contradiction", "uncertainty"):
764
+ _PROBE_OVERRIDES[_missing_key] = lambda q, k: None
765
  r2 = build_rollup("q", k=4)
766
  # exactly two available, both OK, but 3 UNAVAILABLE -> DEGRADED (never TRUSTWORTHY w/ gaps).
767
  assert r2["verdict"] == DEGRADED, r2["verdict"]
test/test_brainhealth_wiring.py ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import unittest
4
+
5
+ import szl_brainhealth as health
6
+
7
+
8
+ class BrainHealthWiringTests(unittest.TestCase):
9
+ def test_real_sibling_entrypoints_and_call_styles_are_explicit(self) -> None:
10
+ specs = {row["key"]: row for row in health.COMPONENTS}
11
+ self.assertEqual(specs["freshness"]["funcs"], ("build_aggregate",))
12
+ self.assertEqual(specs["provenance"]["call_style"], "ns_q_k")
13
+ self.assertEqual(specs["contradiction"]["funcs"], ("run_detection",))
14
+ self.assertEqual(specs["uncertainty"]["funcs"], ("handle_uncertainty",))
15
+
16
+ def test_uncertainty_is_inverted_to_trust_direction(self) -> None:
17
+ payload = health._normalize_component_payload(
18
+ "uncertainty",
19
+ {"label": "MODELED", "verdict": "HIGHLY-UNCERTAIN", "uncertainty": 0.91, "abstain_recommended": True},
20
+ )
21
+ self.assertEqual(payload["trust_value"], 0.09)
22
+ self.assertTrue(payload["abstain"])
23
+
24
+ def test_descriptive_prose_does_not_create_a_false_adverse_signal(self) -> None:
25
+ payload = {
26
+ "label": "MODELED",
27
+ "verdict": "WEAK-GROUNDING",
28
+ "should_abstain": False,
29
+ "note": "the system abstains only when evidence is insufficient",
30
+ }
31
+ self.assertIsNone(health._detect_adverse(payload, ("ungrounded",)))
32
+
33
+ def test_freshness_uses_real_node_distribution(self) -> None:
34
+ payload = health._normalize_component_payload(
35
+ "freshness",
36
+ {"label": "MEASURED", "node_count": 100, "verdict_counts": {"FRESH": 5, "AGING": 15, "STALE": 80}},
37
+ )
38
+ self.assertEqual(payload["trust_value"], 0.2)
39
+ self.assertTrue(payload["stale_dominant"])
40
+ self.assertEqual(payload["verdict"], "STALE-DOMINANT")
41
+
42
+
43
+ if __name__ == "__main__":
44
+ unittest.main()