axis-numeric / axis_numeric /reliability.py
serety95's picture
Update numeric space
f1ba785 verified
Raw History Blame Contribute Delete
2.21 kB
"""@module numeric-inference/reliability — Hoeffding lower bound on P(correct).
`calibrated_confidence` is still probability × 100. Accept uses this bound,
fitted from a calibration count, and stays below 0.95 until that count exists.
"""
from __future__ import annotations
import math
from .contracts import SemanticClass
MIN_RELIABILITY = 0.95
DELTA = 0.05
CHECKSUM_CLASSES: frozenset[SemanticClass] = frozenset({
"nid", "pan", "tax_id", "registration_number",
})
AGREEMENT_CLASSES: frozenset[SemanticClass] = frozenset({
"date", "amount", "percentage", "generic_numeric", "account_number",
})
def accuracy_lower_bound(correct: int, n: int, delta: float = DELTA) -> float:
"""1 − Hoeffding upper bound on the error rate. 0 when the count is unusable."""
if n < 1 or correct < 0 or correct > n or not 0 < delta < 1:
return 0.0
empirical_error = (n - correct) / n
slack = math.sqrt(math.log(1 / delta) / (2 * n))
return max(0.0, 1 - min(1.0, empirical_error + slack))
class ReliabilityTable:
"""Per-class (correct, n) counts. Missing classes report a bound of 0."""
def __init__(self, counts: dict[str, tuple[int, int]] | None = None):
self._counts = dict(counts or {})
@staticmethod
def empty() -> ReliabilityTable:
return ReliabilityTable()
@staticmethod
def from_artifact(raw: object) -> ReliabilityTable:
if not isinstance(raw, dict):
return ReliabilityTable.empty()
counts: dict[str, tuple[int, int]] = {}
for key, value in raw.items():
if not isinstance(key, str) or not isinstance(value, dict):
continue
correct, n = value.get("correct"), value.get("n")
if isinstance(correct, int) and not isinstance(correct, bool) and isinstance(n, int) and not isinstance(n, bool):
counts[key] = (correct, n)
return ReliabilityTable(counts)
def lower_bound(self, semantic_class: str) -> float:
pair = self._counts.get(semantic_class)
if pair is None:
return 0.0
return accuracy_lower_bound(pair[0], pair[1])