File size: 2,206 Bytes
f1ba785
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
"""@module numeric-inference/reliability — Hoeffding lower bound on P(correct).



`calibrated_confidence` is still probability × 100. Accept uses this bound,

fitted from a calibration count, and stays below 0.95 until that count exists.

"""
from __future__ import annotations

import math

from .contracts import SemanticClass

MIN_RELIABILITY = 0.95
DELTA = 0.05

CHECKSUM_CLASSES: frozenset[SemanticClass] = frozenset({
    "nid", "pan", "tax_id", "registration_number",
})
AGREEMENT_CLASSES: frozenset[SemanticClass] = frozenset({
    "date", "amount", "percentage", "generic_numeric", "account_number",
})


def accuracy_lower_bound(correct: int, n: int, delta: float = DELTA) -> float:
    """1 − Hoeffding upper bound on the error rate. 0 when the count is unusable."""
    if n < 1 or correct < 0 or correct > n or not 0 < delta < 1:
        return 0.0
    empirical_error = (n - correct) / n
    slack = math.sqrt(math.log(1 / delta) / (2 * n))
    return max(0.0, 1 - min(1.0, empirical_error + slack))


class ReliabilityTable:
    """Per-class (correct, n) counts. Missing classes report a bound of 0."""

    def __init__(self, counts: dict[str, tuple[int, int]] | None = None):
        self._counts = dict(counts or {})

    @staticmethod
    def empty() -> ReliabilityTable:
        return ReliabilityTable()

    @staticmethod
    def from_artifact(raw: object) -> ReliabilityTable:
        if not isinstance(raw, dict):
            return ReliabilityTable.empty()
        counts: dict[str, tuple[int, int]] = {}
        for key, value in raw.items():
            if not isinstance(key, str) or not isinstance(value, dict):
                continue
            correct, n = value.get("correct"), value.get("n")
            if isinstance(correct, int) and not isinstance(correct, bool) and isinstance(n, int) and not isinstance(n, bool):
                counts[key] = (correct, n)
        return ReliabilityTable(counts)

    def lower_bound(self, semantic_class: str) -> float:
        pair = self._counts.get(semantic_class)
        if pair is None:
            return 0.0
        return accuracy_lower_bound(pair[0], pair[1])