Spaces:
Sleeping
Sleeping
File size: 3,809 Bytes
b2e9550 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 | """Deterministic numeric fact extraction with exact source spans.
This module never validates or corrects a value. Every extracted number starts
with status=pending and must pass the dedicated numeric validator later.
"""
from __future__ import annotations
import re
from dataclasses import dataclass
@dataclass(frozen=True)
class NumericMatch:
kind: str
raw_text: str
value: float | str | None
unit: str | None
start: int
end: int
_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
("confidence_interval", re.compile(r"(?:95\s*%?\s*(?:CI|IC))\s*[:=\[(]?\s*(-?\d+(?:\.\d+)?)\s*(?:[-–,]|to)\s*(-?\d+(?:\.\d+)?)", re.I)),
("p_value", re.compile(r"\bp\s*([<>=]=?)\s*(0?\.\d+|\d+(?:\.\d+)?(?:e-?\d+)?)", re.I)),
("effect_estimate", re.compile(r"\b(OR|RR|HR|SMD|MD)\s*[=:]?\s*(-?\d+(?:\.\d+)?)", re.I)),
("dose", re.compile(r"\b(\d+(?:\.\d+)?)\s*(mg|g|µg|ug|mcg|IU|UI|mL|ml|L|mmol|µmol|umol|nmol)(?:\s*/\s*(?:kg|day|d|jour))?\b", re.I)),
("percentage", re.compile(r"\b(\d+(?:\.\d+)?)\s*%(?!\s*(?:CI|IC))", re.I)),
("sample_size", re.compile(r"\bn\s*=\s*(\d[\d ,]*)", re.I)),
("sample_size", re.compile(r"\b(\d[\d ,]*)\s+(?:(?:[A-Za-zÀ-ÿ-]+)\s+){0,2}(?:participants?|patients?|subjects?|students?|volunteers?|adults?|children|individuals?|women|men)\b", re.I)),
("duration", re.compile(r"\b(\d+(?:\.\d+)?)\s*(hours?|days?|weeks?|months?|years?)\b", re.I)),
("age", re.compile(r"\b(?:aged?|age)\s*(\d+(?:\.\d+)?)\s*(?:years?|yrs?)\b", re.I)),
("frequency", re.compile(r"\b(once|twice|three times|four times)\s+(?:a|per)\s+(day|week)\b", re.I)),
]
def _number(value: str) -> float | str:
cleaned = value.replace(",", "").replace(" ", "")
try:
return float(cleaned)
except ValueError:
return value
def extract_numeric_matches(text: str) -> list[NumericMatch]:
matches: list[NumericMatch] = []
occupied: list[tuple[int, int]] = []
for kind, pattern in _PATTERNS:
for match in pattern.finditer(text):
span = match.span()
if any(span[0] < end and span[1] > start for start, end in occupied):
continue
groups = match.groups()
raw = match.group(0)
value: float | str | None = None
unit: str | None = None
if kind == "confidence_interval":
value = f"{groups[0]} to {groups[1]}"
unit = "95% CI"
elif kind == "p_value":
value = _number(groups[1])
unit = f"p{groups[0]}"
elif kind == "effect_estimate":
value = _number(groups[1])
unit = groups[0].upper()
elif kind == "dose":
value = _number(groups[0])
unit = groups[1]
elif kind in {"percentage", "sample_size", "duration", "age"}:
value = _number(groups[0])
unit = groups[1] if len(groups) > 1 and groups[1] else {
"percentage": "%",
"sample_size": "participants",
"age": "years",
}.get(kind)
elif kind == "frequency":
value = groups[0]
unit = groups[1]
matches.append(NumericMatch(kind, raw, value, unit, span[0], span[1]))
occupied.append(span)
return sorted(matches, key=lambda item: (item.start, item.end, item.kind))
def to_numeric_facts(text: str) -> list[dict]:
return [
{
"kind": item.kind,
"raw_text": item.raw_text,
"value": item.value,
"unit": item.unit,
"context": text,
"status": "pending",
"correction": None,
}
for item in extract_numeric_matches(text)
]
|