from __future__ import annotations import datetime import hashlib import itertools import math import re from typing import Any, cast READ_TYPES = ("text", "number", "date", "time", "datetime", "duration") FIELD_TYPES = READ_TYPES + ("list", "estimate") EVIDENCE_CONSTRUCTION = "evidence-v1" ANSWER_ROUTES = ("copy", "normalize", "derive", "knowledge", "abstain") REQUEST_TYPES = FIELD_TYPES + ("compute", "forecast") LIST_ITEM_TYPES = ("text", "number", "date") DURATION_UNITS = ("seconds", "minutes", "hours", "days", "weeks", "months", "years") MAX_LIST_ITEMS = 32 MAX_DECIMALS = 6 MAX_FORECAST_HORIZON = 64 FORECAST_INTERVALS = ( "step", "minute", "hour", "day", "week", "month", "quarter", "year", ) DEFAULT_NULL_THRESHOLD = 0.5 def list_item_name(parent: str, index: int) -> str: suffix = f".item{index}" if len(parent) + len(suffix) <= 64: return parent + suffix digest = hashlib.sha256(parent.encode()).hexdigest()[:8] return parent[: 64 - len(suffix) - 9] + "_" + digest + suffix def list_query_fields(fields: dict[str, Any], parent: str) -> list[str]: spec = fields[parent] item_type = ( spec["items"].get("type") if isinstance(spec["items"], dict) else spec["items"] ) queries = {} for name, child in fields.items(): (_, separator, ordinal) = name.rpartition(".item") if not separator or not ordinal.isascii() or (not ordinal.isdigit()): continue index = int(ordinal) if ( index > 0 and name == list_item_name(parent, index) and (child.get("type") == item_type) ): queries[index] = name if sorted(queries) != list(range(1, len(queries) + 1)): raise ValueError(f"noncontiguous scalar list queries for {parent!r}") return [queries[index] for index in range(1, len(queries) + 1)] def assemble_list_predictions( record: dict[str, Any], predictions: dict[str, dict[str, Any]] ) -> dict[str, dict[str, Any]]: if (record.get("_meta") or {}).get("construction") != EVIDENCE_CONSTRUCTION: return predictions fields = record["fields"] merged = dict(predictions) for parent, spec in fields.items(): if spec.get("type") != "list": continue queries = list_query_fields(fields, parent) if not queries: continue (values, confidences, probabilities) = ([], [], []) for name in queries: item = predictions.get(name) if item is None or "value" not in item: raise ValueError(f"missing scalar list prediction for {name!r}") if item["value"] is None: probabilities.append(item.get("p_null")) break values.append(item["value"]) confidences.append(item.get("confidence")) probabilities.append( (1.0 - item["p_null"]) * item["confidence"] if "p_null" in item and "confidence" in item else None ) assembled: dict[str, Any] = {"value": values} if all((value is not None for value in confidences)): assembled["confidences"] = confidences if probabilities and all((value is not None for value in probabilities)): assembled["confidence"] = math.prod(cast(list[float], probabilities)) merged[parent] = assembled return merged SYSTEM_PREAMBLE = "Fill each field from the input with a value of its type. Text copies a span of the input. Numbers, dates, times, datetimes, and durations are read from the input or worked out from it. A list holds every matching value in input order. A field is not stated when the input does not determine it. Estimates predict a value inside the declared range." EVIDENCE_SYSTEM_PREAMBLE = "Fill each field with a value of its type, following its instructions and the question's scope. For document-specific extraction, use the supplied input and return not stated when the requested information is missing. Preserve exact text, names and numbers unless normalization or a derived answer is requested. General factual questions may use reliable prior knowledge unless the instructions restrict answers to the supplied evidence; return not stated when the answer cannot be determined reliably. A list holds every matching item in the requested order, including repeated items. Estimates and forecasts predict distributions over the declared range and supported below-range or above-range outcomes. Follow the requested units, time intervals and horizons." FIELD_NAME = re.compile("^[A-Za-z_][A-Za-z0-9_.\\-]{0,63}$") def format_bound(value: float) -> str: if float(value).is_integer() and abs(value) < 1000000000000000.0: return str(int(value)) return f"{value:.6g}" def field_line( name: str, spec: dict[str, Any], preserve_instructions: bool = False ) -> str: kind = spec["type"] unit = str(spec.get("unit") or "").strip() if kind == "estimate" and is_date_estimate(spec): head = f"estimate date, {spec['min']} to {spec['max']}" granularity = date_granularity(spec) if granularity != "day": head += f", by {granularity}" elif spec.get("step") and float(spec["step"]) != 1: head += f", step {format_bound(float(spec['step']))} days" elif kind == "estimate": head = f"estimate{(' ' + unit if unit else '')}, {format_bound(spec['min'])} to {format_bound(spec['max'])}, {spec.get('scale', 'linear')} scale" if spec.get("step"): head += f", step {format_bound(float(spec['step']))}" elif kind == "list": head = f"list of {spec['items']}{number_options(spec)}{(' (' + unit + ')' if unit else '')}" elif kind == "duration": head = f"duration in {duration_unit(spec)}{number_options(spec)}" else: head = f"{kind}{(number_options(spec) if kind == 'number' else '')}{(' (' + unit + ')' if unit else '')}" instructions = str(spec.get("instructions") or "") if not preserve_instructions: instructions = " ".join(instructions.split()) return f"{name}: {head}." + (f" {instructions}" if instructions else "") def number_options(spec: dict[str, Any]) -> str: if spec.get("integer"): return ", integer" if spec.get("decimals") is not None: return f", {int(spec['decimals'])} decimals" return "" def duration_unit(spec: dict[str, Any]) -> str: return str(spec.get("unit") or "seconds") def allowed_decimals(spec: dict[str, Any]) -> int | None: if spec.get("integer"): return 0 return None if spec.get("decimals") is None else int(spec["decimals"]) def parse_time(value: Any) -> tuple[int, int, int]: if not isinstance(value, str): raise ValueError(f"time must be a string, got {value!r}") parsed = datetime.time.fromisoformat(value) if parsed.microsecond or parsed.tzinfo is not None: raise ValueError(f"time {value!r} must be HH:MM or HH:MM:SS") return (parsed.hour, parsed.minute, parsed.second) def format_time(hour: int, minute: int, second: int) -> str: return f"{hour:02d}:{minute:02d}:{second:02d}" def parse_datetime(value: Any) -> tuple[str, tuple[int, int, int]]: if not isinstance(value, str) or "T" not in value: raise ValueError(f"datetime must be YYYY-MM-DDTHH:MM[:SS], got {value!r}") (day, clock) = value.split("T", 1) return (datetime.date.fromisoformat(day).isoformat(), parse_time(clock)) def forecast_instructions(instructions: str, step: int, interval: str) -> str: what = " ".join(str(instructions or "").split()).rstrip(".") or "the series" unit = "step" if interval == "step" else interval return f"Forecast {what} {step} {unit}{('s' if step > 1 else '')} after the last observation." def expand_forecasts( fields: dict[str, Any], ) -> tuple[dict[str, Any], dict[str, list[str]]]: expanded: dict[str, Any] = {} groups: dict[str, list[str]] = {} for name, spec in fields.items(): if not isinstance(spec, dict) or spec.get("type") != "forecast": expanded[name] = spec continue horizon = spec.get("horizon") if ( isinstance(horizon, bool) or not isinstance(horizon, int) or (not 1 <= horizon <= MAX_FORECAST_HORIZON) ): raise ValueError( f"forecast {name} needs an integer horizon from 1 to {MAX_FORECAST_HORIZON}" ) interval = spec.get("interval", "step") if interval not in FORECAST_INTERVALS: raise ValueError(f"forecast {name} has an unknown interval {interval!r}") base = { key: value for (key, value) in spec.items() if key not in ("type", "horizon", "interval", "instructions") } groups[name] = [] for step in range(1, horizon + 1): step_name = f"{name}.{step}" if step_name in fields or step_name in expanded: raise ValueError( f"forecast {name} step {step_name} collides with another field" ) expanded[step_name] = { "type": "estimate", **base, "instructions": forecast_instructions( spec.get("instructions", ""), step, interval ), } groups[name].append(step_name) return (expanded, groups) def to_unit(value: float, low: float, high: float, scale: str) -> float: return min(1.0, max(0.0, raw_unit(value, low, high, scale))) def raw_unit(value: float, low: float, high: float, scale: str) -> float: if scale == "log": if value <= 0: return -1.0 return (math.log(value) - math.log(low)) / (math.log(high) - math.log(low)) return (value - low) / (high - low) def from_unit(position: float, low: float, high: float, scale: str) -> float: position = min(1.0, max(0.0, position)) if scale == "log": value = math.exp(math.log(low) + position * (math.log(high) - math.log(low))) else: value = low + position * (high - low) return min(high, max(low, value)) def resolution_in_unit(spec: dict[str, Any], resolution: float | None) -> float: if not resolution: return 0.0 (low, high, scale) = estimate_range(spec) if scale == "log": return math.log1p(resolution) / (math.log(high) - math.log(low)) return resolution / (high - low) def is_date_estimate(spec: dict[str, Any]) -> bool: return spec.get("type") == "estimate" and isinstance(spec.get("min"), str) def date_granularity(spec: dict[str, Any]) -> str: step = spec.get("step") return step if step in ("month", "year") else "day" def date_index(value: str, granularity: str = "day") -> float: date = datetime.date.fromisoformat(value) if granularity == "month": return float(date.year * 12 + date.month - 1) if granularity == "year": return float(date.year) return float(date.toordinal()) def index_date(index: float, granularity: str = "day") -> str: whole = int(round(index)) if granularity == "month": (year, month) = divmod(whole, 12) return datetime.date(year, month + 1, 1).isoformat() if granularity == "year": return datetime.date(whole, 1, 1).isoformat() return datetime.date.fromordinal(whole).isoformat() def estimate_range(spec: dict[str, Any]) -> tuple[float, float, str]: if is_date_estimate(spec): granularity = date_granularity(spec) return ( date_index(spec["min"], granularity), date_index(spec["max"], granularity), "linear", ) return (float(spec["min"]), float(spec["max"]), spec.get("scale", "linear")) def estimate_step(spec: dict[str, Any]) -> float: if is_date_estimate(spec): return ( 1.0 if date_granularity(spec) != "day" else float(spec.get("step") or 1.0) ) return float(spec.get("step") or 0.0) def estimate_number(spec: dict[str, Any], value: Any) -> float: if is_date_estimate(spec): return date_index(str(value), date_granularity(spec)) return float(value) def estimate_value(spec: dict[str, Any], number: float) -> float | str: if is_date_estimate(spec): (low, high, _) = estimate_range(spec) return index_date(min(max(number, low), high), date_granularity(spec)) return number def grid_bounds(step: float, low: float, high: float) -> tuple[float, float]: first = math.ceil(low / step - 1e-09) * step last = math.floor(high / step + 1e-09) * step return (float(f"{first:.12g}"), float(f"{last:.12g}")) def snap(value: float, step: float, low: float, high: float) -> float: (first, last) = grid_bounds(step, low, high) return float(f"{min(last, max(first, round(value / step) * step)):.12g}") def snap_quantile(value: float, step: float, low: float, high: float) -> float: (first, last) = grid_bounds(step, low, high) return float( f"{min(last, max(first, math.ceil((value - step / 2) / step - 1e-09) * step)):.12g}" ) def step_interval(spec: dict[str, Any], value: float) -> tuple[float, float]: step = estimate_step(spec) (low, high, _) = estimate_range(spec) if step <= 0: return (value, value) return (max(low, value - step / 2), min(high, value + step / 2)) def _cuts_word(text: str, start: int, end: int) -> bool: def joined(left: str, right: str) -> bool: return left.isascii() and left.isalnum() and right.isascii() and right.isalnum() return ( start > 0 and joined(text[start - 1], text[start]) or (end < len(text) and joined(text[end - 1], text[end])) ) def _prefer_whole( text: str, spans: list[tuple[int, int]], limit: int ) -> list[tuple[int, int]]: whole = [span for span in spans if not _cuts_word(text, *span)] return (whole or spans)[:limit] def find_spans(text: str, value: str, limit: int = 16) -> list[tuple[int, int]]: value = value.strip() if not value: return [] scan = max(limit * 8, 64) spans: list[tuple[int, int]] = [] start = text.find(value) while start >= 0 and len(spans) < scan: spans.append((start, start + len(value))) start = text.find(value, start + 1) if spans: return _prefer_whole(text, spans, limit) (lowered, target) = (text.lower(), value.lower()) if len(lowered) == len(text): start = lowered.find(target) while start >= 0 and len(spans) < scan: spans.append((start, start + len(value))) start = lowered.find(target, start + 1) if spans: return _prefer_whole(text, spans, limit) pieces = value.split() if not pieces: return [] pattern = re.compile( "\\s+".join((re.escape(piece) for piece in pieces)), re.IGNORECASE ) spans = [ (match.start(), match.end()) for match in itertools.islice(pattern.finditer(text), scan) ] return _prefer_whole(text, spans, limit) def validate_fields(fields: Any) -> None: if not fields or not isinstance(fields, dict): raise ValueError("record must declare at least one field") for name, spec in fields.items(): if not isinstance(name, str) or not FIELD_NAME.match(name): raise ValueError(f"invalid field name {name!r}") kind = spec.get("type") if isinstance(spec, dict) else None if isinstance(spec, dict) and "knowledge" in spec: if not isinstance(spec["knowledge"], bool): raise ValueError(f"field {name}: knowledge must be true or false") if kind not in ("text", "number") and ( not (kind == "list" and spec.get("items") in ("text", "number")) ): raise ValueError( f"field {name}: knowledge applies to text and number fields only" ) if kind == "compute": if spec.get("nullable", True) is not True or any( ( key in spec for key in ( "minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum", "enum", ) ) ): raise ValueError( f"compute {name} does not support AR grammar constraints" ) kind = "number" if kind not in FIELD_TYPES + ("boolean", "enum", "object"): raise ValueError(f"field {name} has unknown type {kind!r}") if "null_threshold" in spec: threshold = spec["null_threshold"] if kind in ("estimate", "list"): raise ValueError(f"{kind} {name} does not take null_threshold") if ( isinstance(threshold, bool) or not isinstance(threshold, (int, float)) or (not 0 < threshold <= 1) ): raise ValueError(f"field {name} needs null_threshold in (0, 1]") row_items = ( isinstance(spec.get("items"), dict) and spec["items"].get("type") == "object" ) if ( kind == "list" and (not row_items) and (spec.get("items") not in LIST_ITEM_TYPES) ): raise ValueError(f"list {name} needs items in {LIST_ITEM_TYPES}") if kind == "duration" and duration_unit(spec) not in DURATION_UNITS: raise ValueError(f"duration {name} needs a unit in {DURATION_UNITS}") numeric = kind in ("number", "duration") or ( kind == "list" and spec.get("items") == "number" ) if not numeric and ("integer" in spec or "decimals" in spec): raise ValueError( f"field {name}: integer and decimals apply to numbers and durations only" ) if "integer" in spec and (not isinstance(spec["integer"], bool)): raise ValueError(f"field {name}: integer must be true or false") if "decimals" in spec: decimals = spec["decimals"] if ( isinstance(decimals, bool) or not isinstance(decimals, int) or (not 0 <= decimals <= MAX_DECIMALS) ): raise ValueError( f"field {name}: decimals must be an integer from 0 to {MAX_DECIMALS}" ) if spec.get("integer") and decimals: raise ValueError(f"field {name}: an integer field has 0 decimals") if kind == "estimate" and is_date_estimate(spec): validate_date_range(name, spec) elif kind == "estimate": validate_number_range(name, spec) def null_threshold(spec: dict[str, Any]) -> float: return float(spec.get("null_threshold", DEFAULT_NULL_THRESHOLD)) def validate_number_range(name: str, spec: dict[str, Any]) -> None: try: (low, high) = (float(spec["min"]), float(spec["max"])) step = None if spec.get("step") is None else float(spec["step"]) except (KeyError, TypeError, ValueError) as error: raise ValueError(f"estimate {name} needs numeric min, max, and step") from error if not (math.isfinite(low) and math.isfinite(high) and (low < high)) or spec.get( "scale", "linear" ) not in ("linear", "log"): raise ValueError(f"field {name} has an invalid range") if spec.get("scale") == "log" and low <= 0: raise ValueError(f"log field {name} needs a positive minimum") if step is not None and ( not math.isfinite(step) or step <= 0 or (high - low) / step < 1 or ((high - low) / step > 1000000.0) ): raise ValueError(f"estimate {name} has an invalid step {spec['step']!r}") def validate_reference_date(value: Any) -> None: if value is None: return try: datetime.date.fromisoformat(value) except (TypeError, ValueError) as error: raise ValueError( f"reference_date must be an ISO date, got {value!r}" ) from error def validate_date_range(name: str, spec: dict[str, Any]) -> None: try: (low, high, _) = estimate_range(spec) step = ( float(spec["step"]) if date_granularity(spec) == "day" and spec.get("step") is not None else 1.0 ) except (KeyError, TypeError, ValueError) as error: raise ValueError( f"date estimate {name} needs ISO bounds and a step in days, 'month', or 'year'" ) from error if ( not isinstance(spec.get("max"), str) or spec.get("scale", "linear") != "linear" or (not low < high) ): raise ValueError(f"date estimate {name} has an invalid range") if not step.is_integer() or step < 1 or (high - low) / step < 1: raise ValueError(f"date estimate {name} has an invalid step {spec['step']!r}")