lucataco's picture
Clairic Flash, MLX 4-bit, with runtime, server and conversion script
ae97c5d verified
Raw History Blame Contribute Delete
21.1 kB
from __future__ import annotations
import datetime
import hashlib
import itertools
import math
import re
from typing import Any, cast
READ_TYPES = ("text", "number", "date", "time", "datetime", "duration")
FIELD_TYPES = READ_TYPES + ("list", "estimate")
EVIDENCE_CONSTRUCTION = "evidence-v1"
ANSWER_ROUTES = ("copy", "normalize", "derive", "knowledge", "abstain")
REQUEST_TYPES = FIELD_TYPES + ("compute", "forecast")
LIST_ITEM_TYPES = ("text", "number", "date")
DURATION_UNITS = ("seconds", "minutes", "hours", "days", "weeks", "months", "years")
MAX_LIST_ITEMS = 32
MAX_DECIMALS = 6
MAX_FORECAST_HORIZON = 64
FORECAST_INTERVALS = (
"step",
"minute",
"hour",
"day",
"week",
"month",
"quarter",
"year",
)
DEFAULT_NULL_THRESHOLD = 0.5
def list_item_name(parent: str, index: int) -> str:
suffix = f".item{index}"
if len(parent) + len(suffix) <= 64:
return parent + suffix
digest = hashlib.sha256(parent.encode()).hexdigest()[:8]
return parent[: 64 - len(suffix) - 9] + "_" + digest + suffix
def list_query_fields(fields: dict[str, Any], parent: str) -> list[str]:
spec = fields[parent]
item_type = (
spec["items"].get("type") if isinstance(spec["items"], dict) else spec["items"]
)
queries = {}
for name, child in fields.items():
(_, separator, ordinal) = name.rpartition(".item")
if not separator or not ordinal.isascii() or (not ordinal.isdigit()):
continue
index = int(ordinal)
if (
index > 0
and name == list_item_name(parent, index)
and (child.get("type") == item_type)
):
queries[index] = name
if sorted(queries) != list(range(1, len(queries) + 1)):
raise ValueError(f"noncontiguous scalar list queries for {parent!r}")
return [queries[index] for index in range(1, len(queries) + 1)]
def assemble_list_predictions(
record: dict[str, Any], predictions: dict[str, dict[str, Any]]
) -> dict[str, dict[str, Any]]:
if (record.get("_meta") or {}).get("construction") != EVIDENCE_CONSTRUCTION:
return predictions
fields = record["fields"]
merged = dict(predictions)
for parent, spec in fields.items():
if spec.get("type") != "list":
continue
queries = list_query_fields(fields, parent)
if not queries:
continue
(values, confidences, probabilities) = ([], [], [])
for name in queries:
item = predictions.get(name)
if item is None or "value" not in item:
raise ValueError(f"missing scalar list prediction for {name!r}")
if item["value"] is None:
probabilities.append(item.get("p_null"))
break
values.append(item["value"])
confidences.append(item.get("confidence"))
probabilities.append(
(1.0 - item["p_null"]) * item["confidence"]
if "p_null" in item and "confidence" in item
else None
)
assembled: dict[str, Any] = {"value": values}
if all((value is not None for value in confidences)):
assembled["confidences"] = confidences
if probabilities and all((value is not None for value in probabilities)):
assembled["confidence"] = math.prod(cast(list[float], probabilities))
merged[parent] = assembled
return merged
SYSTEM_PREAMBLE = "Fill each field from the input with a value of its type. Text copies a span of the input. Numbers, dates, times, datetimes, and durations are read from the input or worked out from it. A list holds every matching value in input order. A field is not stated when the input does not determine it. Estimates predict a value inside the declared range."
EVIDENCE_SYSTEM_PREAMBLE = "Fill each field with a value of its type, following its instructions and the question's scope. For document-specific extraction, use the supplied input and return not stated when the requested information is missing. Preserve exact text, names and numbers unless normalization or a derived answer is requested. General factual questions may use reliable prior knowledge unless the instructions restrict answers to the supplied evidence; return not stated when the answer cannot be determined reliably. A list holds every matching item in the requested order, including repeated items. Estimates and forecasts predict distributions over the declared range and supported below-range or above-range outcomes. Follow the requested units, time intervals and horizons."
FIELD_NAME = re.compile("^[A-Za-z_][A-Za-z0-9_.\\-]{0,63}$")
def format_bound(value: float) -> str:
if float(value).is_integer() and abs(value) < 1000000000000000.0:
return str(int(value))
return f"{value:.6g}"
def field_line(
name: str, spec: dict[str, Any], preserve_instructions: bool = False
) -> str:
kind = spec["type"]
unit = str(spec.get("unit") or "").strip()
if kind == "estimate" and is_date_estimate(spec):
head = f"estimate date, {spec['min']} to {spec['max']}"
granularity = date_granularity(spec)
if granularity != "day":
head += f", by {granularity}"
elif spec.get("step") and float(spec["step"]) != 1:
head += f", step {format_bound(float(spec['step']))} days"
elif kind == "estimate":
head = f"estimate{(' ' + unit if unit else '')}, {format_bound(spec['min'])} to {format_bound(spec['max'])}, {spec.get('scale', 'linear')} scale"
if spec.get("step"):
head += f", step {format_bound(float(spec['step']))}"
elif kind == "list":
head = f"list of {spec['items']}{number_options(spec)}{(' (' + unit + ')' if unit else '')}"
elif kind == "duration":
head = f"duration in {duration_unit(spec)}{number_options(spec)}"
else:
head = f"{kind}{(number_options(spec) if kind == 'number' else '')}{(' (' + unit + ')' if unit else '')}"
instructions = str(spec.get("instructions") or "")
if not preserve_instructions:
instructions = " ".join(instructions.split())
return f"{name}: {head}." + (f" {instructions}" if instructions else "")
def number_options(spec: dict[str, Any]) -> str:
if spec.get("integer"):
return ", integer"
if spec.get("decimals") is not None:
return f", {int(spec['decimals'])} decimals"
return ""
def duration_unit(spec: dict[str, Any]) -> str:
return str(spec.get("unit") or "seconds")
def allowed_decimals(spec: dict[str, Any]) -> int | None:
if spec.get("integer"):
return 0
return None if spec.get("decimals") is None else int(spec["decimals"])
def parse_time(value: Any) -> tuple[int, int, int]:
if not isinstance(value, str):
raise ValueError(f"time must be a string, got {value!r}")
parsed = datetime.time.fromisoformat(value)
if parsed.microsecond or parsed.tzinfo is not None:
raise ValueError(f"time {value!r} must be HH:MM or HH:MM:SS")
return (parsed.hour, parsed.minute, parsed.second)
def format_time(hour: int, minute: int, second: int) -> str:
return f"{hour:02d}:{minute:02d}:{second:02d}"
def parse_datetime(value: Any) -> tuple[str, tuple[int, int, int]]:
if not isinstance(value, str) or "T" not in value:
raise ValueError(f"datetime must be YYYY-MM-DDTHH:MM[:SS], got {value!r}")
(day, clock) = value.split("T", 1)
return (datetime.date.fromisoformat(day).isoformat(), parse_time(clock))
def forecast_instructions(instructions: str, step: int, interval: str) -> str:
what = " ".join(str(instructions or "").split()).rstrip(".") or "the series"
unit = "step" if interval == "step" else interval
return f"Forecast {what} {step} {unit}{('s' if step > 1 else '')} after the last observation."
def expand_forecasts(
fields: dict[str, Any],
) -> tuple[dict[str, Any], dict[str, list[str]]]:
expanded: dict[str, Any] = {}
groups: dict[str, list[str]] = {}
for name, spec in fields.items():
if not isinstance(spec, dict) or spec.get("type") != "forecast":
expanded[name] = spec
continue
horizon = spec.get("horizon")
if (
isinstance(horizon, bool)
or not isinstance(horizon, int)
or (not 1 <= horizon <= MAX_FORECAST_HORIZON)
):
raise ValueError(
f"forecast {name} needs an integer horizon from 1 to {MAX_FORECAST_HORIZON}"
)
interval = spec.get("interval", "step")
if interval not in FORECAST_INTERVALS:
raise ValueError(f"forecast {name} has an unknown interval {interval!r}")
base = {
key: value
for (key, value) in spec.items()
if key not in ("type", "horizon", "interval", "instructions")
}
groups[name] = []
for step in range(1, horizon + 1):
step_name = f"{name}.{step}"
if step_name in fields or step_name in expanded:
raise ValueError(
f"forecast {name} step {step_name} collides with another field"
)
expanded[step_name] = {
"type": "estimate",
**base,
"instructions": forecast_instructions(
spec.get("instructions", ""), step, interval
),
}
groups[name].append(step_name)
return (expanded, groups)
def to_unit(value: float, low: float, high: float, scale: str) -> float:
return min(1.0, max(0.0, raw_unit(value, low, high, scale)))
def raw_unit(value: float, low: float, high: float, scale: str) -> float:
if scale == "log":
if value <= 0:
return -1.0
return (math.log(value) - math.log(low)) / (math.log(high) - math.log(low))
return (value - low) / (high - low)
def from_unit(position: float, low: float, high: float, scale: str) -> float:
position = min(1.0, max(0.0, position))
if scale == "log":
value = math.exp(math.log(low) + position * (math.log(high) - math.log(low)))
else:
value = low + position * (high - low)
return min(high, max(low, value))
def resolution_in_unit(spec: dict[str, Any], resolution: float | None) -> float:
if not resolution:
return 0.0
(low, high, scale) = estimate_range(spec)
if scale == "log":
return math.log1p(resolution) / (math.log(high) - math.log(low))
return resolution / (high - low)
def is_date_estimate(spec: dict[str, Any]) -> bool:
return spec.get("type") == "estimate" and isinstance(spec.get("min"), str)
def date_granularity(spec: dict[str, Any]) -> str:
step = spec.get("step")
return step if step in ("month", "year") else "day"
def date_index(value: str, granularity: str = "day") -> float:
date = datetime.date.fromisoformat(value)
if granularity == "month":
return float(date.year * 12 + date.month - 1)
if granularity == "year":
return float(date.year)
return float(date.toordinal())
def index_date(index: float, granularity: str = "day") -> str:
whole = int(round(index))
if granularity == "month":
(year, month) = divmod(whole, 12)
return datetime.date(year, month + 1, 1).isoformat()
if granularity == "year":
return datetime.date(whole, 1, 1).isoformat()
return datetime.date.fromordinal(whole).isoformat()
def estimate_range(spec: dict[str, Any]) -> tuple[float, float, str]:
if is_date_estimate(spec):
granularity = date_granularity(spec)
return (
date_index(spec["min"], granularity),
date_index(spec["max"], granularity),
"linear",
)
return (float(spec["min"]), float(spec["max"]), spec.get("scale", "linear"))
def estimate_step(spec: dict[str, Any]) -> float:
if is_date_estimate(spec):
return (
1.0 if date_granularity(spec) != "day" else float(spec.get("step") or 1.0)
)
return float(spec.get("step") or 0.0)
def estimate_number(spec: dict[str, Any], value: Any) -> float:
if is_date_estimate(spec):
return date_index(str(value), date_granularity(spec))
return float(value)
def estimate_value(spec: dict[str, Any], number: float) -> float | str:
if is_date_estimate(spec):
(low, high, _) = estimate_range(spec)
return index_date(min(max(number, low), high), date_granularity(spec))
return number
def grid_bounds(step: float, low: float, high: float) -> tuple[float, float]:
first = math.ceil(low / step - 1e-09) * step
last = math.floor(high / step + 1e-09) * step
return (float(f"{first:.12g}"), float(f"{last:.12g}"))
def snap(value: float, step: float, low: float, high: float) -> float:
(first, last) = grid_bounds(step, low, high)
return float(f"{min(last, max(first, round(value / step) * step)):.12g}")
def snap_quantile(value: float, step: float, low: float, high: float) -> float:
(first, last) = grid_bounds(step, low, high)
return float(
f"{min(last, max(first, math.ceil((value - step / 2) / step - 1e-09) * step)):.12g}"
)
def step_interval(spec: dict[str, Any], value: float) -> tuple[float, float]:
step = estimate_step(spec)
(low, high, _) = estimate_range(spec)
if step <= 0:
return (value, value)
return (max(low, value - step / 2), min(high, value + step / 2))
def _cuts_word(text: str, start: int, end: int) -> bool:
def joined(left: str, right: str) -> bool:
return left.isascii() and left.isalnum() and right.isascii() and right.isalnum()
return (
start > 0
and joined(text[start - 1], text[start])
or (end < len(text) and joined(text[end - 1], text[end]))
)
def _prefer_whole(
text: str, spans: list[tuple[int, int]], limit: int
) -> list[tuple[int, int]]:
whole = [span for span in spans if not _cuts_word(text, *span)]
return (whole or spans)[:limit]
def find_spans(text: str, value: str, limit: int = 16) -> list[tuple[int, int]]:
value = value.strip()
if not value:
return []
scan = max(limit * 8, 64)
spans: list[tuple[int, int]] = []
start = text.find(value)
while start >= 0 and len(spans) < scan:
spans.append((start, start + len(value)))
start = text.find(value, start + 1)
if spans:
return _prefer_whole(text, spans, limit)
(lowered, target) = (text.lower(), value.lower())
if len(lowered) == len(text):
start = lowered.find(target)
while start >= 0 and len(spans) < scan:
spans.append((start, start + len(value)))
start = lowered.find(target, start + 1)
if spans:
return _prefer_whole(text, spans, limit)
pieces = value.split()
if not pieces:
return []
pattern = re.compile(
"\\s+".join((re.escape(piece) for piece in pieces)), re.IGNORECASE
)
spans = [
(match.start(), match.end())
for match in itertools.islice(pattern.finditer(text), scan)
]
return _prefer_whole(text, spans, limit)
def validate_fields(fields: Any) -> None:
if not fields or not isinstance(fields, dict):
raise ValueError("record must declare at least one field")
for name, spec in fields.items():
if not isinstance(name, str) or not FIELD_NAME.match(name):
raise ValueError(f"invalid field name {name!r}")
kind = spec.get("type") if isinstance(spec, dict) else None
if isinstance(spec, dict) and "knowledge" in spec:
if not isinstance(spec["knowledge"], bool):
raise ValueError(f"field {name}: knowledge must be true or false")
if kind not in ("text", "number") and (
not (kind == "list" and spec.get("items") in ("text", "number"))
):
raise ValueError(
f"field {name}: knowledge applies to text and number fields only"
)
if kind == "compute":
if spec.get("nullable", True) is not True or any(
(
key in spec
for key in (
"minimum",
"maximum",
"exclusiveMinimum",
"exclusiveMaximum",
"enum",
)
)
):
raise ValueError(
f"compute {name} does not support AR grammar constraints"
)
kind = "number"
if kind not in FIELD_TYPES + ("boolean", "enum", "object"):
raise ValueError(f"field {name} has unknown type {kind!r}")
if "null_threshold" in spec:
threshold = spec["null_threshold"]
if kind in ("estimate", "list"):
raise ValueError(f"{kind} {name} does not take null_threshold")
if (
isinstance(threshold, bool)
or not isinstance(threshold, (int, float))
or (not 0 < threshold <= 1)
):
raise ValueError(f"field {name} needs null_threshold in (0, 1]")
row_items = (
isinstance(spec.get("items"), dict)
and spec["items"].get("type") == "object"
)
if (
kind == "list"
and (not row_items)
and (spec.get("items") not in LIST_ITEM_TYPES)
):
raise ValueError(f"list {name} needs items in {LIST_ITEM_TYPES}")
if kind == "duration" and duration_unit(spec) not in DURATION_UNITS:
raise ValueError(f"duration {name} needs a unit in {DURATION_UNITS}")
numeric = kind in ("number", "duration") or (
kind == "list" and spec.get("items") == "number"
)
if not numeric and ("integer" in spec or "decimals" in spec):
raise ValueError(
f"field {name}: integer and decimals apply to numbers and durations only"
)
if "integer" in spec and (not isinstance(spec["integer"], bool)):
raise ValueError(f"field {name}: integer must be true or false")
if "decimals" in spec:
decimals = spec["decimals"]
if (
isinstance(decimals, bool)
or not isinstance(decimals, int)
or (not 0 <= decimals <= MAX_DECIMALS)
):
raise ValueError(
f"field {name}: decimals must be an integer from 0 to {MAX_DECIMALS}"
)
if spec.get("integer") and decimals:
raise ValueError(f"field {name}: an integer field has 0 decimals")
if kind == "estimate" and is_date_estimate(spec):
validate_date_range(name, spec)
elif kind == "estimate":
validate_number_range(name, spec)
def null_threshold(spec: dict[str, Any]) -> float:
return float(spec.get("null_threshold", DEFAULT_NULL_THRESHOLD))
def validate_number_range(name: str, spec: dict[str, Any]) -> None:
try:
(low, high) = (float(spec["min"]), float(spec["max"]))
step = None if spec.get("step") is None else float(spec["step"])
except (KeyError, TypeError, ValueError) as error:
raise ValueError(f"estimate {name} needs numeric min, max, and step") from error
if not (math.isfinite(low) and math.isfinite(high) and (low < high)) or spec.get(
"scale", "linear"
) not in ("linear", "log"):
raise ValueError(f"field {name} has an invalid range")
if spec.get("scale") == "log" and low <= 0:
raise ValueError(f"log field {name} needs a positive minimum")
if step is not None and (
not math.isfinite(step)
or step <= 0
or (high - low) / step < 1
or ((high - low) / step > 1000000.0)
):
raise ValueError(f"estimate {name} has an invalid step {spec['step']!r}")
def validate_reference_date(value: Any) -> None:
if value is None:
return
try:
datetime.date.fromisoformat(value)
except (TypeError, ValueError) as error:
raise ValueError(
f"reference_date must be an ISO date, got {value!r}"
) from error
def validate_date_range(name: str, spec: dict[str, Any]) -> None:
try:
(low, high, _) = estimate_range(spec)
step = (
float(spec["step"])
if date_granularity(spec) == "day" and spec.get("step") is not None
else 1.0
)
except (KeyError, TypeError, ValueError) as error:
raise ValueError(
f"date estimate {name} needs ISO bounds and a step in days, 'month', or 'year'"
) from error
if (
not isinstance(spec.get("max"), str)
or spec.get("scale", "linear") != "linear"
or (not low < high)
):
raise ValueError(f"date estimate {name} has an invalid range")
if not step.is_integer() or step < 1 or (high - low) / step < 1:
raise ValueError(f"date estimate {name} has an invalid step {spec['step']!r}")