Commit ·
1195870
1
Parent(s): f87e5ec
[KM-630] Data-Access Tools: coerce DB Decimal to float
Browse filesDB NUMERIC columns arrive as decimal.Decimal (asyncpg). This broke
analyze_contribution with a `float + Decimal` TypeError, and the same
Decimal would also break JSON serialization (SSE / analysis_record) of
query_structured output.
- query_structured: _json_safe coerces Decimal -> float when building
ToolOutput.rows, so the output is JSON-safe at the source.
- _materialize: _coerce_decimals normalizes Decimal object-columns to
float64 so the whole analyze_* family receives consistent float input.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
- src/tools/data_access.py +20 -2
- src/tools/invoker.py +26 -3
src/tools/data_access.py
CHANGED
|
@@ -25,6 +25,7 @@ Frozen guarantee (§8.4): **never throws.** Any failure returns
|
|
| 25 |
from __future__ import annotations
|
| 26 |
|
| 27 |
from collections.abc import Callable
|
|
|
|
| 28 |
from typing import Any, Protocol
|
| 29 |
|
| 30 |
from pydantic import ValidationError
|
|
@@ -216,8 +217,13 @@ class DataAccessToolInvoker:
|
|
| 216 |
)
|
| 217 |
|
| 218 |
# QueryResult.rows is list[dict]; ToolOutput.rows is list[list] ordered
|
| 219 |
-
# by `columns` so downstream materialization is positional.
|
| 220 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 221 |
return ToolOutput(
|
| 222 |
tool="query_structured",
|
| 223 |
kind="table",
|
|
@@ -296,6 +302,18 @@ class DataAccessToolInvoker:
|
|
| 296 |
)
|
| 297 |
|
| 298 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 299 |
def _result_source_id(result: RetrievalResult) -> str | None:
|
| 300 |
"""Best-effort extraction of a source_id from a retrieval result's metadata.
|
| 301 |
|
|
|
|
| 25 |
from __future__ import annotations
|
| 26 |
|
| 27 |
from collections.abc import Callable
|
| 28 |
+
from decimal import Decimal
|
| 29 |
from typing import Any, Protocol
|
| 30 |
|
| 31 |
from pydantic import ValidationError
|
|
|
|
| 217 |
)
|
| 218 |
|
| 219 |
# QueryResult.rows is list[dict]; ToolOutput.rows is list[list] ordered
|
| 220 |
+
# by `columns` so downstream materialization is positional. DB NUMERIC
|
| 221 |
+
# columns arrive as `Decimal` (asyncpg) — coerce to float here so the
|
| 222 |
+
# output is JSON-serializable (SSE / analysis_record persistence) and
|
| 223 |
+
# plays nicely with the float math in the analyze_* tools.
|
| 224 |
+
rows = [
|
| 225 |
+
[_json_safe(row.get(c)) for c in result.columns] for row in result.rows
|
| 226 |
+
]
|
| 227 |
return ToolOutput(
|
| 228 |
tool="query_structured",
|
| 229 |
kind="table",
|
|
|
|
| 302 |
)
|
| 303 |
|
| 304 |
|
| 305 |
+
def _json_safe(value: Any) -> Any:
|
| 306 |
+
"""Coerce DB scalar types that JSON can't represent into plain Python.
|
| 307 |
+
|
| 308 |
+
DB drivers return NUMERIC/DECIMAL as `decimal.Decimal`, which is neither
|
| 309 |
+
JSON-serializable nor mixable with `float` math. Convert those to `float`;
|
| 310 |
+
everything else passes through unchanged.
|
| 311 |
+
"""
|
| 312 |
+
if isinstance(value, Decimal):
|
| 313 |
+
return float(value)
|
| 314 |
+
return value
|
| 315 |
+
|
| 316 |
+
|
| 317 |
def _result_source_id(result: RetrievalResult) -> str | None:
|
| 318 |
"""Best-effort extraction of a source_id from a retrieval result's metadata.
|
| 319 |
|
src/tools/invoker.py
CHANGED
|
@@ -20,6 +20,7 @@ keeps working.
|
|
| 20 |
|
| 21 |
from __future__ import annotations
|
| 22 |
|
|
|
|
| 23 |
from collections.abc import Callable
|
| 24 |
from typing import Any
|
| 25 |
|
|
@@ -126,21 +127,43 @@ def _materialize(data: Any) -> tuple[pd.DataFrame, None] | tuple[None, str]:
|
|
| 126 |
Accepts the upstream `ToolOutput` (kind="table"), a raw DataFrame, or a
|
| 127 |
{"columns", "rows"} dict (a serialized table). Returns (df, None) on success
|
| 128 |
or (None, error_message) on failure — the caller wraps the message.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
"""
|
| 130 |
if data is None:
|
| 131 |
return None, "missing 'data' argument (no upstream table to analyze)"
|
| 132 |
|
| 133 |
if isinstance(data, pd.DataFrame):
|
| 134 |
-
return data, None
|
| 135 |
|
| 136 |
if isinstance(data, ToolOutput):
|
| 137 |
if data.kind == "error":
|
| 138 |
return None, f"upstream data unavailable: {data.error}"
|
| 139 |
if data.kind != "table" or data.columns is None:
|
| 140 |
return None, f"cannot materialize 'data' of kind {data.kind!r}"
|
| 141 |
-
return pd.DataFrame(data.rows or [], columns=data.columns), None
|
| 142 |
|
| 143 |
if isinstance(data, dict) and "columns" in data:
|
| 144 |
-
|
|
|
|
| 145 |
|
| 146 |
return None, f"unsupported 'data' type: {type(data).__name__}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
|
| 21 |
from __future__ import annotations
|
| 22 |
|
| 23 |
+
import decimal
|
| 24 |
from collections.abc import Callable
|
| 25 |
from typing import Any
|
| 26 |
|
|
|
|
| 127 |
Accepts the upstream `ToolOutput` (kind="table"), a raw DataFrame, or a
|
| 128 |
{"columns", "rows"} dict (a serialized table). Returns (df, None) on success
|
| 129 |
or (None, error_message) on failure — the caller wraps the message.
|
| 130 |
+
|
| 131 |
+
Numeric columns are coerced to float (see `_coerce_decimals`): DB NUMERIC
|
| 132 |
+
columns arrive as Python `Decimal`, which mixes badly with the float math in
|
| 133 |
+
the `analyze_*` compute functions (e.g. `float + Decimal` -> TypeError).
|
| 134 |
+
Normalizing here fixes the whole tool family in one place.
|
| 135 |
"""
|
| 136 |
if data is None:
|
| 137 |
return None, "missing 'data' argument (no upstream table to analyze)"
|
| 138 |
|
| 139 |
if isinstance(data, pd.DataFrame):
|
| 140 |
+
return _coerce_decimals(data), None
|
| 141 |
|
| 142 |
if isinstance(data, ToolOutput):
|
| 143 |
if data.kind == "error":
|
| 144 |
return None, f"upstream data unavailable: {data.error}"
|
| 145 |
if data.kind != "table" or data.columns is None:
|
| 146 |
return None, f"cannot materialize 'data' of kind {data.kind!r}"
|
| 147 |
+
return _coerce_decimals(pd.DataFrame(data.rows or [], columns=data.columns)), None
|
| 148 |
|
| 149 |
if isinstance(data, dict) and "columns" in data:
|
| 150 |
+
df = pd.DataFrame(data.get("rows") or [], columns=data["columns"])
|
| 151 |
+
return _coerce_decimals(df), None
|
| 152 |
|
| 153 |
return None, f"unsupported 'data' type: {type(data).__name__}"
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def _coerce_decimals(df: pd.DataFrame) -> pd.DataFrame:
|
| 157 |
+
"""Convert `decimal.Decimal` object-columns to float64 in place.
|
| 158 |
+
|
| 159 |
+
DB drivers (asyncpg) return NUMERIC/DECIMAL values as Python `Decimal`, which
|
| 160 |
+
land in object-dtype columns. The `analyze_*` compute functions do float math
|
| 161 |
+
on these (share-of-total, cumulative sums), and `float + Decimal` raises
|
| 162 |
+
TypeError. We only touch columns that actually contain a `Decimal`, so real
|
| 163 |
+
string/categorical columns are left untouched. `None`/missing values become
|
| 164 |
+
NaN, which the compute functions already handle.
|
| 165 |
+
"""
|
| 166 |
+
for col in df.columns:
|
| 167 |
+
if df[col].dtype == object and df[col].map(lambda v: isinstance(v, decimal.Decimal)).any():
|
| 168 |
+
df[col] = df[col].astype(float)
|
| 169 |
+
return df
|