fix: rework monetary_fields as in-place feature-extract config with status report

Replace the auto-discovered monetary_fields array (removed from /json/extract)
with an optional per-item request config on /json/feature-extract
({object_name, field_names}) that price-parses the declared fields in place
inside the extracted data. The response now carries a per-field status/error
report (parsed / not_found / not_parsable / skipped) so callers can see what
happened to every declared monetary field.

Generated with Codebuff 🤖
Co-Authored-By: Codebuff <noreply@codebuff.com>

app/api/v1/json_extract.py

@@ -12,7 +12,7 @@ from app.core.logger import get_logger
 from app.core.thread_pool import run_in_executor
 from app.services.gliner_service import gliner_service
 from app.services.json_service import extract_json
-from app.services.monetary_field_service import MonetaryField, find_monetary_fields
+from app.services.monetary_field_service import MonetaryFieldStatus, apply_monetary_fields
 
 logger = get_logger(__name__)
 
@@ -46,13 +46,6 @@ class ExtractJsonResponse(BaseModel):
     data: Any = None
     count: int = 0
     error_message: Optional[str] = None
-    monetary_fields: List[MonetaryField] = Field(
-        default_factory=list,
-        description=(
-            "Monetary values found inside the extracted data (via the JSON key "
-            "finder + amount parser). Empty when none are found."
-        ),
-    )
 
 
 @router.post(
@@ -113,8 +106,6 @@ async def extract_json_endpoint(
 
     response_data = result.data[0] if body.mode == "first" else result.data
 
-    monetary = find_monetary_fields(response_data)
-
     logger.info(
         "JSON extraction successful",
         extra={
@@ -131,7 +122,6 @@ async def extract_json_endpoint(
         data=response_data,
         count=result.total_extracted,
         error_message=None,
-        monetary_fields=monetary,
     )
 
 
@@ -171,6 +161,32 @@ class NoAiExtractRequest(BaseModel):
         le=1.0,
         description="Confidence threshold (0.0-1.0). Lower includes more candidates.",
     )
+    monetary_fields: Optional["MonetaryFieldsConfig"] = Field(
+        default=None,
+        description=(
+            "OPTIONAL. Declare which extracted fields hold monetary values so they "
+            "are price-parsed in place: `object_name` selects the container key in "
+            "the extracted `data` (e.g. 'invoice') and `field_names` lists the "
+            "monetary fields inside each object (e.g. ['total']). Found values are "
+            "replaced with their parsed numeric amount; missing/blank/unparseable "
+            "values are skipped. Omit to leave the extracted data untouched."
+        ),
+    )
+
+
+class MonetaryFieldsConfig(BaseModel):
+    """Declares which extracted fields are monetary so they are price-parsed."""
+
+    object_name: str = Field(
+        ...,
+        min_length=1,
+        description="Key in the extracted `data` whose value holds the objects to process, e.g. 'invoice'.",
+    )
+    field_names: List[str] = Field(
+        ...,
+        min_length=1,
+        description="Monetary field names inside each object, e.g. ['total', 'cgst'].",
+    )
 
 
 class NoAiExtractResponse(BaseModel):
@@ -180,11 +196,13 @@ class NoAiExtractResponse(BaseModel):
     data: Any = None
     count: int = 0
     error_message: Optional[str] = None
-    monetary_fields: List[MonetaryField] = Field(
+    monetary_fields: List[MonetaryFieldStatus] = Field(
         default_factory=list,
         description=(
-            "Monetary values found inside this item's extracted data (via the "
-            "JSON key finder + amount parser). Empty when none are found."
+            "Per-field outcome of the optional `monetary_fields` config: status "
+            "is 'parsed', 'not_found', 'not_parsable' or 'skipped', with the "
+            "parsed amount and a human-readable error. Empty when no config "
+            "was sent."
         ),
     )
 
@@ -212,7 +230,9 @@ def _count_extracted(result: Any) -> int:
         "for private or invoice/OCR data. mode='json' uses a structure schema to "
         "pull named fields; mode='entities' detects a flat list of entity types. "
         "Each item reports its own success/error. "
-        "Returns HTTP 503 if the model failed to load or is disabled."
+        "Returns HTTP 503 if the model failed to load or is disabled. "
+        "OPTIONAL per-item `monetary_fields` ({\"object_name\", \"field_names\"}) "
+        "price-parses the declared fields in place inside the extracted data."
     ),
 )
 async def no_ai_extract_endpoint(
@@ -312,14 +332,27 @@ async def no_ai_extract_endpoint(
             )
 
         elapsed = round((time.perf_counter() - start) * 1000, 3)
-        monetary = find_monetary_fields(result)
+
+        # OPTIONAL post-processing: price-parse the declared monetary fields
+        # in place inside the extracted data (e.g. "1250.75" -> 1250.75) and
+        # report the per-field outcome so callers can understand any failures.
+        monetary_report: List[MonetaryFieldStatus] = []
+        parsed_monetary = 0
+        if item.monetary_fields is not None:
+            monetary_report = apply_monetary_fields(
+                result,
+                item.monetary_fields.object_name,
+                item.monetary_fields.field_names,
+            )
+            parsed_monetary = sum(1 for s in monetary_report if s.status == "parsed")
+
         logger.info(
             "Feature-extract item extracted",
             extra={
                 "index": index,
                 "mode": item.mode,
                 "count": _count_extracted(result),
-                "monetary_fields": len(monetary),
+                "monetary_fields_parsed": parsed_monetary,
                 "time_ms": elapsed,
             },
         )
@@ -330,7 +363,7 @@ async def no_ai_extract_endpoint(
             data=result,
             count=_count_extracted(result),
             error_message=None,
-            monetary_fields=monetary,
+            monetary_fields=monetary_report,
         )
 
     results = await asyncio.gather(

app/services/monetary_field_service.py

@@ -1,22 +1,26 @@
-"""Monetary-field detection for feature-extraction results.
-
-Given the ``data`` payload of a JSON / feature-extraction response, locate the
-values of known monetary field names (``total``, ``subtotal``, ``cgst``,
-``balance`` ...) at any nesting depth using the recursive JSON key finder
-(:mod:`app.services.keys_extractor_service`), then normalise each found value
-with the amount parser (:mod:`app.services.price_parser`).
-
-Usage
------
->>> from app.services.monetary_field_service import find_monetary_fields
->>> data = {"invoice": [{"total": "INR 572,300.00", "vendor": "Acme"}]}
->>> find_monetary_fields(data)
-[MonetaryField(key='total', raw='INR 572,300.00', amount=572300.0,
-               currency='INR', currency_code='INR', confidence=1.0)]
-
-When no monetary field is found (or none of the found values parse as an
-amount) the result is an empty list — callers should treat that as
-"no monetary fields found".
+"""In-place monetary-field parsing for the feature-extraction pipeline.
+
+When a feature-extract request declares ``monetary_fields`` — an object name
+plus a list of field names — the extracted ``data`` payload is post-processed
+so those fields become real numbers instead of raw strings, and a per-field
+status report is returned so callers can see exactly what happened to each
+declared field.
+
+Example
+-------
+Request config::
+
+    {"object_name": "invoice", "field_names": ["total", "invoice_no"]}
+
+Extracted data::
+
+    {"invoice": [{"number": "INV-2026-00125", "total": "1250.75"}]}
+
+After :func:`apply_monetary_fields` the same dict has ``"total": 1250.75``
+(price-parsed) and the returned report explains each field:
+
+* ``total``      -> status ``parsed``, amount 1250.75
+* ``invoice_no`` -> status ``not_found`` (no such field in the object)
 """
 
 from __future__ import annotations
@@ -25,103 +29,152 @@ from typing import Any, List, Optional
 
 from pydantic import BaseModel, Field
 
-from app.services.keys_extractor_service import KeysExtractor
 from app.services.price_parser import parse_amount
 
-#: Field names that typically hold monetary values, in priority order.
-#: Matched by exact key name at any depth (e.g. ``data["invoice"][0]["total"]``).
-DEFAULT_MONETARY_KEYS: List[str] = [
-    "grand_total",
-    "total_amount",
-    "amount_due",
-    "total",
-    "subtotal",
-    "sub_total",
-    "balance",
-    "payable",
-    "amount",
-    "price",
-    "unit_price",
-    "unitprice",
-    "rate",
-    "cgst",
-    "sgst",
-    "igst",
-    "total_tax",
-    "tax",
-    "vat",
-    "gst",
-    "discount",
-    "shipping",
-    "freight",
-    "fee",
-    "deposit",
-    "paid",
-    "due",
-    "net",
-    "gross",
-    "value",
-    "sum",
-]
-
-
-class MonetaryField(BaseModel):
-    """One parsed monetary field found inside an extraction result."""
-
-    key: str
-    """JSON key name the value was found under (exact match, any depth)."""
-    raw: Optional[str] = None
-    """The value as it appeared in the extraction result (string form)."""
+
+class MonetaryFieldStatus(BaseModel):
+    """Outcome of one declared monetary field, per extracted object item."""
+
+    object_name: str
+    """Container key the field was looked up in (from the request config)."""
+    field: str
+    """Monetary field name (from the request config)."""
+    item_index: int = 0
+    """Index of the object inside the container this report refers to."""
+    status: str = Field(
+        default="not_found",
+        description="One of: parsed, not_found, not_parsable, skipped.",
+    )
+    value: Optional[str] = None
+    """The raw value as extracted (string form), when present."""
     amount: Optional[float] = None
-    """Normalised numeric amount, or ``None`` when the value did not parse."""
+    """Parsed numeric amount when status == 'parsed'."""
     currency: Optional[str] = None
-    """Currency marker found in the value (symbol, code or word)."""
+    """Currency marker found in the value, when parsed."""
     currency_code: Optional[str] = None
-    """ISO 4217 code when determinable, else ``None``."""
-    confidence: float = Field(default=0.0, ge=0.0, le=1.0)
-    """Parser confidence in [0, 1] — higher is more likely a real amount."""
+    """ISO 4217 code when determinable, else None."""
+    confidence: float = 0.0
+    """Parser confidence in [0, 1] when parsed."""
+    error: Optional[str] = None
+    """Human-readable reason when the field was not parsed."""
+
+
+def _status(object_name: str, field: str, item_index: int, **kw) -> MonetaryFieldStatus:
+    return MonetaryFieldStatus(object_name=object_name, field=field, item_index=item_index, **kw)
+
 
+def apply_monetary_fields(data: Any, object_name: str, field_names: List[str]) -> List[MonetaryFieldStatus]:
+    """Price-parse the configured monetary fields inside ``data``, in place.
 
-def find_monetary_fields(
-    data: Any,
-    key_names: Optional[List[str]] = None,
-) -> List[MonetaryField]:
-    """Return parsed monetary fields found in ``data`` (empty list if none).
+    ``object_name`` selects the container key in ``data`` (e.g. ``"invoice"``);
+    its value may be a list of objects or a single object. For every listed
+    ``field_names`` entry inside each object:
 
-    ``data`` may be a ``dict`` or a ``list`` (any nesting depth). Only values
-    that parse as amounts are returned; found-but-unparseable values are
-    skipped. Invalid ``data`` types simply yield an empty list.
+    * value missing -> status ``not_found`` (error ``field 'x' not found``)
+    * value ``None`` / blank -> status ``skipped`` (error ``value is empty``)
+    * value that does not parse -> status ``not_parsable`` with the raw value
+    * value that parses -> replaced with the numeric amount, status ``parsed``
+
+    Returns one :class:`MonetaryFieldStatus` per (object item, field).
     """
-    keys = key_names or DEFAULT_MONETARY_KEYS
-    if not isinstance(data, (dict, list)):
-        return []
-
-    try:
-        extractor = KeysExtractor(data, key_names=keys, result_limit=None)
-        found = extractor.extract()
-    except (TypeError, ValueError):
-        return []
-
-    results: List[MonetaryField] = []
-    for key in keys:
-        for value in found.get(key, []):
-            if value is None or isinstance(value, bool):
+    report: List[MonetaryFieldStatus] = []
+
+    if not isinstance(data, dict) or not object_name:
+        for field in field_names:
+            report.append(
+                _status(
+                    object_name,
+                    field,
+                    0,
+                    status="not_found",
+                    error=f"object '{object_name}' not found in extracted data",
+                )
+            )
+        return report
+
+    container = data.get(object_name)
+    if isinstance(container, list):
+        items: List[Any] = container
+    elif isinstance(container, dict):
+        items = [container]
+    else:
+        for field in field_names:
+            report.append(
+                _status(
+                    object_name,
+                    field,
+                    0,
+                    status="not_found",
+                    error=f"object '{object_name}' not found in extracted data",
+                )
+            )
+        return report
+
+    for idx, item in enumerate(items):
+        if not isinstance(item, dict):
+            for field in field_names:
+                report.append(
+                    _status(
+                        object_name,
+                        field,
+                        idx,
+                        status="not_found",
+                        error="item is not an object",
+                    )
+                )
+            continue
+        for field in field_names:
+            if field not in item:
+                report.append(
+                    _status(
+                        object_name,
+                        field,
+                        idx,
+                        status="not_found",
+                        error=f"field '{field}' not found",
+                    )
+                )
+                continue
+            value = item[field]
+            if value is None or (isinstance(value, str) and not value.strip()):
+                report.append(
+                    _status(
+                        object_name,
+                        field,
+                        idx,
+                        status="skipped",
+                        error="value is empty",
+                    )
+                )
                 continue
             parsed = parse_amount(value)
             if parsed is None:
+                report.append(
+                    _status(
+                        object_name,
+                        field,
+                        idx,
+                        status="not_parsable",
+                        value=str(value),
+                        error=f"value is non-parsable: {value}",
+                    )
+                )
                 continue
-            raw = parsed.raw if isinstance(value, str) else str(value)
-            results.append(
-                MonetaryField(
-                    key=key,
-                    raw=raw,
+            item[field] = parsed.amount_float
+            report.append(
+                _status(
+                    object_name,
+                    field,
+                    idx,
+                    status="parsed",
+                    value=str(value),
                     amount=parsed.amount_float,
                     currency=parsed.currency,
                     currency_code=parsed.currency_code,
                     confidence=parsed.confidence,
                 )
             )
-    return results
+    return report
 
 
-__all__ = ["MonetaryField", "find_monetary_fields", "DEFAULT_MONETARY_KEYS"]
+__all__ = ["MonetaryFieldStatus", "apply_monetary_fields"]