Spaces:
Running
Running
Commit ·
f81d9cb
1
Parent(s): b7071f3
feat: add pure amount parser, monetary-field detection and batch price-parser API
Browse filesAdd a dependency-free amount/price parser (app/services/price_parser.py)
tolerant of OCR distortion, US/EU/Indian separators, magnitudes, negatives
and non-amount patterns. Populate an optional monetary_fields array on the
JSON extract and feature-extract responses via the recursive JSON key
finder, and expose a concurrent batch POST /api/v1/price-parser endpoint.
Generated with Codebuff 🤖
Co-Authored-By: Codebuff <noreply@codebuff.com>
- app/api/v1/json_extract.py +21 -0
- app/api/v1/price_parser.py +193 -0
- app/api/v1/router.py +2 -0
- app/services/monetary_field_service.py +127 -0
- app/services/price_parser.py +814 -0
- services.yaml +3 -0
app/api/v1/json_extract.py
CHANGED
|
@@ -12,6 +12,7 @@ from app.core.logger import get_logger
|
|
| 12 |
from app.core.thread_pool import run_in_executor
|
| 13 |
from app.services.gliner_service import gliner_service
|
| 14 |
from app.services.json_service import extract_json
|
|
|
|
| 15 |
|
| 16 |
logger = get_logger(__name__)
|
| 17 |
|
|
@@ -45,6 +46,13 @@ class ExtractJsonResponse(BaseModel):
|
|
| 45 |
data: Any = None
|
| 46 |
count: int = 0
|
| 47 |
error_message: Optional[str] = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
|
| 49 |
|
| 50 |
@router.post(
|
|
@@ -105,6 +113,8 @@ async def extract_json_endpoint(
|
|
| 105 |
|
| 106 |
response_data = result.data[0] if body.mode == "first" else result.data
|
| 107 |
|
|
|
|
|
|
|
| 108 |
logger.info(
|
| 109 |
"JSON extraction successful",
|
| 110 |
extra={
|
|
@@ -121,6 +131,7 @@ async def extract_json_endpoint(
|
|
| 121 |
data=response_data,
|
| 122 |
count=result.total_extracted,
|
| 123 |
error_message=None,
|
|
|
|
| 124 |
)
|
| 125 |
|
| 126 |
|
|
@@ -169,6 +180,13 @@ class NoAiExtractResponse(BaseModel):
|
|
| 169 |
data: Any = None
|
| 170 |
count: int = 0
|
| 171 |
error_message: Optional[str] = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 172 |
|
| 173 |
|
| 174 |
def _count_extracted(result: Any) -> int:
|
|
@@ -294,12 +312,14 @@ async def no_ai_extract_endpoint(
|
|
| 294 |
)
|
| 295 |
|
| 296 |
elapsed = round((time.perf_counter() - start) * 1000, 3)
|
|
|
|
| 297 |
logger.info(
|
| 298 |
"Feature-extract item extracted",
|
| 299 |
extra={
|
| 300 |
"index": index,
|
| 301 |
"mode": item.mode,
|
| 302 |
"count": _count_extracted(result),
|
|
|
|
| 303 |
"time_ms": elapsed,
|
| 304 |
},
|
| 305 |
)
|
|
@@ -310,6 +330,7 @@ async def no_ai_extract_endpoint(
|
|
| 310 |
data=result,
|
| 311 |
count=_count_extracted(result),
|
| 312 |
error_message=None,
|
|
|
|
| 313 |
)
|
| 314 |
|
| 315 |
results = await asyncio.gather(
|
|
|
|
| 12 |
from app.core.thread_pool import run_in_executor
|
| 13 |
from app.services.gliner_service import gliner_service
|
| 14 |
from app.services.json_service import extract_json
|
| 15 |
+
from app.services.monetary_field_service import MonetaryField, find_monetary_fields
|
| 16 |
|
| 17 |
logger = get_logger(__name__)
|
| 18 |
|
|
|
|
| 46 |
data: Any = None
|
| 47 |
count: int = 0
|
| 48 |
error_message: Optional[str] = None
|
| 49 |
+
monetary_fields: List[MonetaryField] = Field(
|
| 50 |
+
default_factory=list,
|
| 51 |
+
description=(
|
| 52 |
+
"Monetary values found inside the extracted data (via the JSON key "
|
| 53 |
+
"finder + amount parser). Empty when none are found."
|
| 54 |
+
),
|
| 55 |
+
)
|
| 56 |
|
| 57 |
|
| 58 |
@router.post(
|
|
|
|
| 113 |
|
| 114 |
response_data = result.data[0] if body.mode == "first" else result.data
|
| 115 |
|
| 116 |
+
monetary = find_monetary_fields(response_data)
|
| 117 |
+
|
| 118 |
logger.info(
|
| 119 |
"JSON extraction successful",
|
| 120 |
extra={
|
|
|
|
| 131 |
data=response_data,
|
| 132 |
count=result.total_extracted,
|
| 133 |
error_message=None,
|
| 134 |
+
monetary_fields=monetary,
|
| 135 |
)
|
| 136 |
|
| 137 |
|
|
|
|
| 180 |
data: Any = None
|
| 181 |
count: int = 0
|
| 182 |
error_message: Optional[str] = None
|
| 183 |
+
monetary_fields: List[MonetaryField] = Field(
|
| 184 |
+
default_factory=list,
|
| 185 |
+
description=(
|
| 186 |
+
"Monetary values found inside this item's extracted data (via the "
|
| 187 |
+
"JSON key finder + amount parser). Empty when none are found."
|
| 188 |
+
),
|
| 189 |
+
)
|
| 190 |
|
| 191 |
|
| 192 |
def _count_extracted(result: Any) -> int:
|
|
|
|
| 312 |
)
|
| 313 |
|
| 314 |
elapsed = round((time.perf_counter() - start) * 1000, 3)
|
| 315 |
+
monetary = find_monetary_fields(result)
|
| 316 |
logger.info(
|
| 317 |
"Feature-extract item extracted",
|
| 318 |
extra={
|
| 319 |
"index": index,
|
| 320 |
"mode": item.mode,
|
| 321 |
"count": _count_extracted(result),
|
| 322 |
+
"monetary_fields": len(monetary),
|
| 323 |
"time_ms": elapsed,
|
| 324 |
},
|
| 325 |
)
|
|
|
|
| 330 |
data=result,
|
| 331 |
count=_count_extracted(result),
|
| 332 |
error_message=None,
|
| 333 |
+
monetary_fields=monetary,
|
| 334 |
)
|
| 335 |
|
| 336 |
results = await asyncio.gather(
|
app/api/v1/price_parser.py
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Batch amount / price parsing API.
|
| 2 |
+
|
| 3 |
+
Exposes the pure amount parser (:mod:`app.services.price_parser`) as a
|
| 4 |
+
batch endpoint: send up to 100 raw text fields and get every candidate amount
|
| 5 |
+
plus the single best one back per item, processed concurrently.
|
| 6 |
+
|
| 7 |
+
POST /api/v1/price-parser
|
| 8 |
+
[{"text": "I want to go home with ssja^67.00 and $896,009.0 0"}]
|
| 9 |
+
-> [{"success": true, "count": 2,
|
| 10 |
+
"amounts": [{"amount": 896009.0, "currency": "$", ...}, ...],
|
| 11 |
+
"best": {"amount": 896009.0, ...}, ...}]
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
import asyncio
|
| 17 |
+
import time
|
| 18 |
+
from typing import Annotated, List, Optional
|
| 19 |
+
|
| 20 |
+
from fastapi import APIRouter
|
| 21 |
+
from pydantic import BaseModel, Field
|
| 22 |
+
|
| 23 |
+
from app.core.logger import get_logger
|
| 24 |
+
from app.core.thread_pool import thread_pool
|
| 25 |
+
from app.services.price_parser import CONFIDENCE_FLOOR, extract_amounts, parse_amount
|
| 26 |
+
|
| 27 |
+
logger = get_logger(__name__)
|
| 28 |
+
|
| 29 |
+
router = APIRouter()
|
| 30 |
+
|
| 31 |
+
MAX_BATCH_SIZE = 100
|
| 32 |
+
MAX_AMOUNTS_PER_ITEM = 100
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class PriceParseRequest(BaseModel):
|
| 36 |
+
text: str = Field(
|
| 37 |
+
...,
|
| 38 |
+
min_length=1,
|
| 39 |
+
max_length=1_000_000,
|
| 40 |
+
description="Raw text field to parse amounts from (OCR output, invoice lines, emails, ...).",
|
| 41 |
+
)
|
| 42 |
+
currency_hint: Optional[str] = Field(
|
| 43 |
+
default=None,
|
| 44 |
+
description=(
|
| 45 |
+
"Currency to assume for values with no currency marker of their own, "
|
| 46 |
+
"e.g. 'USD', 'руб'. Ignored when a value already carries a currency."
|
| 47 |
+
),
|
| 48 |
+
)
|
| 49 |
+
default_currency: Optional[str] = Field(
|
| 50 |
+
default=None,
|
| 51 |
+
description="Currency label applied to the final result when no currency was found.",
|
| 52 |
+
)
|
| 53 |
+
limit: Optional[int] = Field(
|
| 54 |
+
default=None,
|
| 55 |
+
ge=1,
|
| 56 |
+
le=MAX_AMOUNTS_PER_ITEM,
|
| 57 |
+
description="Maximum number of candidate amounts to return for this item. Omit for all.",
|
| 58 |
+
)
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
class ParsedAmount(BaseModel):
|
| 62 |
+
"""One candidate amount found in a text field."""
|
| 63 |
+
|
| 64 |
+
amount: float
|
| 65 |
+
amount_text: str
|
| 66 |
+
raw: str
|
| 67 |
+
currency: Optional[str] = None
|
| 68 |
+
currency_code: Optional[str] = None
|
| 69 |
+
confidence: float
|
| 70 |
+
is_negative: bool
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class PriceParseItemResponse(BaseModel):
|
| 74 |
+
success: bool
|
| 75 |
+
time_ms: float
|
| 76 |
+
text: str
|
| 77 |
+
count: int
|
| 78 |
+
amounts: List[ParsedAmount] = Field(
|
| 79 |
+
default_factory=list,
|
| 80 |
+
description="All candidate amounts found, ranked by confidence (best first).",
|
| 81 |
+
)
|
| 82 |
+
best: Optional[ParsedAmount] = Field(
|
| 83 |
+
default=None,
|
| 84 |
+
description="The single most likely amount, or null when none was found.",
|
| 85 |
+
)
|
| 86 |
+
error_message: Optional[str] = None
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _to_parsed(amount) -> ParsedAmount:
|
| 90 |
+
return ParsedAmount(
|
| 91 |
+
amount=amount.amount_float,
|
| 92 |
+
amount_text=amount.amount_text,
|
| 93 |
+
raw=amount.raw,
|
| 94 |
+
currency=amount.currency,
|
| 95 |
+
currency_code=amount.currency_code,
|
| 96 |
+
confidence=amount.confidence,
|
| 97 |
+
is_negative=amount.is_negative,
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def _run_parse(text: str, limit, hint, default_currency):
|
| 102 |
+
"""CPU-bound parse work, executed on the shared thread pool."""
|
| 103 |
+
amounts = extract_amounts(text, limit=limit, currency_hint=hint)
|
| 104 |
+
# Only candidates that clear the confidence floor are real amounts;
|
| 105 |
+
# reject-word/date/quantity leftovers (confidence 0) must not count.
|
| 106 |
+
amounts = [a for a in amounts if a.confidence >= CONFIDENCE_FLOOR]
|
| 107 |
+
best = parse_amount(text, currency_hint=hint, default_currency=default_currency)
|
| 108 |
+
return amounts, best
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def _error_response(start: float, text: str, message: str) -> PriceParseItemResponse:
|
| 112 |
+
return PriceParseItemResponse(
|
| 113 |
+
success=False,
|
| 114 |
+
time_ms=round((time.perf_counter() - start) * 1000, 3),
|
| 115 |
+
text=text,
|
| 116 |
+
count=0,
|
| 117 |
+
amounts=[],
|
| 118 |
+
best=None,
|
| 119 |
+
error_message=message,
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
async def _parse_one(index: int, body: PriceParseRequest) -> PriceParseItemResponse:
|
| 124 |
+
start = time.perf_counter()
|
| 125 |
+
try:
|
| 126 |
+
loop = asyncio.get_running_loop()
|
| 127 |
+
amounts, best = await loop.run_in_executor(
|
| 128 |
+
thread_pool,
|
| 129 |
+
_run_parse,
|
| 130 |
+
body.text,
|
| 131 |
+
body.limit,
|
| 132 |
+
body.currency_hint,
|
| 133 |
+
body.default_currency,
|
| 134 |
+
)
|
| 135 |
+
except ValueError as exc:
|
| 136 |
+
return _error_response(start, body.text, str(exc))
|
| 137 |
+
except Exception as exc: # pragma: no cover - defensive
|
| 138 |
+
logger.exception("price_parser item %s failed", index)
|
| 139 |
+
return _error_response(start, body.text, f"parsing failed: {exc}")
|
| 140 |
+
|
| 141 |
+
elapsed = round((time.perf_counter() - start) * 1000, 3)
|
| 142 |
+
logger.info(
|
| 143 |
+
"price_parser item parsed",
|
| 144 |
+
extra={"index": index, "count": len(amounts), "time_ms": elapsed},
|
| 145 |
+
)
|
| 146 |
+
return PriceParseItemResponse(
|
| 147 |
+
success=True,
|
| 148 |
+
time_ms=elapsed,
|
| 149 |
+
text=body.text,
|
| 150 |
+
count=len(amounts),
|
| 151 |
+
amounts=[_to_parsed(a) for a in amounts],
|
| 152 |
+
best=_to_parsed(best) if best is not None else None,
|
| 153 |
+
error_message=None,
|
| 154 |
+
)
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
@router.post(
|
| 158 |
+
"/price-parser",
|
| 159 |
+
response_model=List[PriceParseItemResponse],
|
| 160 |
+
summary="Parse monetary amounts from dirty text fields (batch of up to 100, processed concurrently)",
|
| 161 |
+
description=(
|
| 162 |
+
"Send up to 100 raw text fields as a JSON array (1-100 items). Each item "
|
| 163 |
+
"returns every candidate amount found (ranked by confidence) plus the "
|
| 164 |
+
"single best one, with currency, confidence and the matched substrings. "
|
| 165 |
+
"Supports US/EU/Indian number formats, OCR distortion ('$iom89.00', "
|
| 166 |
+
"'896,009.0 0'), magnitudes ('₹5 Cr', '1.5M'), negatives and "
|
| 167 |
+
"currency hints. Fields with only dates/quantities/references return "
|
| 168 |
+
"count=0 with an empty amounts array. Parsing is exact (Decimal) and "
|
| 169 |
+
"dependency-free."
|
| 170 |
+
),
|
| 171 |
+
)
|
| 172 |
+
async def price_parser_batch(
|
| 173 |
+
body: Annotated[
|
| 174 |
+
List[PriceParseRequest],
|
| 175 |
+
Field(
|
| 176 |
+
min_length=1,
|
| 177 |
+
max_length=MAX_BATCH_SIZE,
|
| 178 |
+
description="Array of 1-100 text fields to parse, processed concurrently.",
|
| 179 |
+
),
|
| 180 |
+
],
|
| 181 |
+
) -> List[PriceParseItemResponse]:
|
| 182 |
+
start = time.perf_counter()
|
| 183 |
+
logger.info("price_parser_batch | start count=%d", len(body))
|
| 184 |
+
results = await asyncio.gather(*[_parse_one(index, item) for index, item in enumerate(body)])
|
| 185 |
+
elapsed = round((time.perf_counter() - start) * 1000, 3)
|
| 186 |
+
succeeded = sum(1 for r in results if r.success)
|
| 187 |
+
logger.info(
|
| 188 |
+
"price_parser_batch | done total=%d succeeded=%d time_ms=%s",
|
| 189 |
+
len(results),
|
| 190 |
+
succeeded,
|
| 191 |
+
elapsed,
|
| 192 |
+
)
|
| 193 |
+
return results
|
app/api/v1/router.py
CHANGED
|
@@ -18,6 +18,7 @@ from app.api.v1 import (
|
|
| 18 |
json_extract,
|
| 19 |
keys_extract,
|
| 20 |
media_convert,
|
|
|
|
| 21 |
qr_decoder,
|
| 22 |
qr_generator,
|
| 23 |
reconcile,
|
|
@@ -75,6 +76,7 @@ _include(gmail.router, "google:gmail", tags=["Gmail"])
|
|
| 75 |
_include(sheets.router, "google:sheets_api", tags=["Google Sheets"])
|
| 76 |
_include(media_convert.router, "media_convert", tags=["Media-to-Media Conversion"])
|
| 77 |
_include(json_extract.router, "json_extract", tags=["JSON Extractor"])
|
|
|
|
| 78 |
_include(keys_extract.router, "keys_extract", prefix="/json", tags=["Keys Extractor"])
|
| 79 |
_include(qr_decoder.router, "qr_decoder", tags=["QR Decoder"])
|
| 80 |
_include(qr_generator.router, "qr_generator", tags=["QR Generator"])
|
|
|
|
| 18 |
json_extract,
|
| 19 |
keys_extract,
|
| 20 |
media_convert,
|
| 21 |
+
price_parser,
|
| 22 |
qr_decoder,
|
| 23 |
qr_generator,
|
| 24 |
reconcile,
|
|
|
|
| 76 |
_include(sheets.router, "google:sheets_api", tags=["Google Sheets"])
|
| 77 |
_include(media_convert.router, "media_convert", tags=["Media-to-Media Conversion"])
|
| 78 |
_include(json_extract.router, "json_extract", tags=["JSON Extractor"])
|
| 79 |
+
_include(price_parser.router, "price_parser", tags=["Price Parser"])
|
| 80 |
_include(keys_extract.router, "keys_extract", prefix="/json", tags=["Keys Extractor"])
|
| 81 |
_include(qr_decoder.router, "qr_decoder", tags=["QR Decoder"])
|
| 82 |
_include(qr_generator.router, "qr_generator", tags=["QR Generator"])
|
app/services/monetary_field_service.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Monetary-field detection for feature-extraction results.
|
| 2 |
+
|
| 3 |
+
Given the ``data`` payload of a JSON / feature-extraction response, locate the
|
| 4 |
+
values of known monetary field names (``total``, ``subtotal``, ``cgst``,
|
| 5 |
+
``balance`` ...) at any nesting depth using the recursive JSON key finder
|
| 6 |
+
(:mod:`app.services.keys_extractor_service`), then normalise each found value
|
| 7 |
+
with the amount parser (:mod:`app.services.price_parser`).
|
| 8 |
+
|
| 9 |
+
Usage
|
| 10 |
+
-----
|
| 11 |
+
>>> from app.services.monetary_field_service import find_monetary_fields
|
| 12 |
+
>>> data = {"invoice": [{"total": "INR 572,300.00", "vendor": "Acme"}]}
|
| 13 |
+
>>> find_monetary_fields(data)
|
| 14 |
+
[MonetaryField(key='total', raw='INR 572,300.00', amount=572300.0,
|
| 15 |
+
currency='INR', currency_code='INR', confidence=1.0)]
|
| 16 |
+
|
| 17 |
+
When no monetary field is found (or none of the found values parse as an
|
| 18 |
+
amount) the result is an empty list — callers should treat that as
|
| 19 |
+
"no monetary fields found".
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
from __future__ import annotations
|
| 23 |
+
|
| 24 |
+
from typing import Any, List, Optional
|
| 25 |
+
|
| 26 |
+
from pydantic import BaseModel, Field
|
| 27 |
+
|
| 28 |
+
from app.services.keys_extractor_service import KeysExtractor
|
| 29 |
+
from app.services.price_parser import parse_amount
|
| 30 |
+
|
| 31 |
+
#: Field names that typically hold monetary values, in priority order.
|
| 32 |
+
#: Matched by exact key name at any depth (e.g. ``data["invoice"][0]["total"]``).
|
| 33 |
+
DEFAULT_MONETARY_KEYS: List[str] = [
|
| 34 |
+
"grand_total",
|
| 35 |
+
"total_amount",
|
| 36 |
+
"amount_due",
|
| 37 |
+
"total",
|
| 38 |
+
"subtotal",
|
| 39 |
+
"sub_total",
|
| 40 |
+
"balance",
|
| 41 |
+
"payable",
|
| 42 |
+
"amount",
|
| 43 |
+
"price",
|
| 44 |
+
"unit_price",
|
| 45 |
+
"unitprice",
|
| 46 |
+
"rate",
|
| 47 |
+
"cgst",
|
| 48 |
+
"sgst",
|
| 49 |
+
"igst",
|
| 50 |
+
"total_tax",
|
| 51 |
+
"tax",
|
| 52 |
+
"vat",
|
| 53 |
+
"gst",
|
| 54 |
+
"discount",
|
| 55 |
+
"shipping",
|
| 56 |
+
"freight",
|
| 57 |
+
"fee",
|
| 58 |
+
"deposit",
|
| 59 |
+
"paid",
|
| 60 |
+
"due",
|
| 61 |
+
"net",
|
| 62 |
+
"gross",
|
| 63 |
+
"value",
|
| 64 |
+
"sum",
|
| 65 |
+
]
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
class MonetaryField(BaseModel):
|
| 69 |
+
"""One parsed monetary field found inside an extraction result."""
|
| 70 |
+
|
| 71 |
+
key: str
|
| 72 |
+
"""JSON key name the value was found under (exact match, any depth)."""
|
| 73 |
+
raw: Optional[str] = None
|
| 74 |
+
"""The value as it appeared in the extraction result (string form)."""
|
| 75 |
+
amount: Optional[float] = None
|
| 76 |
+
"""Normalised numeric amount, or ``None`` when the value did not parse."""
|
| 77 |
+
currency: Optional[str] = None
|
| 78 |
+
"""Currency marker found in the value (symbol, code or word)."""
|
| 79 |
+
currency_code: Optional[str] = None
|
| 80 |
+
"""ISO 4217 code when determinable, else ``None``."""
|
| 81 |
+
confidence: float = Field(default=0.0, ge=0.0, le=1.0)
|
| 82 |
+
"""Parser confidence in [0, 1] — higher is more likely a real amount."""
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def find_monetary_fields(
|
| 86 |
+
data: Any,
|
| 87 |
+
key_names: Optional[List[str]] = None,
|
| 88 |
+
) -> List[MonetaryField]:
|
| 89 |
+
"""Return parsed monetary fields found in ``data`` (empty list if none).
|
| 90 |
+
|
| 91 |
+
``data`` may be a ``dict`` or a ``list`` (any nesting depth). Only values
|
| 92 |
+
that parse as amounts are returned; found-but-unparseable values are
|
| 93 |
+
skipped. Invalid ``data`` types simply yield an empty list.
|
| 94 |
+
"""
|
| 95 |
+
keys = key_names or DEFAULT_MONETARY_KEYS
|
| 96 |
+
if not isinstance(data, (dict, list)):
|
| 97 |
+
return []
|
| 98 |
+
|
| 99 |
+
try:
|
| 100 |
+
extractor = KeysExtractor(data, key_names=keys, result_limit=None)
|
| 101 |
+
found = extractor.extract()
|
| 102 |
+
except (TypeError, ValueError):
|
| 103 |
+
return []
|
| 104 |
+
|
| 105 |
+
results: List[MonetaryField] = []
|
| 106 |
+
for key in keys:
|
| 107 |
+
for value in found.get(key, []):
|
| 108 |
+
if value is None or isinstance(value, bool):
|
| 109 |
+
continue
|
| 110 |
+
parsed = parse_amount(value)
|
| 111 |
+
if parsed is None:
|
| 112 |
+
continue
|
| 113 |
+
raw = parsed.raw if isinstance(value, str) else str(value)
|
| 114 |
+
results.append(
|
| 115 |
+
MonetaryField(
|
| 116 |
+
key=key,
|
| 117 |
+
raw=raw,
|
| 118 |
+
amount=parsed.amount_float,
|
| 119 |
+
currency=parsed.currency,
|
| 120 |
+
currency_code=parsed.currency_code,
|
| 121 |
+
confidence=parsed.confidence,
|
| 122 |
+
)
|
| 123 |
+
)
|
| 124 |
+
return results
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
__all__ = ["MonetaryField", "find_monetary_fields", "DEFAULT_MONETARY_KEYS"]
|
app/services/price_parser.py
ADDED
|
@@ -0,0 +1,814 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Enterprise-grade amount / price parser (pure stdlib, zero dependencies).
|
| 2 |
+
|
| 3 |
+
Extracts a monetary amount from noisy or distorted text fields: OCR output,
|
| 4 |
+
scraped HTML, invoice lines, emails, database columns, etc.
|
| 5 |
+
|
| 6 |
+
Design goals
|
| 7 |
+
------------
|
| 8 |
+
* **Pure and dependency-free** — stdlib only (``re``, ``decimal``, ``bisect``,
|
| 9 |
+
``dataclasses``). No external AI or parsing libraries.
|
| 10 |
+
* **Deterministic and explainable** — every candidate carries a confidence in
|
| 11 |
+
[0, 1] so callers can set their own acceptance threshold.
|
| 12 |
+
* **Locale-aware separators**:
|
| 13 |
+
- US: ``1,234.56``
|
| 14 |
+
- EU: ``22,90 €``, ``1.234,56``, ``1.234.567,89``
|
| 15 |
+
- Indian: ``1,50,087.99`` (lakh grouping), ``₹5 Cr``
|
| 16 |
+
- Space: ``15 130 Р``, ``75 990,00 Kč``
|
| 17 |
+
* **Distortion tolerance** (OCR / data-entry noise):
|
| 18 |
+
- Junk between currency and digits: ``$iom89.00`` -> ``89.00``
|
| 19 |
+
- Stray letters inside a number: ``8a9.00`` -> ``89.00``, ``1i0,000`` -> ``10000``
|
| 20 |
+
* **Negatives**: ``-1,234.56``, ``1,234.56-``, unicode minus ``−89.00``,
|
| 21 |
+
accounting parentheses ``(78,000)``.
|
| 22 |
+
* **Magnitude suffixes**: ``K``/``k`` (thousand), ``M`` (million), ``B``,
|
| 23 |
+
``T``, ``Cr``/``crore`` (10M), ``L``/``lakh``/``lac`` (100k).
|
| 24 |
+
* **Currency detection**: symbols (``$ € £ ¥ ₹ ...``), ISO codes (``USD``,
|
| 25 |
+
``INR``), short forms (``US$``, ``R$``, ``Rp``, ``Rs``, ``Kč`` ...) and
|
| 26 |
+
words (``dollars``, ``rupees`` ...), on either side of the number.
|
| 27 |
+
* **Anti-pattern rejection** — dates, times, phone numbers, IPs, reference
|
| 28 |
+
numbers, percentages, versions and quantities are filtered or penalised so
|
| 29 |
+
they never win over a real amount.
|
| 30 |
+
* **Exact arithmetic** — values are ``decimal.Decimal``, never binary floats.
|
| 31 |
+
|
| 32 |
+
Quick start
|
| 33 |
+
-----------
|
| 34 |
+
>>> from app.services.price_parser import parse_amount, extract_amounts
|
| 35 |
+
>>> parse_amount("$iom89.00")
|
| 36 |
+
Amount(amount=Decimal('89.00'), currency='$', currency_code=None, ...)
|
| 37 |
+
>>> parse_amount("USD 78,000").amount
|
| 38 |
+
Decimal('78000')
|
| 39 |
+
>>> parse_amount("1,50,087.99").amount # Indian lakh grouping
|
| 40 |
+
Decimal('150087.99')
|
| 41 |
+
>>> parse_amount("Page 3 of 10") is None
|
| 42 |
+
True
|
| 43 |
+
|
| 44 |
+
NOTE: this module intentionally does NOT vendor any third-party parser (see
|
| 45 |
+
``price-parser``, ``number-parser``, ``money-parser`` on PyPI) because none of
|
| 46 |
+
them combine OCR-junk tolerance, Indian lakh/crore grouping, magnitude
|
| 47 |
+
suffixes, negatives and multi-candidate disambiguation in a single
|
| 48 |
+
dependency-free implementation.
|
| 49 |
+
"""
|
| 50 |
+
|
| 51 |
+
from __future__ import annotations
|
| 52 |
+
|
| 53 |
+
import re
|
| 54 |
+
from bisect import bisect_right
|
| 55 |
+
from dataclasses import dataclass
|
| 56 |
+
from decimal import Decimal
|
| 57 |
+
from typing import Iterator, List, Optional
|
| 58 |
+
|
| 59 |
+
MAX_INPUT_LENGTH = 1_000_000
|
| 60 |
+
"""Hard cap on input size (characters) to bound worst-case work."""
|
| 61 |
+
|
| 62 |
+
CONFIDENCE_FLOOR = 0.20
|
| 63 |
+
"""Minimum confidence for a candidate to count as a real amount.
|
| 64 |
+
|
| 65 |
+
``parse_amount`` returns ``None`` when the best candidate scores below this,
|
| 66 |
+
and API consumers use it to decide which candidates to report.
|
| 67 |
+
"""
|
| 68 |
+
|
| 69 |
+
_CONFIDENCE_FLOOR = CONFIDENCE_FLOOR
|
| 70 |
+
|
| 71 |
+
# ---------------------------------------------------------------------------
|
| 72 |
+
# Currency tables
|
| 73 |
+
# ---------------------------------------------------------------------------
|
| 74 |
+
|
| 75 |
+
#: Symbol -> ISO 4217 code when the symbol is unambiguous.
|
| 76 |
+
_SYMBOL_TO_ISO = {
|
| 77 |
+
"€": "EUR",
|
| 78 |
+
"£": "GBP",
|
| 79 |
+
"₹": "INR",
|
| 80 |
+
"₩": "KRW",
|
| 81 |
+
"₽": "RUB",
|
| 82 |
+
"฿": "THB",
|
| 83 |
+
"₫": "VND",
|
| 84 |
+
"₦": "NGN",
|
| 85 |
+
"₱": "PHP",
|
| 86 |
+
"₴": "UAH",
|
| 87 |
+
"₪": "ILS",
|
| 88 |
+
"₸": "KZT",
|
| 89 |
+
"₲": "PYG",
|
| 90 |
+
"₡": "CRC",
|
| 91 |
+
"₾": "GEL",
|
| 92 |
+
"৳": "BDT",
|
| 93 |
+
"₵": "GHS",
|
| 94 |
+
"₼": "AZN",
|
| 95 |
+
"៛": "KHR",
|
| 96 |
+
"₭": "LAK",
|
| 97 |
+
"₮": "MNT",
|
| 98 |
+
"₺": "TRY",
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
#: Symbols that are ambiguous without extra context (no fixed ISO code).
|
| 102 |
+
_AMBIGUOUS_SYMBOLS = frozenset("$¥₨")
|
| 103 |
+
|
| 104 |
+
#: Common ISO 4217 codes we recognise when written as 3-letter codes.
|
| 105 |
+
_ISO_CODES = frozenset(
|
| 106 |
+
{
|
| 107 |
+
"USD", "EUR", "GBP", "INR", "JPY", "CNY", "AUD", "CAD", "CHF", "SGD",
|
| 108 |
+
"HKD", "NZD", "SEK", "NOK", "DKK", "PLN", "CZK", "HUF", "RON", "BGN",
|
| 109 |
+
"HRK", "RUB", "TRY", "ZAR", "BRL", "MXN", "ARS", "CLP", "COP", "PEN",
|
| 110 |
+
"UYU", "VES", "IDR", "MYR", "PHP", "THB", "VND", "KRW", "TWD", "AED",
|
| 111 |
+
"SAR", "QAR", "KWD", "BHD", "OMR", "JOD", "LBP", "IQD", "IRR", "EGP",
|
| 112 |
+
"NGN", "KES", "GHS", "ZMW", "TZS", "UGX", "MAD", "TND", "DZD", "PKR",
|
| 113 |
+
"BDT", "LKR", "NPR", "MMK", "KZT", "UZS", "AZN", "GEL", "AMD", "BYN",
|
| 114 |
+
"MNT", "KHR", "LAK", "MOP", "BND", "XOF", "XAF", "MUR", "MVR", "MWK",
|
| 115 |
+
"MZN", "NAD", "BWP", "GMD", "GNF", "HTG", "ISK", "JMD", "KMF", "LSL",
|
| 116 |
+
"LYD", "MGA", "MKD", "PGK", "RSD", "SOS", "SRD", "SZL", "TOP", "TTD",
|
| 117 |
+
"WST", "XCD", "FJD", "ALL", "BAM", "ETB", "GIP", "GYD", "KGS", "KPW",
|
| 118 |
+
"MRO", "SCR", "SYP", "TJS", "TMT", "VUV", "YER",
|
| 119 |
+
}
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
#: Short/compound currency forms -> ISO code (None when still ambiguous).
|
| 123 |
+
_SHORT_TO_ISO = {
|
| 124 |
+
"US$": "USD",
|
| 125 |
+
"R$": "BRL",
|
| 126 |
+
"Rp": "IDR",
|
| 127 |
+
"Rs": "INR",
|
| 128 |
+
"Re": "INR",
|
| 129 |
+
"Kč": "CZK",
|
| 130 |
+
"zł": "PLN",
|
| 131 |
+
"Ft": "HUF",
|
| 132 |
+
"руб": "RUB",
|
| 133 |
+
"р.": "RUB",
|
| 134 |
+
"lei": "RON",
|
| 135 |
+
"лв": "BGN",
|
| 136 |
+
"S/": "PEN",
|
| 137 |
+
"kr": None,
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
#: Currency words -> ISO code (None when ambiguous).
|
| 141 |
+
_WORD_TO_ISO = {
|
| 142 |
+
"dollars": "USD", "dollar": "USD", "usd": "USD",
|
| 143 |
+
"euros": "EUR", "euro": "EUR",
|
| 144 |
+
"pounds": "GBP", "pound": "GBP",
|
| 145 |
+
"rupees": "INR", "rupee": "INR", "inr": "INR",
|
| 146 |
+
"yen": "JPY", "yuan": "CNY", "rand": "ZAR",
|
| 147 |
+
"reais": "BRL", "real": "BRL",
|
| 148 |
+
"pesos": None, "peso": None,
|
| 149 |
+
"won": "KRW", "baht": "THB", "dong": "VND", "naira": "NGN",
|
| 150 |
+
"ringgit": "MYR", "rupiah": "IDR", "zloty": "PLN", "forint": "HUF",
|
| 151 |
+
"leu": "RON", "lev": "BGN", "dirham": "AED", "riyal": "SAR",
|
| 152 |
+
"lari": "GEL", "tenge": "KZT", "hryvnia": "UAH", "shekel": "ILS",
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
+
# ---------------------------------------------------------------------------
|
| 156 |
+
# Amount-indicating and non-amount word lists (scored context)
|
| 157 |
+
# ---------------------------------------------------------------------------
|
| 158 |
+
|
| 159 |
+
_STRONG_AMOUNT_RE = re.compile(
|
| 160 |
+
r"\b(?:grand total|total|payable|balance)\b", re.IGNORECASE
|
| 161 |
+
)
|
| 162 |
+
_MEDIUM_AMOUNT_RE = re.compile(
|
| 163 |
+
r"\b(?:subtotal|sub-total|amount|price|charge|cost|value|sum|fee|deposit|"
|
| 164 |
+
r"paid|received|amt|due|net|gross)\b",
|
| 165 |
+
re.IGNORECASE,
|
| 166 |
+
)
|
| 167 |
+
_REJECT_WORD_RE = re.compile(
|
| 168 |
+
r"\b(?:qty|quantity|page|no\.?|number|ref\.?|date|tel|phone|mobile|fax|"
|
| 169 |
+
r"pin|zip|gst|pan|ifsc|swift|vat|tin|weight|height|width|length|depth|"
|
| 170 |
+
r"pcs|unit|units|kg|gms?|ml|ltr|est\.?|reg\.?|sr\.?|sl\.?|po\b|a/c|acct|"
|
| 171 |
+
r"account|contact|inv\.?)\b",
|
| 172 |
+
re.IGNORECASE,
|
| 173 |
+
)
|
| 174 |
+
_OF_RE = re.compile(r"\bof\b", re.IGNORECASE)
|
| 175 |
+
|
| 176 |
+
_FREE_RE = re.compile(r"(?i)^\s*(?:free|complimentary|n/?a|na|no charge|no cost|zero)\s*$")
|
| 177 |
+
|
| 178 |
+
# ---------------------------------------------------------------------------
|
| 179 |
+
# Tokenisation
|
| 180 |
+
# ---------------------------------------------------------------------------
|
| 181 |
+
|
| 182 |
+
#: A digit run with optional grouping/separator characters.
|
| 183 |
+
#: The space-grouped alternative ("15 130", "1 234.56") is tried first so a
|
| 184 |
+
#: bare "500.00 600.00" still tokenises as two separate numbers.
|
| 185 |
+
_NUMBER_TOKEN_RE = re.compile(
|
| 186 |
+
r"(?<![0-9.,])(?:"
|
| 187 |
+
r"[0-9]{1,3}(?: [0-9]{3})+(?:[.,][0-9]{1,2})?|" # space-grouped thousands
|
| 188 |
+
r"[0-9][0-9.,]*[0-9]|" # grouped/plain with dots/commas
|
| 189 |
+
r"[0-9]+|" # single digit
|
| 190 |
+
r"\.[0-9]+" # ".99"
|
| 191 |
+
r")"
|
| 192 |
+
)
|
| 193 |
+
|
| 194 |
+
#: Patterns whose matches mark whole regions that must never yield an amount.
|
| 195 |
+
_DATE_RE = re.compile(r"\b\d{1,4}[-/.]\d{1,2}[-/.]\d{1,4}\b")
|
| 196 |
+
_TIME_RE = re.compile(r"\b\d{1,2}:\d{2}(?::\d{2})?\b")
|
| 197 |
+
_IP_RE = re.compile(r"\b\d{1,3}(?:\.\d{1,3}){3}\b")
|
| 198 |
+
_ID_RE = re.compile(r"\b\d{3,4}(?:[- ]\d{3,4}){2,}\b")
|
| 199 |
+
_PHONE_RE = re.compile(r"\+[\d\s()\-]{7,}\d")
|
| 200 |
+
_AREA_CODE_RE = re.compile(r"\(\d{3}\)")
|
| 201 |
+
_SCI_RE = re.compile(r"[eE][+-]?\d+")
|
| 202 |
+
|
| 203 |
+
_GROUPED_COMMA_WEST = re.compile(r"\d{1,3}(?:,\d{3})+")
|
| 204 |
+
_GROUPED_COMMA_IND = re.compile(r"\d{1,2}(?:,\d{2})*,\d{3}")
|
| 205 |
+
_GROUPED_DOT = re.compile(r"\d{1,3}(?:\.\d{3})+")
|
| 206 |
+
|
| 207 |
+
_CURRENCY_FINDER = re.compile(
|
| 208 |
+
r"US\$|R\$|Rp\.?|Rs\.?|Re\.?|Kč|zł|Ft|руб|р\.|lei|лв|S/|"
|
| 209 |
+
r"[$€£¥₹₩₽฿₫₦₱₴₪₸₲₡₾৳₵₼៛₭₮₺₨₧]|"
|
| 210 |
+
r"\b(?:dollars?|euros?|pounds?|rupees?|yen|yuan|rand|reais|real|pesos?|"
|
| 211 |
+
r"won|baht|dong|naira|ringgit|rupiah|zloty|forint|leu|lev|dirham|riyal|"
|
| 212 |
+
r"lari|tenge|hryvnia|shekel)\b|"
|
| 213 |
+
r"[A-Za-z]{3}"
|
| 214 |
+
)
|
| 215 |
+
|
| 216 |
+
# Currency / magnitude scan windows (chars). Generous for OCR junk.
|
| 217 |
+
_CURRENCY_WINDOW = 14
|
| 218 |
+
_CURRENCY_GAP_TOLERANCE = 10
|
| 219 |
+
_WORD_WINDOW_BEFORE = 10
|
| 220 |
+
_WORD_WINDOW_AFTER = 12
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
@dataclass(frozen=True)
|
| 224 |
+
class Amount:
|
| 225 |
+
"""A single extracted monetary amount with its context and confidence."""
|
| 226 |
+
|
| 227 |
+
amount: Decimal
|
| 228 |
+
"""Numeric value (always ``Decimal`` — never a binary float)."""
|
| 229 |
+
currency: Optional[str]
|
| 230 |
+
"""Currency marker as found in the text (symbol, code or word), or hint/default."""
|
| 231 |
+
currency_code: Optional[str]
|
| 232 |
+
"""ISO 4217 code when determinable, else ``None`` (e.g. bare ``$``)."""
|
| 233 |
+
amount_text: str
|
| 234 |
+
"""The number part as it appeared (junk letters removed, separators kept)."""
|
| 235 |
+
raw: str
|
| 236 |
+
"""Full matched substring: currency marker + sign + number + magnitude suffix."""
|
| 237 |
+
confidence: float
|
| 238 |
+
"""Heuristic confidence in [0, 1] — higher is more likely a real amount."""
|
| 239 |
+
is_negative: bool = False
|
| 240 |
+
"""True when a leading/trailing minus or accounting parentheses were found."""
|
| 241 |
+
position: int = 0
|
| 242 |
+
"""Character offset of the digit token in the source text."""
|
| 243 |
+
|
| 244 |
+
@property
|
| 245 |
+
def amount_float(self) -> float:
|
| 246 |
+
"""Float convenience view of ``amount`` (prefer ``amount`` for money math)."""
|
| 247 |
+
return float(self.amount)
|
| 248 |
+
|
| 249 |
+
@property
|
| 250 |
+
def is_zero(self) -> bool:
|
| 251 |
+
return self.amount == 0
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
# ---------------------------------------------------------------------------
|
| 255 |
+
# Number-body interpretation (separator disambiguation)
|
| 256 |
+
# ---------------------------------------------------------------------------
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def _clean_grouped_int(int_part: str, sep: str) -> Optional[str]:
|
| 260 |
+
"""Validate a grouped integer part; return clean digits or ``None``.
|
| 261 |
+
|
| 262 |
+
``sep`` is the grouping separator actually used (``","`` or ``"."``).
|
| 263 |
+
Accepts Western (``1,234``), Indian (``1,50,087``) and dot-grouped
|
| 264 |
+
(``1.234``) conventions.
|
| 265 |
+
"""
|
| 266 |
+
if int_part.isdigit():
|
| 267 |
+
return int_part
|
| 268 |
+
if sep == ",":
|
| 269 |
+
if _GROUPED_COMMA_WEST.fullmatch(int_part) or _GROUPED_COMMA_IND.fullmatch(int_part):
|
| 270 |
+
return int_part.replace(",", "")
|
| 271 |
+
else:
|
| 272 |
+
if _GROUPED_DOT.fullmatch(int_part):
|
| 273 |
+
return int_part.replace(".", "")
|
| 274 |
+
return None
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
def _frac_decimal(int_clean: str, frac: str) -> Decimal:
|
| 278 |
+
return Decimal(f"{int_clean}.{frac}") if frac else Decimal(int_clean)
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
def _parse_decimal(body: str) -> Optional[Decimal]:
|
| 282 |
+
"""Interpret a raw digit/separator token as a ``Decimal``, or ``None``.
|
| 283 |
+
|
| 284 |
+
Rules (mirroring the well-tested ``price-parser`` heuristics, extended for
|
| 285 |
+
Indian lakh/crore grouping):
|
| 286 |
+
|
| 287 |
+
* both separators present -> the *last* one is the decimal separator;
|
| 288 |
+
* a single separator whose trailing group is 1-2 digits -> decimal
|
| 289 |
+
(``22,90`` -> 22.90, ``78.90`` -> 78.90);
|
| 290 |
+
* a single separator whose trailing group is exactly 3 digits -> thousands
|
| 291 |
+
(``78,000`` -> 78000, ``1.234.567`` -> 1234567);
|
| 292 |
+
* invalid grouping (``1,23,45``, ``1.2.3``, ``12 34``) -> ``None``.
|
| 293 |
+
"""
|
| 294 |
+
body = body.rstrip(".,").strip()
|
| 295 |
+
if not body:
|
| 296 |
+
return None
|
| 297 |
+
if not re.fullmatch(r"[0-9 .,]+", body):
|
| 298 |
+
return None
|
| 299 |
+
if body.count(",") + body.count(".") > 6:
|
| 300 |
+
return None
|
| 301 |
+
|
| 302 |
+
has_comma = "," in body
|
| 303 |
+
has_dot = "." in body
|
| 304 |
+
has_space = " " in body
|
| 305 |
+
|
| 306 |
+
if has_space:
|
| 307 |
+
if has_comma or has_dot:
|
| 308 |
+
body = body.replace(" ", "")
|
| 309 |
+
has_comma = "," in body
|
| 310 |
+
has_dot = "." in body
|
| 311 |
+
else:
|
| 312 |
+
groups = body.split(" ")
|
| 313 |
+
if (
|
| 314 |
+
len(groups) >= 2
|
| 315 |
+
and all(len(g) == 3 for g in groups[1:])
|
| 316 |
+
and 1 <= len(groups[0]) <= 3
|
| 317 |
+
):
|
| 318 |
+
return Decimal("".join(groups))
|
| 319 |
+
return None
|
| 320 |
+
|
| 321 |
+
if has_comma and has_dot:
|
| 322 |
+
dec = body[max(body.rfind(","), body.rfind("."))]
|
| 323 |
+
int_part, _, frac = body.rpartition(dec)
|
| 324 |
+
if not (1 <= len(frac) <= 2 and frac.isdigit()):
|
| 325 |
+
return None
|
| 326 |
+
clean = _clean_grouped_int(int_part, "," if dec == "." else ".")
|
| 327 |
+
if clean is None:
|
| 328 |
+
return None
|
| 329 |
+
return _frac_decimal(clean, frac)
|
| 330 |
+
|
| 331 |
+
if has_comma:
|
| 332 |
+
return _parse_single_sep(body, ",")
|
| 333 |
+
|
| 334 |
+
if has_dot:
|
| 335 |
+
return _parse_single_sep(body, ".")
|
| 336 |
+
|
| 337 |
+
return Decimal(body)
|
| 338 |
+
|
| 339 |
+
|
| 340 |
+
def _parse_single_sep(body: str, sep: str) -> Optional[Decimal]:
|
| 341 |
+
if body.startswith(sep):
|
| 342 |
+
frac = body[1:]
|
| 343 |
+
if 1 <= len(frac) <= 2 and frac.isdigit():
|
| 344 |
+
return Decimal(f"0.{frac}")
|
| 345 |
+
return None
|
| 346 |
+
groups = body.split(sep)
|
| 347 |
+
trailing = groups[-1]
|
| 348 |
+
if 1 <= len(trailing) <= 2:
|
| 349 |
+
int_part = sep.join(groups[:-1])
|
| 350 |
+
if int_part.isdigit():
|
| 351 |
+
return _frac_decimal(int_part, trailing)
|
| 352 |
+
clean = _clean_grouped_int(int_part, sep)
|
| 353 |
+
if clean is not None:
|
| 354 |
+
return _frac_decimal(clean, trailing)
|
| 355 |
+
return None
|
| 356 |
+
if len(trailing) == 3:
|
| 357 |
+
clean = _clean_grouped_int(body, sep)
|
| 358 |
+
return Decimal(clean) if clean is not None else None
|
| 359 |
+
return None
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
# ---------------------------------------------------------------------------
|
| 363 |
+
# Context helpers
|
| 364 |
+
# ---------------------------------------------------------------------------
|
| 365 |
+
|
| 366 |
+
|
| 367 |
+
def _resolve_currency(raw: str) -> tuple[Optional[str], Optional[str]]:
|
| 368 |
+
"""Map a raw currency token to ``(display, iso_code)``; unknown -> (None, None)."""
|
| 369 |
+
r = raw.strip()
|
| 370 |
+
if r in _SYMBOL_TO_ISO:
|
| 371 |
+
return r, _SYMBOL_TO_ISO[r]
|
| 372 |
+
if r in _AMBIGUOUS_SYMBOLS:
|
| 373 |
+
return r, None
|
| 374 |
+
up = r.upper()
|
| 375 |
+
if up in _ISO_CODES:
|
| 376 |
+
return r, up
|
| 377 |
+
if r in _SHORT_TO_ISO:
|
| 378 |
+
return r, _SHORT_TO_ISO[r]
|
| 379 |
+
low = r.lower()
|
| 380 |
+
if low in _WORD_TO_ISO:
|
| 381 |
+
return low, _WORD_TO_ISO[low]
|
| 382 |
+
return None, None
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
def _nearest_currency(
|
| 386 |
+
text: str, raw_start: int, raw_end: int
|
| 387 |
+
) -> tuple[Optional[str], Optional[str], Optional[int], Optional[int], Optional[int]]:
|
| 388 |
+
"""Find the currency marker nearest to the number span.
|
| 389 |
+
|
| 390 |
+
Returns ``(currency, iso_code, gap, abs_start, abs_end)`` or all-``None``.
|
| 391 |
+
``gap`` is the number of characters between the marker and the number;
|
| 392 |
+
OCR junk between them is tolerated up to ``_CURRENCY_GAP_TOLERANCE``.
|
| 393 |
+
"""
|
| 394 |
+
prefix = text[max(0, raw_start - _CURRENCY_WINDOW):raw_start]
|
| 395 |
+
suffix = text[raw_end:raw_end + _CURRENCY_WINDOW]
|
| 396 |
+
best = None # (currency, code, gap, abs_start, abs_end)
|
| 397 |
+
|
| 398 |
+
for m in _CURRENCY_FINDER.finditer(prefix):
|
| 399 |
+
cur, code = _resolve_currency(m.group(0))
|
| 400 |
+
if cur is None:
|
| 401 |
+
continue
|
| 402 |
+
gap = len(prefix) - m.end()
|
| 403 |
+
if gap <= _CURRENCY_GAP_TOLERANCE and (best is None or gap < best[2]):
|
| 404 |
+
base = raw_start - len(prefix)
|
| 405 |
+
best = (cur, code, gap, base + m.start(), base + m.end())
|
| 406 |
+
|
| 407 |
+
for m in _CURRENCY_FINDER.finditer(suffix):
|
| 408 |
+
cur, code = _resolve_currency(m.group(0))
|
| 409 |
+
if cur is None:
|
| 410 |
+
continue
|
| 411 |
+
gap = m.start()
|
| 412 |
+
if gap <= _CURRENCY_GAP_TOLERANCE and (best is None or gap < best[2]):
|
| 413 |
+
best = (cur, code, gap, raw_end + m.start(), raw_end + m.end())
|
| 414 |
+
|
| 415 |
+
if best is None:
|
| 416 |
+
return None, None, None, None, None
|
| 417 |
+
return best
|
| 418 |
+
|
| 419 |
+
|
| 420 |
+
def _detect_magnitude(text: str, pos: int) -> tuple[Optional[Decimal], int]:
|
| 421 |
+
"""Detect a magnitude suffix right after the number; return (multiplier, end).
|
| 422 |
+
|
| 423 |
+
Single letters must NOT be followed by another letter so currency codes and
|
| 424 |
+
units are never eaten: "50K" -> x1000 but "Kč" and "Kg" are not magnitudes.
|
| 425 |
+
"""
|
| 426 |
+
i = pos
|
| 427 |
+
if i < len(text) and text[i] == " ":
|
| 428 |
+
i += 1
|
| 429 |
+
rest = text[i:i + 10]
|
| 430 |
+
m = re.match(r"(?i)(?:crore|cr|lakh|lac)\b", rest)
|
| 431 |
+
if m:
|
| 432 |
+
word = m.group(0).lower()
|
| 433 |
+
mult = Decimal("10000000") if word in ("crore", "cr") else Decimal("100000")
|
| 434 |
+
return mult, i + m.end()
|
| 435 |
+
if not rest:
|
| 436 |
+
return None, pos
|
| 437 |
+
c = rest[0]
|
| 438 |
+
if len(rest) > 1 and rest[1].isalpha():
|
| 439 |
+
return None, pos # "Kč", "Kg", "MOP", "LKR" ... not magnitudes
|
| 440 |
+
if c in "Kk":
|
| 441 |
+
return Decimal("1000"), i + 1
|
| 442 |
+
if c == "M":
|
| 443 |
+
return Decimal("1000000"), i + 1
|
| 444 |
+
if c == "B":
|
| 445 |
+
return Decimal("1000000000"), i + 1
|
| 446 |
+
if c == "T":
|
| 447 |
+
return Decimal("1000000000000"), i + 1
|
| 448 |
+
if c == "L":
|
| 449 |
+
return Decimal("100000"), i + 1
|
| 450 |
+
return None, pos
|
| 451 |
+
|
| 452 |
+
|
| 453 |
+
def _merge_stray_letters(text: str, start: int, end: int) -> tuple[int, str]:
|
| 454 |
+
"""Extend a digit token across <=2 stray letters followed by digits.
|
| 455 |
+
|
| 456 |
+
Handles OCR noise like ``8a9.00`` -> ``89.00``. Returns ``(new_end, letters)``
|
| 457 |
+
or ``(end, "")`` when there is nothing to merge. Never merges across
|
| 458 |
+
``e``/``E`` (scientific) or ``x``/``X`` (hex) markers.
|
| 459 |
+
"""
|
| 460 |
+
i = end
|
| 461 |
+
letters = 0
|
| 462 |
+
while i < len(text) and text[i].isalpha() and letters < 3:
|
| 463 |
+
letters += 1
|
| 464 |
+
i += 1
|
| 465 |
+
if letters == 0 or letters > 2:
|
| 466 |
+
return end, ""
|
| 467 |
+
if any(c in "eExX" for c in text[end:i]):
|
| 468 |
+
return end, ""
|
| 469 |
+
j = i
|
| 470 |
+
if j >= len(text) or not text[j].isdigit():
|
| 471 |
+
return end, ""
|
| 472 |
+
k = j
|
| 473 |
+
while k < len(text) and (text[k].isdigit() or text[k] in " .,"):
|
| 474 |
+
k += 1
|
| 475 |
+
return k, text[end:i]
|
| 476 |
+
|
| 477 |
+
|
| 478 |
+
def _merge_space_continuation(text: str, start: int, end: int) -> int:
|
| 479 |
+
"""Merge a wrapped decimal continuation across a space.
|
| 480 |
+
|
| 481 |
+
OCR / text-extraction artefacts sometimes split a number as
|
| 482 |
+
``"896,009.0 0"`` -> ``"896,009.00"``. We merge only when the current
|
| 483 |
+
token is already a decimal (1-2 fraction digits after the last separator)
|
| 484 |
+
and the run right after the space is 1-2 digits, so ``"500.00 600.00"``
|
| 485 |
+
and ``"10 20"`` stay untouched.
|
| 486 |
+
"""
|
| 487 |
+
i = end
|
| 488 |
+
if i >= len(text) or text[i] != " ":
|
| 489 |
+
return end
|
| 490 |
+
j = i + 1
|
| 491 |
+
k = j
|
| 492 |
+
while k < len(text) and text[k].isdigit():
|
| 493 |
+
k += 1
|
| 494 |
+
run_len = k - j
|
| 495 |
+
if not 1 <= run_len <= 2:
|
| 496 |
+
return end
|
| 497 |
+
if k < len(text) and (text[k].isdigit() or text[k] in ".,"):
|
| 498 |
+
return end # the run continues into another number
|
| 499 |
+
body = text[start:end]
|
| 500 |
+
last_sep = max(body.rfind(","), body.rfind("."))
|
| 501 |
+
if last_sep < 0:
|
| 502 |
+
return end
|
| 503 |
+
frac = body[last_sep + 1:]
|
| 504 |
+
if not (1 <= len(frac) <= 2 and frac.isdigit()):
|
| 505 |
+
return end
|
| 506 |
+
return k
|
| 507 |
+
|
| 508 |
+
|
| 509 |
+
def _sci_guards(text: str, start: int, end: int) -> bool:
|
| 510 |
+
"""Skip tokens that are part of scientific notation or hex literals."""
|
| 511 |
+
lo = max(0, start - 3)
|
| 512 |
+
hi = min(len(text), end + 3)
|
| 513 |
+
for m in _SCI_RE.finditer(text, lo, hi):
|
| 514 |
+
if m.end() > start and (m.start() < end or m.start() <= end + 2):
|
| 515 |
+
return True
|
| 516 |
+
if text[max(0, start - 1):end + 2].lower().startswith("0x"):
|
| 517 |
+
return True
|
| 518 |
+
return False
|
| 519 |
+
|
| 520 |
+
|
| 521 |
+
def _has_percent(text: str, raw_start: int, raw_end: int) -> bool:
|
| 522 |
+
if "%" in text[max(0, raw_start - 1):raw_end + 1]:
|
| 523 |
+
return True
|
| 524 |
+
after = text[raw_end:raw_end + 8]
|
| 525 |
+
return bool(re.search(r"(?i)\bpercent\b|\bper cent\b", after))
|
| 526 |
+
|
| 527 |
+
|
| 528 |
+
def _skip_regions(text: str) -> list[tuple[int, int]]:
|
| 529 |
+
regions: list[tuple[int, int]] = []
|
| 530 |
+
for pat in (_DATE_RE, _TIME_RE, _IP_RE, _ID_RE, _PHONE_RE, _AREA_CODE_RE):
|
| 531 |
+
regions.extend((m.start(), m.end()) for m in pat.finditer(text))
|
| 532 |
+
regions.sort()
|
| 533 |
+
return regions
|
| 534 |
+
|
| 535 |
+
|
| 536 |
+
def _in_skip_region(start: int, end: int, regions: list[tuple[int, int]], starts: list[int]) -> bool:
|
| 537 |
+
i = bisect_right(starts, end - 1)
|
| 538 |
+
return i > 0 and regions[i - 1][1] > start
|
| 539 |
+
|
| 540 |
+
|
| 541 |
+
# ---------------------------------------------------------------------------
|
| 542 |
+
# Candidate generation
|
| 543 |
+
# ---------------------------------------------------------------------------
|
| 544 |
+
|
| 545 |
+
|
| 546 |
+
def _iter_candidates(
|
| 547 |
+
text: str,
|
| 548 |
+
regions: list[tuple[int, int]],
|
| 549 |
+
starts: list[int],
|
| 550 |
+
currency_hint: Optional[str],
|
| 551 |
+
) -> Iterator[Amount]:
|
| 552 |
+
n = len(text)
|
| 553 |
+
consumed = 0
|
| 554 |
+
hint_cur, hint_code = _resolve_currency(currency_hint) if currency_hint else (None, None)
|
| 555 |
+
|
| 556 |
+
for m in _NUMBER_TOKEN_RE.finditer(text):
|
| 557 |
+
start, end = m.start(), m.end()
|
| 558 |
+
if start < consumed:
|
| 559 |
+
continue
|
| 560 |
+
if _sci_guards(text, start, end):
|
| 561 |
+
continue
|
| 562 |
+
|
| 563 |
+
merged_end, letters = _merge_stray_letters(text, start, end)
|
| 564 |
+
merged_end = _merge_space_continuation(text, start, merged_end)
|
| 565 |
+
if merged_end > consumed:
|
| 566 |
+
consumed = merged_end
|
| 567 |
+
# Number body with stray OCR letters ("8a9.00" -> "89.00") and wrapped
|
| 568 |
+
# decimal continuations ("896,009.0 0" -> "896,009.00") cleaned up.
|
| 569 |
+
if merged_end > end:
|
| 570 |
+
seg = text[end + len(letters):merged_end].replace(" ", "")
|
| 571 |
+
body = text[start:end] + seg
|
| 572 |
+
else:
|
| 573 |
+
body = text[start:merged_end]
|
| 574 |
+
|
| 575 |
+
# --- sign ---
|
| 576 |
+
negative = False
|
| 577 |
+
raw_start = start
|
| 578 |
+
if raw_start > 0 and text[raw_start - 1] in "-−+":
|
| 579 |
+
negative = text[raw_start - 1] in "-−"
|
| 580 |
+
raw_start -= 1
|
| 581 |
+
raw_end = merged_end
|
| 582 |
+
if raw_end < n and text[raw_end] in "-−":
|
| 583 |
+
negative = True
|
| 584 |
+
raw_end += 1
|
| 585 |
+
if raw_start > 0 and text[raw_start - 1] == "(":
|
| 586 |
+
close = text.find(")", raw_end, raw_end + 5)
|
| 587 |
+
if close != -1 and not any(c.isdigit() for c in text[close + 1:close + 4]):
|
| 588 |
+
negative = True
|
| 589 |
+
raw_start -= 1
|
| 590 |
+
raw_end = close + 1
|
| 591 |
+
|
| 592 |
+
value = _parse_decimal(body)
|
| 593 |
+
if value is None:
|
| 594 |
+
continue
|
| 595 |
+
if negative:
|
| 596 |
+
value = -value
|
| 597 |
+
|
| 598 |
+
# --- magnitude ---
|
| 599 |
+
mult, mag_end = _detect_magnitude(text, raw_end)
|
| 600 |
+
magnitude = mult is not None
|
| 601 |
+
if mult is not None:
|
| 602 |
+
value *= mult
|
| 603 |
+
raw_end = mag_end
|
| 604 |
+
|
| 605 |
+
# --- currency ---
|
| 606 |
+
cur, code, gap, cur_start, cur_end = _nearest_currency(text, raw_start, raw_end)
|
| 607 |
+
if cur_start is not None:
|
| 608 |
+
raw_start = min(raw_start, cur_start)
|
| 609 |
+
raw_end = max(raw_end, cur_end)
|
| 610 |
+
|
| 611 |
+
# --- context words ---
|
| 612 |
+
# Amount-indicating words only count BEFORE the number ("Total: 550.00"),
|
| 613 |
+
# so "50.00, Total 550.00" never boosts the wrong candidate. Reject words
|
| 614 |
+
# (quantities, references, units) count on both sides.
|
| 615 |
+
prefix = text[max(0, raw_start - _WORD_WINDOW_BEFORE):raw_start]
|
| 616 |
+
suffix = text[raw_end:raw_end + _WORD_WINDOW_AFTER]
|
| 617 |
+
amount_strong = bool(_STRONG_AMOUNT_RE.search(prefix))
|
| 618 |
+
amount_medium = bool(_MEDIUM_AMOUNT_RE.search(prefix))
|
| 619 |
+
reject = bool(_REJECT_WORD_RE.search(prefix + " " + suffix))
|
| 620 |
+
of_penalty = bool(_OF_RE.search(prefix[-4:]))
|
| 621 |
+
|
| 622 |
+
int_digits = len(str(abs(value).to_integral_value()))
|
| 623 |
+
frac_digits = max(-value.as_tuple().exponent, 0)
|
| 624 |
+
seg = text[raw_start:raw_end]
|
| 625 |
+
|
| 626 |
+
# --- hard skips (region / pattern based) ---
|
| 627 |
+
if _in_skip_region(start, merged_end, regions, starts):
|
| 628 |
+
continue
|
| 629 |
+
if _has_percent(text, raw_start, raw_end):
|
| 630 |
+
continue
|
| 631 |
+
if (
|
| 632 |
+
not magnitude
|
| 633 |
+
and cur is None
|
| 634 |
+
and not amount_strong
|
| 635 |
+
and not amount_medium
|
| 636 |
+
and re.fullmatch(r"\d{4}", body)
|
| 637 |
+
and 1900 <= value <= 2099
|
| 638 |
+
):
|
| 639 |
+
continue # year
|
| 640 |
+
if (
|
| 641 |
+
cur is None
|
| 642 |
+
and not amount_strong
|
| 643 |
+
and not amount_medium
|
| 644 |
+
and not magnitude
|
| 645 |
+
and int_digits >= 8
|
| 646 |
+
and not any(c in seg for c in " .,")
|
| 647 |
+
):
|
| 648 |
+
continue # long bare digit run (IDs, phone numbers)
|
| 649 |
+
if (
|
| 650 |
+
cur is None
|
| 651 |
+
and not amount_strong
|
| 652 |
+
and not amount_medium
|
| 653 |
+
and not magnitude
|
| 654 |
+
and int_digits >= 7
|
| 655 |
+
and (" " in seg or "-" in seg)
|
| 656 |
+
):
|
| 657 |
+
continue # phone-like number with separators
|
| 658 |
+
|
| 659 |
+
# --- scoring ---
|
| 660 |
+
score = 0.5
|
| 661 |
+
if cur is not None:
|
| 662 |
+
score += 0.35 if gap == 0 else 0.30
|
| 663 |
+
if code is not None:
|
| 664 |
+
score += 0.05
|
| 665 |
+
elif hint_cur is not None:
|
| 666 |
+
cur, code = hint_cur, hint_code
|
| 667 |
+
score += 0.10
|
| 668 |
+
if amount_strong:
|
| 669 |
+
score += 0.25
|
| 670 |
+
elif amount_medium:
|
| 671 |
+
score += 0.15
|
| 672 |
+
if magnitude:
|
| 673 |
+
score += 0.10
|
| 674 |
+
if frac_digits == 2:
|
| 675 |
+
score += 0.05
|
| 676 |
+
if negative:
|
| 677 |
+
score += 0.02
|
| 678 |
+
if of_penalty:
|
| 679 |
+
score -= 0.40
|
| 680 |
+
# A quantity/reference word near the number ("Qty: 3", "Invoice No: 00125")
|
| 681 |
+
# kills the candidate unless an amount word is also present ("Unit price: 25").
|
| 682 |
+
if reject and not (amount_strong or amount_medium):
|
| 683 |
+
score -= 0.60
|
| 684 |
+
if cur is None and not amount_strong and not amount_medium and int_digits <= 3:
|
| 685 |
+
score -= 0.15
|
| 686 |
+
score = max(0.0, min(1.0, score))
|
| 687 |
+
|
| 688 |
+
raw = text[raw_start:raw_end]
|
| 689 |
+
yield Amount(
|
| 690 |
+
amount=value,
|
| 691 |
+
currency=cur,
|
| 692 |
+
currency_code=code,
|
| 693 |
+
amount_text=body,
|
| 694 |
+
raw=raw,
|
| 695 |
+
confidence=round(score, 4),
|
| 696 |
+
is_negative=negative,
|
| 697 |
+
position=start,
|
| 698 |
+
)
|
| 699 |
+
|
| 700 |
+
|
| 701 |
+
# ---------------------------------------------------------------------------
|
| 702 |
+
# Public API
|
| 703 |
+
# ---------------------------------------------------------------------------
|
| 704 |
+
|
| 705 |
+
|
| 706 |
+
def extract_amounts(
|
| 707 |
+
text,
|
| 708 |
+
*,
|
| 709 |
+
limit: Optional[int] = None,
|
| 710 |
+
currency_hint: Optional[str] = None,
|
| 711 |
+
) -> List[Amount]:
|
| 712 |
+
"""Return every candidate amount in ``text``, ranked by confidence.
|
| 713 |
+
|
| 714 |
+
Accepts ``str`` (or a numeric input such as ``int``/``float``/``Decimal``).
|
| 715 |
+
``None`` and non-string inputs return an empty list. Raises ``ValueError``
|
| 716 |
+
for inputs longer than ``MAX_INPUT_LENGTH`` characters.
|
| 717 |
+
"""
|
| 718 |
+
if text is None:
|
| 719 |
+
return []
|
| 720 |
+
if isinstance(text, (bool, int, float, Decimal)):
|
| 721 |
+
text = str(text)
|
| 722 |
+
if not isinstance(text, str):
|
| 723 |
+
return []
|
| 724 |
+
if len(text) > MAX_INPUT_LENGTH:
|
| 725 |
+
raise ValueError(
|
| 726 |
+
f"input exceeds MAX_INPUT_LENGTH={MAX_INPUT_LENGTH:,} characters"
|
| 727 |
+
)
|
| 728 |
+
|
| 729 |
+
regions = _skip_regions(text)
|
| 730 |
+
starts = [s for s, _ in regions]
|
| 731 |
+
results = list(_iter_candidates(text, regions, starts, currency_hint))
|
| 732 |
+
results.sort(key=lambda a: (-a.confidence, a.position))
|
| 733 |
+
if limit is not None:
|
| 734 |
+
results = results[:limit]
|
| 735 |
+
return results
|
| 736 |
+
|
| 737 |
+
|
| 738 |
+
def parse_amount(
|
| 739 |
+
text,
|
| 740 |
+
*,
|
| 741 |
+
currency_hint: Optional[str] = None,
|
| 742 |
+
default_currency: Optional[str] = None,
|
| 743 |
+
) -> Optional[Amount]:
|
| 744 |
+
"""Return the single best amount in ``text``, or ``None``.
|
| 745 |
+
|
| 746 |
+
* ``currency_hint`` labels candidates that have no currency marker of their
|
| 747 |
+
own (e.g. ``parse_amount("34.99", currency_hint="руб")``).
|
| 748 |
+
* ``default_currency`` labels the final result when no currency was found
|
| 749 |
+
(and no hint supplied).
|
| 750 |
+
* Strings that mean "no charge" (``"Free"``, ``"N/A"``, ``"no charge"`` ...)
|
| 751 |
+
return ``Amount(0)`` with high confidence.
|
| 752 |
+
* Returns ``None`` when no candidate clears the confidence floor — e.g.
|
| 753 |
+
dates, phone numbers, percentages, invoice references.
|
| 754 |
+
"""
|
| 755 |
+
if text is None:
|
| 756 |
+
return None
|
| 757 |
+
if isinstance(text, (bool, int, float, Decimal)):
|
| 758 |
+
text = str(text)
|
| 759 |
+
if not isinstance(text, str):
|
| 760 |
+
return None
|
| 761 |
+
stripped = text.strip()
|
| 762 |
+
if not stripped:
|
| 763 |
+
return None
|
| 764 |
+
if len(text) > MAX_INPUT_LENGTH:
|
| 765 |
+
raise ValueError(
|
| 766 |
+
f"input exceeds MAX_INPUT_LENGTH={MAX_INPUT_LENGTH:,} characters"
|
| 767 |
+
)
|
| 768 |
+
if _FREE_RE.fullmatch(stripped):
|
| 769 |
+
return Amount(
|
| 770 |
+
amount=Decimal("0"),
|
| 771 |
+
currency=None,
|
| 772 |
+
currency_code=None,
|
| 773 |
+
amount_text="0",
|
| 774 |
+
raw=stripped,
|
| 775 |
+
confidence=0.95,
|
| 776 |
+
is_negative=False,
|
| 777 |
+
position=0,
|
| 778 |
+
)
|
| 779 |
+
|
| 780 |
+
results = extract_amounts(text, currency_hint=currency_hint)
|
| 781 |
+
if not results:
|
| 782 |
+
return None
|
| 783 |
+
best = results[0]
|
| 784 |
+
if best.confidence < _CONFIDENCE_FLOOR:
|
| 785 |
+
return None
|
| 786 |
+
|
| 787 |
+
if best.currency is None and default_currency is not None:
|
| 788 |
+
cur, code = _resolve_currency(default_currency)
|
| 789 |
+
if cur is not None:
|
| 790 |
+
return Amount(
|
| 791 |
+
amount=best.amount,
|
| 792 |
+
currency=cur,
|
| 793 |
+
currency_code=code,
|
| 794 |
+
amount_text=best.amount_text,
|
| 795 |
+
raw=best.raw,
|
| 796 |
+
confidence=best.confidence,
|
| 797 |
+
is_negative=best.is_negative,
|
| 798 |
+
position=best.position,
|
| 799 |
+
)
|
| 800 |
+
return best
|
| 801 |
+
|
| 802 |
+
|
| 803 |
+
#: Alias mirroring the familiar ``price_parser.parse_price`` name.
|
| 804 |
+
parse_price = parse_amount
|
| 805 |
+
|
| 806 |
+
|
| 807 |
+
__all__ = [
|
| 808 |
+
"Amount",
|
| 809 |
+
"extract_amounts",
|
| 810 |
+
"parse_amount",
|
| 811 |
+
"parse_price",
|
| 812 |
+
"MAX_INPUT_LENGTH",
|
| 813 |
+
"CONFIDENCE_FLOOR",
|
| 814 |
+
]
|
services.yaml
CHANGED
|
@@ -90,6 +90,9 @@ media_convert:
|
|
| 90 |
json_extract:
|
| 91 |
enabled: true
|
| 92 |
|
|
|
|
|
|
|
|
|
|
| 93 |
keys_extract:
|
| 94 |
enabled: true
|
| 95 |
|
|
|
|
| 90 |
json_extract:
|
| 91 |
enabled: true
|
| 92 |
|
| 93 |
+
price_parser:
|
| 94 |
+
enabled: true
|
| 95 |
+
|
| 96 |
keys_extract:
|
| 97 |
enabled: true
|
| 98 |
|