Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -1417,6 +1417,111 @@ def _reconstruct_separator_rows(grid):
|
|
| 1417 |
# Normalizer entry point
|
| 1418 |
# --------------------------
|
| 1419 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1420 |
def normalize_html_tables(text: str) -> str:
|
| 1421 |
"""
|
| 1422 |
For every <table>...</table>:
|
|
@@ -1457,6 +1562,10 @@ def normalize_html_tables(text: str) -> str:
|
|
| 1457 |
# Fix 2: misplaced header row promotion (keyword-score guard β safe on clean PDFs)
|
| 1458 |
grid = _promote_misplaced_header_row(grid)
|
| 1459 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1460 |
# Fix 3: fused key-value rows in summary sections (no-space guard β safe on all PDFs)
|
| 1461 |
grid = _fix_fused_keyvalue_rows(grid)
|
| 1462 |
|
|
|
|
| 1417 |
# Normalizer entry point
|
| 1418 |
# --------------------------
|
| 1419 |
|
| 1420 |
+
def _merge_split_rows(grid):
|
| 1421 |
+
"""
|
| 1422 |
+
Fix OCR artifact where a transaction row is split across two <tr> rows
|
| 1423 |
+
because the description wrapped to a second line in the source PDF.
|
| 1424 |
+
|
| 1425 |
+
Two patterns detected (Hardin County Bank and similar monospace statements):
|
| 1426 |
+
|
| 1427 |
+
Pattern A β description + empty continuation:
|
| 1428 |
+
Row N: [desc, '', '', '', ''] β description, no date/money
|
| 1429 |
+
Row N+1: ['', debit, credit, date, bal] β money/date, no description
|
| 1430 |
+
β Merge into: [desc, debit, credit, date, bal]
|
| 1431 |
+
|
| 1432 |
+
Pattern B β description + text continuation + money:
|
| 1433 |
+
Row N: [desc, '', '', '', ''] β first line of description
|
| 1434 |
+
Row N+1: [desc_cont, debit, credit, date, bal] β continuation + money
|
| 1435 |
+
β Merge into: [desc + ' ' + desc_cont, debit, credit, date, bal]
|
| 1436 |
+
|
| 1437 |
+
Guard conditions (no-op unless both rows match the pattern):
|
| 1438 |
+
- Row N must have non-empty col 0 (description).
|
| 1439 |
+
- Row N must have empty date column (col 3 or wherever DATE is).
|
| 1440 |
+
- Row N must have empty debit AND credit columns.
|
| 1441 |
+
- Row N+1 must have non-empty date column.
|
| 1442 |
+
- Row N+1 must have non-empty debit OR credit column.
|
| 1443 |
+
- Row N balance (if present) must be a short fragment without a decimal
|
| 1444 |
+
(1-6 chars, no '.') β confirms it's a split-off ref number, not a balance.
|
| 1445 |
+
"""
|
| 1446 |
+
if not grid or len(grid) < 3:
|
| 1447 |
+
return grid
|
| 1448 |
+
|
| 1449 |
+
ncols = len(grid[0])
|
| 1450 |
+
if ncols < 3:
|
| 1451 |
+
return grid
|
| 1452 |
+
|
| 1453 |
+
# Identify column indices from header
|
| 1454 |
+
header = [str(c or "").strip().upper() for c in grid[0]]
|
| 1455 |
+
date_col = desc_col = bal_col = None
|
| 1456 |
+
debit_cols = []
|
| 1457 |
+
credit_cols = []
|
| 1458 |
+
|
| 1459 |
+
for ci, h in enumerate(header):
|
| 1460 |
+
if re.search(r"\bDATE\b", h) and date_col is None:
|
| 1461 |
+
date_col = ci
|
| 1462 |
+
if re.search(r"\b(DESCRIPTION|DETAILS?|NARRATION|PARTICULARS?|TRANSACTION)\b", h) and desc_col is None:
|
| 1463 |
+
desc_col = ci
|
| 1464 |
+
if re.search(r"\bBALANCE\b", h) and bal_col is None:
|
| 1465 |
+
bal_col = ci
|
| 1466 |
+
if re.search(r"\b(DEBIT|DEBITS|DR|WITHDRAWAL|SUBTRACTIONS?)\b", h):
|
| 1467 |
+
debit_cols.append(ci)
|
| 1468 |
+
if re.search(r"\b(CREDIT|CREDITS|CR|DEPOSIT|ADDITIONS?)\b", h):
|
| 1469 |
+
credit_cols.append(ci)
|
| 1470 |
+
|
| 1471 |
+
# Need at least date + description + one money column
|
| 1472 |
+
if date_col is None or desc_col is None:
|
| 1473 |
+
return grid
|
| 1474 |
+
if not debit_cols and not credit_cols:
|
| 1475 |
+
return grid
|
| 1476 |
+
|
| 1477 |
+
money_cols = debit_cols + credit_cols
|
| 1478 |
+
_MONEY_RE2 = re.compile(r"^-?\$?\d{1,3}(?:,\d{3})*\.\d{1,4}$")
|
| 1479 |
+
_DATE_RE2 = re.compile(r"^\d{1,2}/\d{2}(?:/\d{2,4})?$|^\d{1,2}-\d{2}(?:-\d{2,4})?$")
|
| 1480 |
+
# Short fragment guard: 1-6 non-space chars, no decimal point β split-off ref tail
|
| 1481 |
+
_FRAGMENT_RE = re.compile(r"^[^\s\.]{1,6}$")
|
| 1482 |
+
|
| 1483 |
+
new_grid = [grid[0]]
|
| 1484 |
+
i = 1
|
| 1485 |
+
while i < len(grid):
|
| 1486 |
+
row = [str(c or "").strip() for c in grid[i]]
|
| 1487 |
+
while len(row) < ncols:
|
| 1488 |
+
row.append("")
|
| 1489 |
+
|
| 1490 |
+
# Check if this row is a "description-only" split row candidate
|
| 1491 |
+
has_desc = bool(row[desc_col])
|
| 1492 |
+
no_date = not row[date_col]
|
| 1493 |
+
no_money = all(not row[c] for c in money_cols)
|
| 1494 |
+
bal_is_frag = (bal_col is not None and
|
| 1495 |
+
(_FRAGMENT_RE.match(row[bal_col]) or not row[bal_col]))
|
| 1496 |
+
|
| 1497 |
+
if has_desc and no_date and no_money and bal_is_frag and (i + 1) < len(grid):
|
| 1498 |
+
next_row = [str(c or "").strip() for c in grid[i + 1]]
|
| 1499 |
+
while len(next_row) < ncols:
|
| 1500 |
+
next_row.append("")
|
| 1501 |
+
|
| 1502 |
+
next_has_date = bool(next_row[date_col]) and _DATE_RE2.match(next_row[date_col])
|
| 1503 |
+
next_has_money = any(bool(next_row[c]) for c in money_cols)
|
| 1504 |
+
|
| 1505 |
+
if next_has_date and next_has_money:
|
| 1506 |
+
# Pattern A or B β merge
|
| 1507 |
+
merged = list(next_row)
|
| 1508 |
+
# Append description (and possible continuation from next row)
|
| 1509 |
+
if next_row[desc_col]:
|
| 1510 |
+
# Pattern B: next row has desc continuation
|
| 1511 |
+
merged[desc_col] = row[desc_col] + " " + next_row[desc_col]
|
| 1512 |
+
else:
|
| 1513 |
+
# Pattern A: next row has no description
|
| 1514 |
+
merged[desc_col] = row[desc_col]
|
| 1515 |
+
new_grid.append(merged)
|
| 1516 |
+
i += 2 # skip both rows, emit merged
|
| 1517 |
+
continue
|
| 1518 |
+
|
| 1519 |
+
new_grid.append(row)
|
| 1520 |
+
i += 1
|
| 1521 |
+
|
| 1522 |
+
return new_grid
|
| 1523 |
+
|
| 1524 |
+
|
| 1525 |
def normalize_html_tables(text: str) -> str:
|
| 1526 |
"""
|
| 1527 |
For every <table>...</table>:
|
|
|
|
| 1562 |
# Fix 2: misplaced header row promotion (keyword-score guard β safe on clean PDFs)
|
| 1563 |
grid = _promote_misplaced_header_row(grid)
|
| 1564 |
|
| 1565 |
+
# Fix 6: merge split rows where description wraps to next line
|
| 1566 |
+
# (Hardin County Bank and similar monospace statement formats)
|
| 1567 |
+
grid = _merge_split_rows(grid)
|
| 1568 |
+
|
| 1569 |
# Fix 3: fused key-value rows in summary sections (no-space guard β safe on all PDFs)
|
| 1570 |
grid = _fix_fused_keyvalue_rows(grid)
|
| 1571 |
|