Spaces:
Sleeping
Sleeping
Download core/extraction.py from Coeinterns/bank-statement-api: direct link, hf CLI and curl.
- Browser
- Download file 10.8 kB
-
https://huggingface.co/spaces/Coeinterns/bank-statement-api/resolve/main/core/extraction.py
- Command line
-
hf download hf://spaces/Coeinterns/bank-statement-api/core/extraction.py
-
curl -L -o extraction.py https://huggingface.co/spaces/Coeinterns/bank-statement-api/resolve/main/core/extraction.py
10.8 kB
| import os | |
| import sys | |
| import json | |
| import logging | |
| import asyncio | |
| from pathlib import Path | |
| from decimal import Decimal | |
| from datetime import datetime | |
| from types import SimpleNamespace | |
| from starlette.concurrency import run_in_threadpool | |
| from core.state import statements, _uploads_dir, _output_dir | |
| from core.logging_utils import _swap_logger | |
| from core.helpers import clean_audit_flags, _to_transaction_row_global, update_statement_confidence | |
| from core.statement_io import _persist_swap_to_xlsx, auto_swap_all_transactions | |
| logger = logging.getLogger("befree.extraction") | |
| async def run_coordinate_extraction_task(statement_id: str): | |
| """Background task to run coordinate locked extraction for multiple PDFs.""" | |
| logger.info("Background extraction task started for statement %s", statement_id) | |
| if statement_id not in statements: | |
| return | |
| stmt = statements[statement_id] | |
| from overwatch.event_log import EventLog as _OWEventLog | |
| document_name = stmt.file_name | |
| _ow_el = _OWEventLog(document_id=statement_id, document_name=document_name) | |
| project_root = Path(__file__).resolve().parent.parent | |
| extractor_dir = project_root / "bank-extractor" | |
| if str(extractor_dir) not in sys.path: | |
| sys.path.insert(0, str(extractor_dir)) | |
| try: | |
| from bank_extractor.pipeline import detect_statement_layout, extract_with_verified_layout | |
| output_path = str(_output_dir() / f"{statement_id}.xlsx") | |
| pdf_paths = getattr(stmt, "pdf_paths", []) | |
| if not pdf_paths and stmt.pdf_path: | |
| pdf_paths = [stmt.pdf_path] | |
| all_transactions = [] | |
| page_offset = 0 | |
| use_verified_layout = bool(stmt.verified_columns) and stmt.header_page is not None | |
| if use_verified_layout: | |
| _verified_cols = stmt.verified_columns | |
| _verified_page = stmt.header_page | |
| detected_layouts = [] | |
| for pdf_path in pdf_paths: | |
| detected_layouts.append(SimpleNamespace( | |
| columns=_verified_cols, | |
| page_num=_verified_page, | |
| found=True, | |
| confidence=1.0, | |
| )) | |
| stmt.layout_detected_count = len(pdf_paths) | |
| else: | |
| stmt.status = "detecting_layout" | |
| stmt.layout_detected_count = 0 | |
| detected_layouts = [] | |
| for idx, pdf_path in enumerate(pdf_paths): | |
| logger.info("Detecting layout for PDF %d/%d: %s", idx + 1, len(pdf_paths), pdf_path) | |
| layout_result = await run_in_threadpool(detect_statement_layout, pdf_path) | |
| detected_layouts.append(layout_result) | |
| stmt.layout_detected_count += 1 | |
| try: | |
| _ow_el.emit_info( | |
| "layout_detection", | |
| f"Layout detected: {len(layout_result.columns)} columns", | |
| document_name=document_name, | |
| document_id=statement_id, | |
| detected_columns=[c.name for c in layout_result.columns], | |
| header_page_number=layout_result.page_num + 1, | |
| detection_confidence=round(float(layout_result.confidence), 4), | |
| ) | |
| except Exception: | |
| pass | |
| if detected_layouts: | |
| stmt.layout_detection_result = detected_layouts[0] | |
| from shared.models import ColumnLayout | |
| stmt.verified_columns = [ | |
| ColumnLayout( | |
| name=c.name, | |
| header_label=c.header_label, | |
| x_min=c.x_min, | |
| x_max=c.x_max, | |
| col_type=c.col_type, | |
| confidence=c.confidence | |
| ) for c in detected_layouts[0].columns | |
| ] | |
| stmt.header_page = detected_layouts[0].page_num | |
| stmt.status = "extracting" | |
| stmt.extracted_count = 0 | |
| for idx, pdf_path in enumerate(pdf_paths): | |
| logger.info("Extracting transactions from PDF %d/%d: %s", idx + 1, len(pdf_paths), pdf_path) | |
| layout_result = detected_layouts[idx] | |
| result = await run_in_threadpool( | |
| extract_with_verified_layout, | |
| pdf_path, | |
| layout_result.columns, | |
| layout_result.page_num, | |
| f"{statement_id}_{idx}", | |
| stmt.bank_name or "generic" | |
| ) | |
| import fitz | |
| doc = fitz.open(pdf_path) | |
| num_pages = len(doc) | |
| doc.close() | |
| for tx in result.transactions: | |
| row = _to_transaction_row_global(tx, statement_id) | |
| row["pageNumber"] = row["pageNumber"] + page_offset | |
| row["id"] = f"{row['id']}_{idx}" | |
| row["file_index"] = idx | |
| all_transactions.append(row) | |
| page_offset += num_pages | |
| stmt.extracted_count += 1 | |
| filtered_txs = [] | |
| for tx in all_transactions: | |
| def is_zero_or_none(val): | |
| if val is None: | |
| return True | |
| try: | |
| return float(val) == 0 | |
| except (ValueError, TypeError): | |
| return True | |
| has_debit = not is_zero_or_none(tx.get('debit')) | |
| has_credit = not is_zero_or_none(tx.get('credit')) | |
| if has_debit or has_credit: | |
| filtered_txs.append(tx) | |
| stmt.transactions = filtered_txs | |
| try: | |
| clean_count = sum(1 for tx in stmt.transactions if not tx.get("auditFlags")) | |
| flagged_count = sum(1 for tx in stmt.transactions if tx.get("auditFlags")) | |
| avg_conf = sum(float(tx.get("confidence", 1.0)) for tx in stmt.transactions) / max(len(stmt.transactions), 1) | |
| _ow_el.emit_info( | |
| "normalisation", | |
| "Normalisation completed", | |
| document_name=document_name, | |
| document_id=statement_id, | |
| normalised_transaction_count=len(stmt.transactions), | |
| ) | |
| _ow_el.emit_info( | |
| "audit", | |
| "Audit completed", | |
| document_name=document_name, | |
| document_id=statement_id, | |
| clean_transaction_count=clean_count, | |
| flagged_transaction_count=flagged_count, | |
| overall_confidence_score=round(avg_conf, 4), | |
| ) | |
| except Exception: | |
| pass | |
| swapped = auto_swap_all_transactions(statement_id) | |
| if swapped: | |
| logger.info(f"[AutoSwapAll] Statement {statement_id}: auto-swapped {swapped} row(s) on initial load") | |
| _persist_swap_to_xlsx(statement_id) | |
| update_statement_confidence(statement_id) | |
| from bank_extractor.writer import write_excel | |
| from shared.models import PageAudit | |
| pages_dict = {} | |
| for tx in stmt.transactions: | |
| pages_dict.setdefault(tx.get("pageNumber", 1), []).append(tx) | |
| pages_audit = [] | |
| for p_num, p_txs in pages_dict.items(): | |
| p_txs_sorted = sorted(p_txs, key=lambda x: x.get("id")) | |
| opening = 0.0 | |
| closing = 0.0 | |
| for tx in p_txs_sorted: | |
| if tx.get("calculatedBalance") is not None: | |
| closing = float(tx.get("calculatedBalance")) | |
| break | |
| opening = closing - sum((float(tx.get("credit") or 0.0) - float(tx.get("debit") or 0.0)) for tx in p_txs) | |
| pages_audit.append(PageAudit( | |
| page_num=p_num, | |
| opening_balance=Decimal(str(opening)), | |
| closing_balance=Decimal(str(closing)), | |
| computed_balance=Decimal(str(closing)), | |
| delta=Decimal("0.0"), | |
| status="clean", | |
| issues=[] | |
| )) | |
| from shared.models import StatementAudit | |
| aud_rep = StatementAudit( | |
| statement_id=statement_id, | |
| bank_id=stmt.bank_name or "generic", | |
| pages=pages_audit | |
| ) | |
| from shared.models import NormTransaction | |
| from shared.constants import TxType | |
| norm_txs_for_excel = [] | |
| for tx in stmt.transactions: | |
| debit = tx.get("debit") | |
| credit = tx.get("credit") | |
| amount = (credit or 0.0) - (debit or 0.0) | |
| page_num_val = tx.get("pageNumber", 1) - 1 | |
| tx_id_val = tx.get("id", "") | |
| parts = tx_id_val.split("_") | |
| row_num_val = 999 | |
| if len(parts) >= 3: | |
| try: | |
| row_num_val = int(parts[2]) | |
| except ValueError: | |
| pass | |
| norm_txs_for_excel.append(NormTransaction( | |
| tx_id=tx_id_val, | |
| page_num=page_num_val, | |
| row_num=row_num_val, | |
| date=datetime.fromisoformat(tx.get("date")).date() if tx.get("date") else None, | |
| description=tx.get("description", ""), | |
| desc_raw=tx.get("description", ""), | |
| amount=Decimal(str(amount)), | |
| tx_type=TxType.DEBIT if amount < 0 else TxType.CREDIT, | |
| balance_after=Decimal(str(tx.get("calculatedBalance"))) if tx.get("calculatedBalance") is not None else None, | |
| confidence=tx.get("confidence", 1.0), | |
| audit_flags=tx.get("auditFlags") or [], | |
| source_extractor=tx.get("source_extractor", "coordinate_locked") | |
| )) | |
| await run_in_threadpool( | |
| write_excel, | |
| norm_txs_for_excel, | |
| aud_rep, | |
| output_path, | |
| statement_id, | |
| "", | |
| "coordinate_locked", | |
| stmt.bank_name or "generic", | |
| 0.0 | |
| ) | |
| stmt.xlsx_path = output_path | |
| stmt.status = "ready" | |
| logger.info("Background coordinate locked extraction complete for statement %s", statement_id) | |
| except Exception as exc: | |
| try: | |
| _ow_el.emit_error( | |
| "coordinate_extraction", | |
| f"Extraction failed: {type(exc).__name__}: {exc}", | |
| document_name=document_name, | |
| document_id=statement_id, | |
| failure_reason=f"{type(exc).__name__}: {exc}", | |
| ) | |
| except Exception: | |
| pass | |
| stmt.status = "failed" | |
| logger.exception("Background extraction failed for statement %s", statement_id) | |