import os import sys import json import logging import asyncio from pathlib import Path from decimal import Decimal from datetime import datetime from types import SimpleNamespace from starlette.concurrency import run_in_threadpool from core.state import statements, _uploads_dir, _output_dir from core.logging_utils import _swap_logger from core.helpers import clean_audit_flags, _to_transaction_row_global, update_statement_confidence from core.statement_io import _persist_swap_to_xlsx, auto_swap_all_transactions logger = logging.getLogger("befree.extraction") async def run_coordinate_extraction_task(statement_id: str): """Background task to run coordinate locked extraction for multiple PDFs.""" logger.info("Background extraction task started for statement %s", statement_id) if statement_id not in statements: return stmt = statements[statement_id] from overwatch.event_log import EventLog as _OWEventLog document_name = stmt.file_name _ow_el = _OWEventLog(document_id=statement_id, document_name=document_name) project_root = Path(__file__).resolve().parent.parent extractor_dir = project_root / "bank-extractor" if str(extractor_dir) not in sys.path: sys.path.insert(0, str(extractor_dir)) try: from bank_extractor.pipeline import detect_statement_layout, extract_with_verified_layout output_path = str(_output_dir() / f"{statement_id}.xlsx") pdf_paths = getattr(stmt, "pdf_paths", []) if not pdf_paths and stmt.pdf_path: pdf_paths = [stmt.pdf_path] all_transactions = [] page_offset = 0 use_verified_layout = bool(stmt.verified_columns) and stmt.header_page is not None if use_verified_layout: _verified_cols = stmt.verified_columns _verified_page = stmt.header_page detected_layouts = [] for pdf_path in pdf_paths: detected_layouts.append(SimpleNamespace( columns=_verified_cols, page_num=_verified_page, found=True, confidence=1.0, )) stmt.layout_detected_count = len(pdf_paths) else: stmt.status = "detecting_layout" stmt.layout_detected_count = 0 detected_layouts = [] for idx, pdf_path in enumerate(pdf_paths): logger.info("Detecting layout for PDF %d/%d: %s", idx + 1, len(pdf_paths), pdf_path) layout_result = await run_in_threadpool(detect_statement_layout, pdf_path) detected_layouts.append(layout_result) stmt.layout_detected_count += 1 try: _ow_el.emit_info( "layout_detection", f"Layout detected: {len(layout_result.columns)} columns", document_name=document_name, document_id=statement_id, detected_columns=[c.name for c in layout_result.columns], header_page_number=layout_result.page_num + 1, detection_confidence=round(float(layout_result.confidence), 4), ) except Exception: pass if detected_layouts: stmt.layout_detection_result = detected_layouts[0] from shared.models import ColumnLayout stmt.verified_columns = [ ColumnLayout( name=c.name, header_label=c.header_label, x_min=c.x_min, x_max=c.x_max, col_type=c.col_type, confidence=c.confidence ) for c in detected_layouts[0].columns ] stmt.header_page = detected_layouts[0].page_num stmt.status = "extracting" stmt.extracted_count = 0 for idx, pdf_path in enumerate(pdf_paths): logger.info("Extracting transactions from PDF %d/%d: %s", idx + 1, len(pdf_paths), pdf_path) layout_result = detected_layouts[idx] result = await run_in_threadpool( extract_with_verified_layout, pdf_path, layout_result.columns, layout_result.page_num, f"{statement_id}_{idx}", stmt.bank_name or "generic" ) import fitz doc = fitz.open(pdf_path) num_pages = len(doc) doc.close() for tx in result.transactions: row = _to_transaction_row_global(tx, statement_id) row["pageNumber"] = row["pageNumber"] + page_offset row["id"] = f"{row['id']}_{idx}" row["file_index"] = idx all_transactions.append(row) page_offset += num_pages stmt.extracted_count += 1 filtered_txs = [] for tx in all_transactions: def is_zero_or_none(val): if val is None: return True try: return float(val) == 0 except (ValueError, TypeError): return True has_debit = not is_zero_or_none(tx.get('debit')) has_credit = not is_zero_or_none(tx.get('credit')) if has_debit or has_credit: filtered_txs.append(tx) stmt.transactions = filtered_txs try: clean_count = sum(1 for tx in stmt.transactions if not tx.get("auditFlags")) flagged_count = sum(1 for tx in stmt.transactions if tx.get("auditFlags")) avg_conf = sum(float(tx.get("confidence", 1.0)) for tx in stmt.transactions) / max(len(stmt.transactions), 1) _ow_el.emit_info( "normalisation", "Normalisation completed", document_name=document_name, document_id=statement_id, normalised_transaction_count=len(stmt.transactions), ) _ow_el.emit_info( "audit", "Audit completed", document_name=document_name, document_id=statement_id, clean_transaction_count=clean_count, flagged_transaction_count=flagged_count, overall_confidence_score=round(avg_conf, 4), ) except Exception: pass swapped = auto_swap_all_transactions(statement_id) if swapped: logger.info(f"[AutoSwapAll] Statement {statement_id}: auto-swapped {swapped} row(s) on initial load") _persist_swap_to_xlsx(statement_id) update_statement_confidence(statement_id) from bank_extractor.writer import write_excel from shared.models import PageAudit pages_dict = {} for tx in stmt.transactions: pages_dict.setdefault(tx.get("pageNumber", 1), []).append(tx) pages_audit = [] for p_num, p_txs in pages_dict.items(): p_txs_sorted = sorted(p_txs, key=lambda x: x.get("id")) opening = 0.0 closing = 0.0 for tx in p_txs_sorted: if tx.get("calculatedBalance") is not None: closing = float(tx.get("calculatedBalance")) break opening = closing - sum((float(tx.get("credit") or 0.0) - float(tx.get("debit") or 0.0)) for tx in p_txs) pages_audit.append(PageAudit( page_num=p_num, opening_balance=Decimal(str(opening)), closing_balance=Decimal(str(closing)), computed_balance=Decimal(str(closing)), delta=Decimal("0.0"), status="clean", issues=[] )) from shared.models import StatementAudit aud_rep = StatementAudit( statement_id=statement_id, bank_id=stmt.bank_name or "generic", pages=pages_audit ) from shared.models import NormTransaction from shared.constants import TxType norm_txs_for_excel = [] for tx in stmt.transactions: debit = tx.get("debit") credit = tx.get("credit") amount = (credit or 0.0) - (debit or 0.0) page_num_val = tx.get("pageNumber", 1) - 1 tx_id_val = tx.get("id", "") parts = tx_id_val.split("_") row_num_val = 999 if len(parts) >= 3: try: row_num_val = int(parts[2]) except ValueError: pass norm_txs_for_excel.append(NormTransaction( tx_id=tx_id_val, page_num=page_num_val, row_num=row_num_val, date=datetime.fromisoformat(tx.get("date")).date() if tx.get("date") else None, description=tx.get("description", ""), desc_raw=tx.get("description", ""), amount=Decimal(str(amount)), tx_type=TxType.DEBIT if amount < 0 else TxType.CREDIT, balance_after=Decimal(str(tx.get("calculatedBalance"))) if tx.get("calculatedBalance") is not None else None, confidence=tx.get("confidence", 1.0), audit_flags=tx.get("auditFlags") or [], source_extractor=tx.get("source_extractor", "coordinate_locked") )) await run_in_threadpool( write_excel, norm_txs_for_excel, aud_rep, output_path, statement_id, "", "coordinate_locked", stmt.bank_name or "generic", 0.0 ) stmt.xlsx_path = output_path stmt.status = "ready" logger.info("Background coordinate locked extraction complete for statement %s", statement_id) except Exception as exc: try: _ow_el.emit_error( "coordinate_extraction", f"Extraction failed: {type(exc).__name__}: {exc}", document_name=document_name, document_id=statement_id, failure_reason=f"{type(exc).__name__}: {exc}", ) except Exception: pass stmt.status = "failed" logger.exception("Background extraction failed for statement %s", statement_id)