3v324v23's picture
Feat: Added New column in xlsx
fbb917f
Raw History Blame Contribute Delete
8.57 kB
import io
import sys
import logging
from pathlib import Path
from typing import Optional, List
from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks
from fastapi.responses import StreamingResponse
from starlette.concurrency import run_in_threadpool
from auth.models import User
from auth.service import get_current_user
from models.layout import (
HeaderDetectionResultModel, ColumnLayoutModel,
VerifyLayoutPayloadModel, ColumnUpdatePayloadModel
)
from core.state import statements, StatementData
from core.extraction import run_coordinate_extraction_task
logger = logging.getLogger("befree.layout")
router = APIRouter(prefix="/api/v1/statements", tags=["layout"])
@router.post("/{statement_id}/detect-layout", response_model=HeaderDetectionResultModel)
async def detect_layout_endpoint(statement_id: str, current_user: User = Depends(get_current_user)):
"""Run layout detection on the statement."""
if statement_id not in statements:
raise HTTPException(status_code=404, detail="Statement not found")
stmt = statements[statement_id]
stmt.status = "detecting_layout"
project_root = Path(__file__).resolve().parent.parent
extractor_dir = project_root / "bank-extractor"
if str(extractor_dir) not in sys.path:
sys.path.insert(0, str(extractor_dir))
try:
from bank_extractor.pipeline import detect_statement_layout
except Exception as exc:
stmt.status = "failed"
raise HTTPException(status_code=500, detail=f"Failed to load layout detector: {exc}")
try:
result = await run_in_threadpool(detect_statement_layout, stmt.pdf_path)
except Exception as exc:
stmt.status = "failed"
raise HTTPException(status_code=422, detail=f"Layout detection failed: {exc}")
stmt.layout_detection_result = result
stmt.status = "awaiting_layout_review"
return HeaderDetectionResultModel(
found=result.found,
page_num=result.page_num,
columns=[
ColumnLayoutModel(
name=c.name,
header_label=c.header_label,
x_min=c.x_min,
x_max=c.x_max,
col_type=c.col_type,
confidence=c.confidence
) for c in result.columns
],
confidence=result.confidence
)
@router.get("/{statement_id}/layout", response_model=HeaderDetectionResultModel)
async def get_layout_endpoint(statement_id: str, current_user: User = Depends(get_current_user)):
"""Get the stored layout detection result."""
if statement_id not in statements:
raise HTTPException(status_code=404, detail="Statement not found")
stmt = statements[statement_id]
if not stmt.layout_detection_result:
raise HTTPException(status_code=404, detail="Layout has not been detected yet")
result = stmt.layout_detection_result
page_width = 612.0
page_height = 792.0
if stmt.pdf_path:
try:
import fitz
doc = fitz.open(str(stmt.pdf_path))
p_idx = max(0, min(result.page_num - 1, len(doc) - 1))
page = doc.load_page(p_idx)
page_width = float(page.rect.width)
page_height = float(page.rect.height)
doc.close()
except Exception as e:
logger.error(f"Error reading page dimensions: {e}")
return HeaderDetectionResultModel(
found=result.found,
page_num=result.page_num,
columns=[
ColumnLayoutModel(
name=c.name,
header_label=c.header_label,
x_min=c.x_min,
x_max=c.x_max,
col_type=c.col_type,
confidence=c.confidence
) for c in result.columns
],
confidence=result.confidence,
page_width=page_width,
page_height=page_height,
)
def resolve_absolute_page(stmt: StatementData, absolute_page: int) -> tuple[str, int]:
"""Given an absolute page number, resolve it to PDF path and relative page."""
pdf_paths = getattr(stmt, "pdf_paths", [])
if not pdf_paths:
if stmt.pdf_path:
return stmt.pdf_path, absolute_page
raise ValueError("No PDF paths found for statement")
import fitz
current_page = absolute_page
for pdf_path in pdf_paths:
try:
doc = fitz.open(pdf_path)
num_pages = len(doc)
doc.close()
except Exception:
num_pages = 0
if current_page <= num_pages:
return pdf_path, current_page
current_page -= num_pages
return pdf_paths[-1], current_page
@router.get("/{statement_id}/page-image/{page_num}")
async def get_page_image(statement_id: str, page_num: int, current_user: User = Depends(get_current_user)):
"""Render a specific page of a statement PDF as a PNG image using PyMuPDF."""
if statement_id not in statements:
raise HTTPException(status_code=404, detail="Statement not found")
stmt = statements[statement_id]
try:
pdf_path_str, relative_page = resolve_absolute_page(stmt, page_num)
except Exception as e:
raise HTTPException(status_code=404, detail=str(e))
pdf_path = Path(pdf_path_str).resolve()
if not pdf_path.exists():
raise HTTPException(status_code=404, detail="PDF file not found")
try:
import fitz
doc = fitz.open(str(pdf_path))
if relative_page < 1 or relative_page > len(doc):
raise HTTPException(status_code=400, detail=f"Page number out of range (1-{len(doc)})")
page = doc.load_page(relative_page - 1)
pix = page.get_pixmap(dpi=150)
png_bytes = pix.tobytes("png")
doc.close()
return StreamingResponse(io.BytesIO(png_bytes), media_type="image/png")
except Exception as e:
logger.error(f"Error rendering PDF page as image: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.post("/{statement_id}/verify-layout")
async def verify_layout_endpoint(
statement_id: str,
payload: VerifyLayoutPayloadModel,
background_tasks: BackgroundTasks,
current_user: User = Depends(get_current_user)
):
"""Verify layout and trigger coordinate-locked extraction in a background thread."""
if statement_id not in statements:
raise HTTPException(status_code=404, detail="Statement not found")
stmt = statements[statement_id]
from shared.models import ColumnLayout
stmt.verified_columns = [
ColumnLayout(
name=c.name,
header_label=c.header_label,
x_min=c.x_min,
x_max=c.x_max,
col_type=c.col_type,
confidence=c.confidence
) for c in payload.columns
]
stmt.header_page = payload.header_page
stmt.status = "extracting"
background_tasks.add_task(run_coordinate_extraction_task, statement_id)
return {"statementId": statement_id, "status": "extracting"}
@router.patch("/{statement_id}/layout/column/{col_name}")
async def patch_layout_column_endpoint(
statement_id: str,
col_name: str,
payload: ColumnUpdatePayloadModel,
current_user: User = Depends(get_current_user)
):
"""Allows updating a single column's x_min/x_max/name without re-detecting."""
if statement_id not in statements:
raise HTTPException(status_code=404, detail="Statement not found")
stmt = statements[statement_id]
if not stmt.layout_detection_result:
raise HTTPException(status_code=404, detail="Layout has not been detected yet")
updated = False
for col in stmt.layout_detection_result.columns:
if col.name == col_name:
col.x_min = payload.x_min
col.x_max = payload.x_max
col.header_label = payload.header_label
updated = True
break
if not updated:
from shared.models import ColumnLayout
new_col = ColumnLayout(
name=col_name,
header_label=payload.header_label,
x_min=payload.x_min,
x_max=payload.x_max,
col_type="numeric" if col_name in ["debit", "credit", "balance", "amount"] else ("date" if "date" in col_name else "text"),
confidence=1.0
)
stmt.layout_detection_result.columns.append(new_col)
return {"status": "success"}