Spaces:
Sleeping
Sleeping
Download routers/classification.py from Coeinterns/bank-statement-api: direct link, hf CLI and curl.
- Browser
- Download file 14.1 kB
-
https://huggingface.co/spaces/Coeinterns/bank-statement-api/resolve/main/routers/classification.py
- Command line
-
hf download hf://spaces/Coeinterns/bank-statement-api/routers/classification.py
-
curl -L -o classification.py https://huggingface.co/spaces/Coeinterns/bank-statement-api/resolve/main/routers/classification.py
14.1 kB
| import os | |
| import uuid | |
| import logging | |
| from typing import Optional, List | |
| from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks, UploadFile, File | |
| import pandas as pd | |
| from starlette.concurrency import run_in_threadpool | |
| from auth.models import User | |
| from auth.service import get_current_user | |
| from models.admin_models import FinalisePayload | |
| from models.classification import CategoryOverride | |
| from models.transactions import BulkConfirmPayload | |
| from models.statements import UploadInitResponse | |
| from core.state import statements, settings, StatementData | |
| from core.helpers import _filter_transactions_by_last_n_months, _normalize_description | |
| from core.statement_io import _persist_swap_to_xlsx | |
| from core.pipeline import get_classification_pipeline | |
| logger = logging.getLogger("befree.classification") | |
| router = APIRouter(prefix="/api/v1", tags=["classification"]) | |
| async def run_statement_classification_task(statement_id: str, extract_last_3_months: Optional[bool] = None): | |
| """Internal helper to run the classification pipeline on a statement in the background.""" | |
| try: | |
| import logging as std_logging | |
| logger = std_logging.getLogger("classification") | |
| logger.setLevel(std_logging.INFO) | |
| if not logger.handlers: | |
| ch = std_logging.StreamHandler() | |
| ch.setFormatter(std_logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) | |
| logger.addHandler(ch) | |
| if not os.environ.get("GOOGLE_API_KEY"): | |
| logger.warning("GOOGLE_API_KEY not found in environment. LLM classification will be skipped.") | |
| stmt = statements[statement_id] | |
| if not stmt.transactions: | |
| logger.error(f"No transactions to classify for {statement_id}") | |
| return {"error": "No transactions to classify"} | |
| use_3_months = extract_last_3_months if extract_last_3_months is not None else settings.extractLast3Months | |
| stmt.extract_last_3_months = use_3_months | |
| classify_candidates = stmt.transactions | |
| if use_3_months: | |
| classify_candidates = _filter_transactions_by_last_n_months(stmt.transactions) | |
| logger.info( | |
| f"Starting classification for statement {statement_id} " | |
| f"(extract_last_3_months={use_3_months}, " | |
| f"{len(classify_candidates)}/{len(stmt.transactions)} transactions selected for classification)" | |
| ) | |
| try: | |
| from overwatch.event_log import EventLog as _OWEventLog | |
| _ow_cls_el = _OWEventLog(document_id=statement_id, document_name=stmt.file_name) | |
| except Exception: | |
| _ow_cls_el = None | |
| pipeline = get_classification_pipeline() | |
| if _ow_cls_el is not None: | |
| try: | |
| pipeline._event_log = _ow_cls_el | |
| except Exception: | |
| pass | |
| classified_subset = await run_in_threadpool( | |
| pipeline.classify_batch, classify_candidates | |
| ) | |
| try: | |
| pipeline._event_log = None | |
| except Exception: | |
| pass | |
| for tx in classified_subset: | |
| if "category_confidence" in tx: | |
| tx["categoryConfidence"] = tx.pop("category_confidence") | |
| if "category_source" in tx: | |
| tx["categorySource"] = tx.pop("category_source") | |
| if "third_party" in tx: | |
| tx["thirdParty"] = tx.pop("third_party") | |
| if "classification_reason" in tx: | |
| tx["classificationReason"] = tx.pop("classification_reason") | |
| classified_by_id = {tx.get("id"): tx for tx in classified_subset} | |
| merged_transactions = [] | |
| for tx in stmt.transactions: | |
| tx_id = tx.get("id") | |
| if tx_id and tx_id in classified_by_id: | |
| merged_transactions.append({**tx, **classified_by_id[tx_id]}) | |
| else: | |
| merged_transactions.append(tx) | |
| stmt.transactions = merged_transactions | |
| stmt.status = "completed" | |
| logger.info(f"Finished classification for statement {statement_id}") | |
| try: | |
| _persist_swap_to_xlsx(statement_id) | |
| except Exception as e: | |
| logger.error(f"Failed to persist classified statement {statement_id}: {e}") | |
| counts = {"classified": 0, "rule": 0, "ml": 0, "llm": 0, "fallback": 0, "uncategorized": 0} | |
| for tx in stmt.transactions: | |
| raw_src = tx.get("categorySource") or tx.get("category_source") or "" | |
| src = raw_src.split(":")[0] if raw_src else "uncategorized" | |
| if src in ("rule", "ml", "llm", "manual"): | |
| counts["classified"] += 1 | |
| if src in counts: | |
| counts[src] += 1 | |
| elif src == "fallback": | |
| counts["fallback"] += 1 | |
| else: | |
| counts["uncategorized"] += 1 | |
| return counts | |
| except Exception as exc: | |
| logger.error(f"Classification failed for {statement_id}: {exc}", exc_info=True) | |
| if statement_id in statements: | |
| statements[statement_id].status = "failed" | |
| return {"error": "Classification failed"} | |
| async def finalise_statement( | |
| statement_id: str, | |
| background_tasks: BackgroundTasks, | |
| payload: FinalisePayload = FinalisePayload(), | |
| current_user: User = Depends(get_current_user), | |
| ): | |
| """Lock the statement and trigger classification pipeline.""" | |
| if statement_id not in statements: | |
| raise HTTPException(status_code=404, detail="Statement not found") | |
| stmt = statements[statement_id] | |
| stmt.status = "classifying" | |
| use_3_months = payload.extract_last_3_months if payload.extract_last_3_months is not None else settings.extractLast3Months | |
| stmt.extract_last_3_months = use_3_months | |
| background_tasks.add_task(run_statement_classification_task, statement_id, use_3_months) | |
| return {"status": "classifying", "message": "Classification started in background"} | |
| async def classify_statement(statement_id: str, current_user: User = Depends(get_current_user)): | |
| """Run the classification pipeline on all transactions.""" | |
| res = await run_statement_classification_task(statement_id) | |
| if isinstance(res, dict) and "error" in res: | |
| raise HTTPException(status_code=500, detail=res["error"]) | |
| return res | |
| async def classify_status(statement_id: str, current_user: User = Depends(get_current_user)): | |
| """Get classification status for a statement.""" | |
| if statement_id not in statements: | |
| raise HTTPException(status_code=404, detail="Statement not found") | |
| stmt = statements[statement_id] | |
| if settings.extractLast3Months: | |
| classify_candidates = _filter_transactions_by_last_n_months(stmt.transactions) | |
| else: | |
| classify_candidates = list(stmt.transactions) | |
| total = len(classify_candidates) | |
| breakdown = {"rule": 0, "ml": 0, "llm": 0, "manual": 0, "uncategorized": 0} | |
| for tx in classify_candidates: | |
| raw_src = tx.get("categorySource") or tx.get("category_source") or "" | |
| src = raw_src.split(":")[0] if raw_src else "uncategorized" | |
| if src in breakdown: | |
| breakdown[src] += 1 | |
| elif src: | |
| breakdown.setdefault(src, 0) | |
| breakdown[src] += 1 | |
| else: | |
| breakdown["uncategorized"] += 1 | |
| classified = total - breakdown.get("uncategorized", 0) | |
| return { | |
| "total": total, | |
| "classified": classified, | |
| "breakdown": breakdown, | |
| "status": stmt.status | |
| } | |
| async def override_classification( | |
| statement_id: str, | |
| tx_id: str, | |
| payload: CategoryOverride, | |
| background_tasks: BackgroundTasks, | |
| current_user: User = Depends(get_current_user), | |
| ): | |
| """Manually override a transaction's category.""" | |
| if statement_id not in statements: | |
| raise HTTPException(status_code=404, detail="Statement not found") | |
| stmt = statements[statement_id] | |
| for tx in stmt.transactions: | |
| if tx.get("id") == tx_id: | |
| tx["category"] = payload.category | |
| tx["parentCategory"] = payload.category | |
| tx["group_name"] = payload.category | |
| tx["groupName"] = payload.category | |
| tx["sub_category"] = None | |
| tx["subCategory"] = None | |
| tx["categoryConfidence"] = "manual" | |
| tx["category_confidence"] = "manual" | |
| tx["categorySource"] = "manual" | |
| tx["category_source"] = "manual" | |
| background_tasks.add_task(_persist_swap_to_xlsx, statement_id) | |
| return {"status": "success", "transaction": tx} | |
| raise HTTPException(status_code=404, detail="Transaction not found") | |
| async def confirm_classify_suggestion( | |
| statement_id: str, | |
| tx_id: str, | |
| background_tasks: BackgroundTasks, | |
| current_user: User = Depends(get_current_user), | |
| ): | |
| """Verify an AI-suggested classification.""" | |
| if statement_id not in statements: | |
| raise HTTPException(status_code=404, detail="Statement not found") | |
| stmt = statements[statement_id] | |
| target_tx = None | |
| for tx in stmt.transactions: | |
| if tx.get("id") == tx_id: | |
| target_tx = tx | |
| break | |
| if target_tx is None: | |
| raise HTTPException(status_code=404, detail="Transaction not found") | |
| target_desc = target_tx.get("description", "") | |
| target_cat = target_tx.get("category") | |
| verified_ids: List[str] = [] | |
| for tx in stmt.transactions: | |
| raw_src = tx.get("categorySource") or "" | |
| is_ai = raw_src.startswith("ml") or raw_src.startswith("llm") | |
| if not is_ai: | |
| continue | |
| if tx.get("description", "") == target_desc and tx.get("category") == target_cat: | |
| tx["verified"] = True | |
| verified_ids.append(tx["id"]) | |
| if tx_id not in verified_ids: | |
| target_tx["verified"] = True | |
| verified_ids.append(tx_id) | |
| if verified_ids: | |
| background_tasks.add_task(_persist_swap_to_xlsx, statement_id) | |
| bulk_applied = len(verified_ids) > 1 | |
| logger.info( | |
| f"[ConfirmClassify] {statement_id}: verified {len(verified_ids)} tx(s) " | |
| f"matching '{target_desc}' (bulk={bulk_applied})" | |
| ) | |
| return { | |
| "success": True, | |
| "verifiedIds": verified_ids, | |
| "count": len(verified_ids), | |
| "bulkApplied": bulk_applied, | |
| } | |
| async def confirm_classify_suggestions_bulk( | |
| statement_id: str, | |
| payload: BulkConfirmPayload, | |
| background_tasks: BackgroundTasks, | |
| current_user: User = Depends(get_current_user), | |
| ): | |
| """Bulk-verify a specific list of transaction IDs.""" | |
| if statement_id not in statements: | |
| raise HTTPException(status_code=404, detail="Statement not found") | |
| stmt = statements[statement_id] | |
| tx_id_set = set(payload.txIds) | |
| verified_ids: List[str] = [] | |
| for tx in stmt.transactions: | |
| if tx.get("id") in tx_id_set: | |
| tx["verified"] = True | |
| verified_ids.append(tx["id"]) | |
| if verified_ids: | |
| background_tasks.add_task(_persist_swap_to_xlsx, statement_id) | |
| logger.info(f"[ConfirmBulk] {statement_id}: verified {len(verified_ids)} tx(s)") | |
| return { | |
| "success": True, | |
| "verifiedIds": verified_ids, | |
| "count": len(verified_ids), | |
| } | |
| async def reset_classification_override( | |
| statement_id: str, | |
| tx_id: str, | |
| background_tasks: BackgroundTasks, | |
| current_user: User = Depends(get_current_user), | |
| ): | |
| """Reset a transaction's manual category override.""" | |
| if statement_id not in statements: | |
| raise HTTPException(status_code=404, detail="Statement not found") | |
| stmt = statements[statement_id] | |
| for tx in stmt.transactions: | |
| if tx.get("id") == tx_id: | |
| tx["category"] = None | |
| tx["categoryConfidence"] = "unclassified" | |
| tx["category_confidence"] = "unclassified" | |
| tx["categorySource"] = "none" | |
| tx["category_source"] = "none" | |
| background_tasks.add_task(_persist_swap_to_xlsx, statement_id) | |
| return {"status": "success", "transaction": tx} | |
| raise HTTPException(status_code=404, detail="Transaction not found") | |
| async def upload_excel_for_classification( | |
| background_tasks: BackgroundTasks, | |
| file: UploadFile = File(...), | |
| current_user: User = Depends(get_current_user) | |
| ): | |
| """Upload an Excel file directly for classification.""" | |
| try: | |
| content = await file.read() | |
| df = pd.read_excel(content) | |
| statement_id = str(uuid.uuid4()) | |
| transactions = [] | |
| for idx, row in df.iterrows(): | |
| tx = { | |
| "id": str(uuid.uuid4()), | |
| "date": str(row.get("Date", "")), | |
| "description": str(row.get("Description", "")), | |
| "amount": float(row.get("Amount", 0)), | |
| "category": None, | |
| "categoryConfidence": "unclassified", | |
| } | |
| if tx["amount"] < 0: | |
| tx["debit"] = abs(tx["amount"]) | |
| tx["credit"] = None | |
| else: | |
| tx["credit"] = tx["amount"] | |
| tx["debit"] = None | |
| transactions.append(tx) | |
| stmt = StatementData(statement_id, file.filename) | |
| stmt.status = "classifying" | |
| stmt.transactions = transactions | |
| statements[statement_id] = stmt | |
| background_tasks.add_task(run_statement_classification_task, statement_id) | |
| return {"statementId": statement_id} | |
| except Exception as e: | |
| raise HTTPException(status_code=500, detail=str(e)) |