import os import uuid import logging from typing import Optional, List from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks, UploadFile, File import pandas as pd from starlette.concurrency import run_in_threadpool from auth.models import User from auth.service import get_current_user from models.admin_models import FinalisePayload from models.classification import CategoryOverride from models.transactions import BulkConfirmPayload from models.statements import UploadInitResponse from core.state import statements, settings, StatementData from core.helpers import _filter_transactions_by_last_n_months, _normalize_description from core.statement_io import _persist_swap_to_xlsx from core.pipeline import get_classification_pipeline logger = logging.getLogger("befree.classification") router = APIRouter(prefix="/api/v1", tags=["classification"]) async def run_statement_classification_task(statement_id: str, extract_last_3_months: Optional[bool] = None): """Internal helper to run the classification pipeline on a statement in the background.""" try: import logging as std_logging logger = std_logging.getLogger("classification") logger.setLevel(std_logging.INFO) if not logger.handlers: ch = std_logging.StreamHandler() ch.setFormatter(std_logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) logger.addHandler(ch) if not os.environ.get("GOOGLE_API_KEY"): logger.warning("GOOGLE_API_KEY not found in environment. LLM classification will be skipped.") stmt = statements[statement_id] if not stmt.transactions: logger.error(f"No transactions to classify for {statement_id}") return {"error": "No transactions to classify"} use_3_months = extract_last_3_months if extract_last_3_months is not None else settings.extractLast3Months stmt.extract_last_3_months = use_3_months classify_candidates = stmt.transactions if use_3_months: classify_candidates = _filter_transactions_by_last_n_months(stmt.transactions) logger.info( f"Starting classification for statement {statement_id} " f"(extract_last_3_months={use_3_months}, " f"{len(classify_candidates)}/{len(stmt.transactions)} transactions selected for classification)" ) try: from overwatch.event_log import EventLog as _OWEventLog _ow_cls_el = _OWEventLog(document_id=statement_id, document_name=stmt.file_name) except Exception: _ow_cls_el = None pipeline = get_classification_pipeline() if _ow_cls_el is not None: try: pipeline._event_log = _ow_cls_el except Exception: pass classified_subset = await run_in_threadpool( pipeline.classify_batch, classify_candidates ) try: pipeline._event_log = None except Exception: pass for tx in classified_subset: if "category_confidence" in tx: tx["categoryConfidence"] = tx.pop("category_confidence") if "category_source" in tx: tx["categorySource"] = tx.pop("category_source") if "third_party" in tx: tx["thirdParty"] = tx.pop("third_party") if "classification_reason" in tx: tx["classificationReason"] = tx.pop("classification_reason") classified_by_id = {tx.get("id"): tx for tx in classified_subset} merged_transactions = [] for tx in stmt.transactions: tx_id = tx.get("id") if tx_id and tx_id in classified_by_id: merged_transactions.append({**tx, **classified_by_id[tx_id]}) else: merged_transactions.append(tx) stmt.transactions = merged_transactions stmt.status = "completed" logger.info(f"Finished classification for statement {statement_id}") try: _persist_swap_to_xlsx(statement_id) except Exception as e: logger.error(f"Failed to persist classified statement {statement_id}: {e}") counts = {"classified": 0, "rule": 0, "ml": 0, "llm": 0, "fallback": 0, "uncategorized": 0} for tx in stmt.transactions: raw_src = tx.get("categorySource") or tx.get("category_source") or "" src = raw_src.split(":")[0] if raw_src else "uncategorized" if src in ("rule", "ml", "llm", "manual"): counts["classified"] += 1 if src in counts: counts[src] += 1 elif src == "fallback": counts["fallback"] += 1 else: counts["uncategorized"] += 1 return counts except Exception as exc: logger.error(f"Classification failed for {statement_id}: {exc}", exc_info=True) if statement_id in statements: statements[statement_id].status = "failed" return {"error": "Classification failed"} @router.post("/statements/{statement_id}/finalise") async def finalise_statement( statement_id: str, background_tasks: BackgroundTasks, payload: FinalisePayload = FinalisePayload(), current_user: User = Depends(get_current_user), ): """Lock the statement and trigger classification pipeline.""" if statement_id not in statements: raise HTTPException(status_code=404, detail="Statement not found") stmt = statements[statement_id] stmt.status = "classifying" use_3_months = payload.extract_last_3_months if payload.extract_last_3_months is not None else settings.extractLast3Months stmt.extract_last_3_months = use_3_months background_tasks.add_task(run_statement_classification_task, statement_id, use_3_months) return {"status": "classifying", "message": "Classification started in background"} @router.post("/statements/{statement_id}/classify") async def classify_statement(statement_id: str, current_user: User = Depends(get_current_user)): """Run the classification pipeline on all transactions.""" res = await run_statement_classification_task(statement_id) if isinstance(res, dict) and "error" in res: raise HTTPException(status_code=500, detail=res["error"]) return res @router.get("/statements/{statement_id}/classify/status") async def classify_status(statement_id: str, current_user: User = Depends(get_current_user)): """Get classification status for a statement.""" if statement_id not in statements: raise HTTPException(status_code=404, detail="Statement not found") stmt = statements[statement_id] if settings.extractLast3Months: classify_candidates = _filter_transactions_by_last_n_months(stmt.transactions) else: classify_candidates = list(stmt.transactions) total = len(classify_candidates) breakdown = {"rule": 0, "ml": 0, "llm": 0, "manual": 0, "uncategorized": 0} for tx in classify_candidates: raw_src = tx.get("categorySource") or tx.get("category_source") or "" src = raw_src.split(":")[0] if raw_src else "uncategorized" if src in breakdown: breakdown[src] += 1 elif src: breakdown.setdefault(src, 0) breakdown[src] += 1 else: breakdown["uncategorized"] += 1 classified = total - breakdown.get("uncategorized", 0) return { "total": total, "classified": classified, "breakdown": breakdown, "status": stmt.status } @router.patch("/statements/{statement_id}/classify/{tx_id}") async def override_classification( statement_id: str, tx_id: str, payload: CategoryOverride, background_tasks: BackgroundTasks, current_user: User = Depends(get_current_user), ): """Manually override a transaction's category.""" if statement_id not in statements: raise HTTPException(status_code=404, detail="Statement not found") stmt = statements[statement_id] for tx in stmt.transactions: if tx.get("id") == tx_id: tx["category"] = payload.category tx["parentCategory"] = payload.category tx["group_name"] = payload.category tx["groupName"] = payload.category tx["sub_category"] = None tx["subCategory"] = None tx["categoryConfidence"] = "manual" tx["category_confidence"] = "manual" tx["categorySource"] = "manual" tx["category_source"] = "manual" background_tasks.add_task(_persist_swap_to_xlsx, statement_id) return {"status": "success", "transaction": tx} raise HTTPException(status_code=404, detail="Transaction not found") @router.post("/statements/{statement_id}/classify/{tx_id}/confirm") async def confirm_classify_suggestion( statement_id: str, tx_id: str, background_tasks: BackgroundTasks, current_user: User = Depends(get_current_user), ): """Verify an AI-suggested classification.""" if statement_id not in statements: raise HTTPException(status_code=404, detail="Statement not found") stmt = statements[statement_id] target_tx = None for tx in stmt.transactions: if tx.get("id") == tx_id: target_tx = tx break if target_tx is None: raise HTTPException(status_code=404, detail="Transaction not found") target_desc = target_tx.get("description", "") target_cat = target_tx.get("category") verified_ids: List[str] = [] for tx in stmt.transactions: raw_src = tx.get("categorySource") or "" is_ai = raw_src.startswith("ml") or raw_src.startswith("llm") if not is_ai: continue if tx.get("description", "") == target_desc and tx.get("category") == target_cat: tx["verified"] = True verified_ids.append(tx["id"]) if tx_id not in verified_ids: target_tx["verified"] = True verified_ids.append(tx_id) if verified_ids: background_tasks.add_task(_persist_swap_to_xlsx, statement_id) bulk_applied = len(verified_ids) > 1 logger.info( f"[ConfirmClassify] {statement_id}: verified {len(verified_ids)} tx(s) " f"matching '{target_desc}' (bulk={bulk_applied})" ) return { "success": True, "verifiedIds": verified_ids, "count": len(verified_ids), "bulkApplied": bulk_applied, } @router.post("/statements/{statement_id}/classify/confirm-bulk") async def confirm_classify_suggestions_bulk( statement_id: str, payload: BulkConfirmPayload, background_tasks: BackgroundTasks, current_user: User = Depends(get_current_user), ): """Bulk-verify a specific list of transaction IDs.""" if statement_id not in statements: raise HTTPException(status_code=404, detail="Statement not found") stmt = statements[statement_id] tx_id_set = set(payload.txIds) verified_ids: List[str] = [] for tx in stmt.transactions: if tx.get("id") in tx_id_set: tx["verified"] = True verified_ids.append(tx["id"]) if verified_ids: background_tasks.add_task(_persist_swap_to_xlsx, statement_id) logger.info(f"[ConfirmBulk] {statement_id}: verified {len(verified_ids)} tx(s)") return { "success": True, "verifiedIds": verified_ids, "count": len(verified_ids), } @router.delete("/statements/{statement_id}/classify/{tx_id}/override") async def reset_classification_override( statement_id: str, tx_id: str, background_tasks: BackgroundTasks, current_user: User = Depends(get_current_user), ): """Reset a transaction's manual category override.""" if statement_id not in statements: raise HTTPException(status_code=404, detail="Statement not found") stmt = statements[statement_id] for tx in stmt.transactions: if tx.get("id") == tx_id: tx["category"] = None tx["categoryConfidence"] = "unclassified" tx["category_confidence"] = "unclassified" tx["categorySource"] = "none" tx["category_source"] = "none" background_tasks.add_task(_persist_swap_to_xlsx, statement_id) return {"status": "success", "transaction": tx} raise HTTPException(status_code=404, detail="Transaction not found") @router.post("/classify/upload-excel") async def upload_excel_for_classification( background_tasks: BackgroundTasks, file: UploadFile = File(...), current_user: User = Depends(get_current_user) ): """Upload an Excel file directly for classification.""" try: content = await file.read() df = pd.read_excel(content) statement_id = str(uuid.uuid4()) transactions = [] for idx, row in df.iterrows(): tx = { "id": str(uuid.uuid4()), "date": str(row.get("Date", "")), "description": str(row.get("Description", "")), "amount": float(row.get("Amount", 0)), "category": None, "categoryConfidence": "unclassified", } if tx["amount"] < 0: tx["debit"] = abs(tx["amount"]) tx["credit"] = None else: tx["credit"] = tx["amount"] tx["debit"] = None transactions.append(tx) stmt = StatementData(statement_id, file.filename) stmt.status = "classifying" stmt.transactions = transactions statements[statement_id] = stmt background_tasks.add_task(run_statement_classification_task, statement_id) return {"statementId": statement_id} except Exception as e: raise HTTPException(status_code=500, detail=str(e))