import os import io import csv import uuid import logging import smtplib from pathlib import Path from datetime import datetime from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.mime.application import MIMEApplication from typing import Optional, List, Dict, Any from fastapi import APIRouter, Depends, HTTPException, Query from fastapi.responses import StreamingResponse from auth.models import User, AuditLog from auth.service import get_current_user from core.state import ( statements, discover_and_load_statements, _uploads_dir, ) from core.pipeline import ( get_classification_pipeline, reset_classification_pipeline, ) from models.admin_models import ( OverwatchEmailRequest, CategoryCreateRequest, CategoryUpdateRequest, ) logger = logging.getLogger("befree.admin") router = APIRouter(prefix="/api/v1/admin", tags=["admin"]) def _get_classification_logs_list() -> List[Dict[str, Any]]: """Helper to parse all entries from classification_latest.jsonl.""" log_path = Path("data/logs/classification_latest.jsonl") if not log_path.exists(): return [] entries = [] try: with open(log_path, "r", encoding="utf-8") as f: for line in f: line = line.strip() if not line: continue try: entries.append(json.loads(line)) except Exception: pass except Exception as e: logger.error(f"Error reading classification log: {e}") return entries @router.get("/audit-logs") async def get_audit_logs( user_id: Optional[str] = None, action: Optional[str] = None, from_date: Optional[str] = None, to_date: Optional[str] = None, page: int = Query(1, ge=1), page_size: int = Query(50, ge=1, le=200), current_user: User = Depends(get_current_user), ): """Paginated audit log viewer — admin only.""" if current_user.role != "admin": raise HTTPException(status_code=403, detail="Admin access required") from auth.database import SessionLocal as _SL _db = _SL() try: q = _db.query(AuditLog) if user_id: q = q.filter(AuditLog.user_id == user_id) if action: q = q.filter(AuditLog.action == action) if from_date: q = q.filter(AuditLog.occurred_at >= datetime.fromisoformat(from_date)) if to_date: q = q.filter(AuditLog.occurred_at <= datetime.fromisoformat(to_date)) total = q.count() logs = q.order_by(AuditLog.occurred_at.desc()).offset((page - 1) * page_size).limit(page_size).all() items = [ { "id": log.id, "user_id": log.user_id, "username": log.username, "action": log.action, "resource_type": log.resource_type, "resource_id": log.resource_id, "http_method": log.http_method, "endpoint": log.endpoint, "response_status": log.response_status, "ip_address": log.ip_address, "duration_ms": log.duration_ms, "occurred_at": log.occurred_at.isoformat() if log.occurred_at else None, } for log in logs ] return {"items": items, "total": total, "page": page, "page_size": page_size} finally: _db.close() @router.get("/overwatch/insights") async def get_overwatch_insights( current_user: User = Depends(get_current_user), ): """Admin-only overview insights for UAT supervision.""" if current_user.role != "admin": raise HTTPException(status_code=403, detail="Admin access required") try: discover_and_load_statements() except Exception: pass all_stmts = list(statements.values()) total_pdfs = len(all_stmts) total_txs = sum(len(s.transactions) for s in all_stmts if getattr(s, "transactions", None)) status_counts = {} bank_counts = {} for s in all_stmts: status = getattr(s, "status", "unknown") status_counts[status] = status_counts.get(status, 0) + 1 bank_name = getattr(s, "bank_name", None) or "Unknown" bank_counts[bank_name] = bank_counts.get(bank_name, 0) + 1 class_logs = _get_classification_logs_list() total_classified = len(class_logs) engine_counts = {} total_tokens = 0 for entry in class_logs: engine = entry.get("source") or "unknown" engine_counts[engine] = engine_counts.get(engine, 0) + 1 total_tokens += entry.get("llm_prompt_tokens") or 0 est_cost = round((total_tokens / 1_000_000) * 0.15, 4) from auth.database import SessionLocal as _SL db = _SL() user_activity = {} active_users = set() total_api_calls = 0 try: logs = db.query(AuditLog).all() total_api_calls = len(logs) for log in logs: uname = log.username or "anonymous" active_users.add(uname) if uname not in user_activity: user_activity[uname] = {} act = log.action user_activity[uname][act] = user_activity[uname].get(act, 0) + 1 except Exception as e: logger.error(f"Error querying audit logs for insights: {e}") finally: db.close() formatted_user_activity = [ {"username": uname, "actions": actions, "total": sum(actions.values())} for uname, actions in user_activity.items() ] formatted_user_activity.sort(key=lambda x: x["total"], reverse=True) return { "pdfs_uploaded": total_pdfs, "transactions_classified": total_txs, "status_distribution": status_counts, "bank_distribution": bank_counts, "classification_logs_count": total_classified, "engine_distribution": engine_counts, "llm_prompt_tokens": total_tokens, "estimated_llm_cost_usd": est_cost, "active_users_count": len(active_users), "total_api_calls": total_api_calls, "user_activity": formatted_user_activity, } @router.get("/uploads/pdfs") async def list_uploaded_pdfs( current_user: User = Depends(get_current_user), ): """Admin-only: list PDFs currently present on disk in the uploads directory.""" if current_user.role != "admin": raise HTTPException(status_code=403, detail="Admin access required") uploads_dir = _uploads_dir().resolve() uploads_dir.mkdir(parents=True, exist_ok=True) pdf_files = sorted(uploads_dir.glob("*.pdf"), key=lambda p: p.name.lower()) items = [] for p in pdf_files: statement_id = None original_filename = p.name parts = p.stem.split("_", 2) if len(parts) == 3 and parts[1].isdigit(): statement_id = parts[0] original_filename = parts[2] + p.suffix view_url = f"/api/v1/statements/{statement_id}/pdf" if statement_id else None items.append( { "stored_name": p.name, "original_filename": original_filename, "statement_id": statement_id, "size_bytes": p.stat().st_size, "view_url": view_url, } ) return { "uploads_dir": str(uploads_dir), "count": len(items), "items": items, } @router.get("/uploads/pdfs-runtime") async def list_uploaded_pdfs_runtime( current_user: User = Depends(get_current_user), ): """Admin-only: list PDFs currently known to the running process.""" if current_user.role != "admin": raise HTTPException(status_code=403, detail="Admin access required") uploads_dir = _uploads_dir().resolve() items = [] for stmt in statements.values(): pdf_paths = getattr(stmt, "pdf_paths", []) or [] if not pdf_paths and getattr(stmt, "pdf_path", None): pdf_paths = [stmt.pdf_path] if not pdf_paths: continue originals = list(getattr(stmt, "original_file_names", []) or []) files = [] for i, p in enumerate(pdf_paths): pth = Path(str(p)).resolve() exists = pth.exists() stored_name = pth.name original_filename = None if i < len(originals): original_filename = originals[i] else: stem_parts = pth.stem.split("_", 2) if len(stem_parts) == 3 and stem_parts[1].isdigit(): original_filename = stem_parts[2] + pth.suffix inside_uploads_dir = str(pth).startswith(str(uploads_dir)) files.append( { "idx": i, "stored_name": stored_name, "original_filename": original_filename or stored_name, "exists_on_disk": exists, "inside_uploads_dir": inside_uploads_dir, } ) items.append( { "statement_id": stmt.id, "merged_display_name": getattr(stmt, "file_name", None), "uploaded_at": getattr(stmt, "created_at", None), "file_count": len(files), "files": files, "view_merged_url": f"/api/v1/statements/{stmt.id}/pdf", } ) items.sort(key=lambda x: x.get("uploaded_at") or "", reverse=True) return { "uploads_dir": str(uploads_dir), "count": len(items), "items": items, } @router.get("/overwatch/logs") async def get_overwatch_logs( log_type: str = Query("audit", pattern="^(audit|classification)$"), username: Optional[str] = None, action: Optional[str] = None, category: Optional[str] = None, engine: Optional[str] = None, from_date: Optional[str] = None, to_date: Optional[str] = None, search: Optional[str] = None, page: int = Query(1, ge=1), page_size: int = Query(50, ge=1, le=200), current_user: User = Depends(get_current_user), ): """Admin-only route to search UAT logs.""" if current_user.role != "admin": raise HTTPException(status_code=403, detail="Admin access required") if log_type == "audit": from auth.database import SessionLocal as _SL db = _SL() try: q = db.query(AuditLog) if username: q = q.filter(AuditLog.username.ilike(f"%{username}%")) if action: q = q.filter(AuditLog.action == action) if from_date: q = q.filter(AuditLog.occurred_at >= datetime.fromisoformat(from_date)) if to_date: q = q.filter(AuditLog.occurred_at <= datetime.fromisoformat(to_date)) if search: q = q.filter( (AuditLog.endpoint.ilike(f"%{search}%")) | (AuditLog.resource_type.ilike(f"%{search}%")) | (AuditLog.resource_id.ilike(f"%{search}%")) ) total = q.count() logs = q.order_by(AuditLog.occurred_at.desc()).offset((page - 1) * page_size).limit(page_size).all() items = [ { "id": log.id, "user_id": log.user_id, "username": log.username, "action": log.action, "resource_type": log.resource_type, "resource_id": log.resource_id, "http_method": log.http_method, "endpoint": log.endpoint, "response_status": log.response_status, "ip_address": log.ip_address, "duration_ms": log.duration_ms, "occurred_at": log.occurred_at.isoformat() if log.occurred_at else None, } for log in logs ] return {"items": items, "total": total, "page": page, "page_size": page_size} finally: db.close() else: all_logs = _get_classification_logs_list() filtered = [] for entry in all_logs: logged_at_str = entry.get("logged_at", "") if from_date: if logged_at_str < from_date: continue if to_date: t_date_str = to_date if "T" in to_date else f"{to_date}T23:59:59" if logged_at_str > t_date_str: continue if category and category.lower() not in (entry.get("category") or "").lower(): continue if engine and engine.lower() != (entry.get("source") or "").lower(): continue if search: s_lower = search.lower() desc = (entry.get("description") or "").lower() cat = (entry.get("category") or "").lower() kw = (entry.get("matched_keyword") or "").lower() rname = (entry.get("matched_rule_name") or "").lower() if s_lower not in desc and s_lower not in cat and s_lower not in kw and s_lower not in rname: continue filtered.append(entry) filtered.sort(key=lambda x: x.get("logged_at", ""), reverse=True) total = len(filtered) start_idx = (page - 1) * page_size end_idx = start_idx + page_size paginated = filtered[start_idx:end_idx] return {"items": paginated, "total": total, "page": page, "page_size": page_size} @router.get("/overwatch/export") async def export_overwatch_logs( log_type: str = Query("audit", pattern="^(audit|classification)$"), username: Optional[str] = None, action: Optional[str] = None, category: Optional[str] = None, engine: Optional[str] = None, from_date: Optional[str] = None, to_date: Optional[str] = None, search: Optional[str] = None, current_user: User = Depends(get_current_user), ): """Admin-only route to download UAT logs as CSV.""" if current_user.role != "admin": raise HTTPException(status_code=403, detail="Admin access required") csv_io = io.StringIO() writer = csv.writer(csv_io) if log_type == "audit": writer.writerow(["ID", "Timestamp", "User ID", "Username", "Action", "Resource Type", "Resource ID", "Method", "Endpoint", "Status", "IP Address", "Duration (ms)"]) from auth.database import SessionLocal as _SL db = _SL() try: q = db.query(AuditLog) if username: q = q.filter(AuditLog.username.ilike(f"%{username}%")) if action: q = q.filter(AuditLog.action == action) if from_date: q = q.filter(AuditLog.occurred_at >= datetime.fromisoformat(from_date)) if to_date: q = q.filter(AuditLog.occurred_at <= datetime.fromisoformat(to_date)) if search: q = q.filter( (AuditLog.endpoint.ilike(f"%{search}%")) | (AuditLog.resource_type.ilike(f"%{search}%")) | (AuditLog.resource_id.ilike(f"%{search}%")) ) logs = q.order_by(AuditLog.occurred_at.desc()).all() for log in logs: writer.writerow([ log.id, log.occurred_at.isoformat() if log.occurred_at else "", log.user_id or "", log.username or "", log.action or "", log.resource_type or "", log.resource_id or "", log.http_method or "", log.endpoint or "", log.response_status or "", log.ip_address or "", log.duration_ms or "" ]) finally: db.close() else: writer.writerow(["Transaction ID", "Logged At", "Transaction Date", "Description", "Amount", "Type", "Category", "Engine Source", "Confidence", "Matched Keyword", "Matched Rule", "Prompt Tokens", "Engines Tried", "Engines Failed"]) all_logs = _get_classification_logs_list() for entry in all_logs: logged_at_str = entry.get("logged_at", "") if from_date and logged_at_str < from_date: continue if to_date: t_date_str = to_date if "T" in to_date else f"{to_date}T23:59:59" if logged_at_str > t_date_str: continue if category and category.lower() not in (entry.get("category") or "").lower(): continue if engine and engine.lower() != (entry.get("source") or "").lower(): continue if search: s_lower = search.lower() desc = (entry.get("description") or "").lower() cat = (entry.get("category") or "").lower() kw = (entry.get("matched_keyword") or "").lower() rname = (entry.get("matched_rule_name") or "").lower() if s_lower not in desc and s_lower not in cat and s_lower not in kw and s_lower not in rname: continue writer.writerow([ entry.get("tx_id") or "", entry.get("logged_at") or "", entry.get("date") or "", entry.get("description") or "", entry.get("amount") or 0, entry.get("tx_type") or "unknown", entry.get("category") or "", entry.get("source") or "", entry.get("confidence") or "", entry.get("matched_keyword") or "", entry.get("matched_rule_name") or "", entry.get("llm_prompt_tokens") or "", ", ".join(entry.get("engines_tried") or []), ", ".join(entry.get("engines_failed") or []) ]) csv_io.seek(0) response_headers = { 'Content-Disposition': f'attachment; filename="overwatch_{log_type}_report_{datetime.now().strftime("%Y-%m-%d")}.csv"' } return StreamingResponse( io.BytesIO(csv_io.getvalue().encode("utf-8")), media_type="text/csv", headers=response_headers ) @router.post("/overwatch/email-report") async def email_overwatch_report( req: OverwatchEmailRequest, current_user: User = Depends(get_current_user), ): """Admin-only endpoint to send daily UAT logs via email with attachments.""" if current_user.role != "admin": raise HTTPException(status_code=403, detail="Admin access required") attachments = [] def _gen_csv(ltype: str) -> str: csv_io = io.StringIO() writer = csv.writer(csv_io) if ltype == "audit": writer.writerow(["ID", "Timestamp", "User ID", "Username", "Action", "Resource Type", "Resource ID", "Method", "Endpoint", "Status", "IP Address", "Duration (ms)"]) from auth.database import SessionLocal as _SL db = _SL() try: q = db.query(AuditLog) if req.username: q = q.filter(AuditLog.username.ilike(f"%{req.username}%")) if req.from_date: q = q.filter(AuditLog.occurred_at >= datetime.fromisoformat(req.from_date)) if req.to_date: q = q.filter(AuditLog.occurred_at <= datetime.fromisoformat(req.to_date)) logs = q.order_by(AuditLog.occurred_at.desc()).all() for log in logs: writer.writerow([ log.id, log.occurred_at.isoformat() if log.occurred_at else "", log.user_id or "", log.username or "", log.action or "", log.resource_type or "", log.resource_id or "", log.http_method or "", log.endpoint or "", log.response_status or "", log.ip_address or "", log.duration_ms or "" ]) finally: db.close() else: writer.writerow(["Transaction ID", "Logged At", "Transaction Date", "Description", "Amount", "Type", "Category", "Engine Source", "Confidence", "Matched Keyword", "Matched Rule", "Prompt Tokens", "Engines Tried", "Engines Failed"]) all_logs = _get_classification_logs_list() for entry in all_logs: logged_at_str = entry.get("logged_at", "") if req.from_date and logged_at_str < req.from_date: continue if req.to_date: t_date_str = req.to_date if "T" in req.to_date else f"{req.to_date}T23:59:59" if logged_at_str > t_date_str: continue if req.category and req.category.lower() not in (entry.get("category") or "").lower(): continue if req.engine and req.engine.lower() != (entry.get("source") or "").lower(): continue writer.writerow([ entry.get("tx_id") or "", entry.get("logged_at") or "", entry.get("date") or "", entry.get("description") or "", entry.get("amount") or 0, entry.get("tx_type") or "unknown", entry.get("category") or "", entry.get("source") or "", entry.get("confidence") or "", entry.get("matched_keyword") or "", entry.get("matched_rule_name") or "", entry.get("llm_prompt_tokens") or "", ", ".join(entry.get("engines_tried") or []), ", ".join(entry.get("engines_failed") or []) ]) return csv_io.getvalue() ltypes = [] if req.log_type in ("audit", "both"): ltypes.append("audit") if req.log_type in ("classification", "both"): ltypes.append("classification") for lt in ltypes: csv_data = _gen_csv(lt) filename = f"overwatch_{lt}_logs_{datetime.now().strftime('%Y-%m-%d')}.csv" attachments.append((filename, csv_data)) subject = f"BeFree UAT Overwatch Logs Report - {datetime.now().strftime('%Y-%m-%d')}" body_text = f"Hello,\n\nPlease find attached the requested UAT logs report.\n\nFilters applied:\n" body_text += f"- Date Range: {req.from_date or 'Beginning'} to {req.to_date or 'Now'}\n" if req.username: body_text += f"- User: {req.username}\n" if req.category: body_text += f"- Category: {req.category}\n" if req.engine: body_text += f"- Engine: {req.engine}\n" body_text += f"\nBest regards,\nBeFree System Watcher" smtp_host = os.getenv("SMTP_HOST") smtp_port = os.getenv("SMTP_PORT") smtp_user = os.getenv("SMTP_USER") smtp_pass = os.getenv("SMTP_PASSWORD") smtp_from = os.getenv("SMTP_FROM", "no-reply@befree.com") if smtp_host and smtp_port: try: msg = MIMEMultipart() msg["Subject"] = subject msg["From"] = smtp_from msg["To"] = req.email msg.attach(MIMEText(body_text, "plain")) for fname, csv_str in attachments: part = MIMEApplication(csv_str.encode("utf-8"), Name=fname) part['Content-Disposition'] = f'attachment; filename="{fname}"' msg.attach(part) port_val = int(smtp_port) server = smtplib.SMTP(smtp_host, port_val) if port_val == 587: server.starttls() if smtp_user and smtp_pass: server.login(smtp_user, smtp_pass) server.sendmail(smtp_from, req.email.split(","), msg.as_string()) server.quit() return {"status": "sent", "message": f"Logs successfully emailed to {req.email}"} except Exception as smtp_err: logger.error(f"Failed to send email via SMTP: {smtp_err}") sim_dir = Path("data/simulated_emails") sim_dir.mkdir(parents=True, exist_ok=True) sim_file = sim_dir / f"email_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:6]}.json" sim_data = { "to": req.email, "from": smtp_from, "subject": subject, "body": body_text, "attachments": [ {"filename": fname, "byte_size": len(csv_str), "snippet": csv_str[:200] + "..." if len(csv_str) > 200 else csv_str} for fname, csv_str in attachments ] } for fname, csv_str in attachments: with open(sim_dir / f"attachment_{fname}", "w", encoding="utf-8") as att_f: att_f.write(csv_str) with open(sim_file, "w", encoding="utf-8") as f: json.dump(sim_data, f, indent=2) return { "status": "simulated", "message": f"SMTP credentials not configured in .env or failed. Simulated email generated in folder data/simulated_emails.", "simulated_path": str(sim_file) } @router.get("/categories") async def list_categories( include_inactive: bool = False, current_user: User = Depends(get_current_user), ): """List all classification categories.""" from auth.database import SessionLocal as _SL from auth.rule_models import ClassificationCategory _db = _SL() try: q = _db.query(ClassificationCategory) if current_user.role != "admin" or not include_inactive: q = q.filter(ClassificationCategory.is_active == True) cats = q.order_by(ClassificationCategory.name).all() return {"items": [c.to_dict() for c in cats], "total": len(cats)} finally: _db.close() @router.post("/categories") async def create_category( payload: CategoryCreateRequest, current_user: User = Depends(get_current_user), ): """Create a new classification category. Admin only.""" if current_user.role != "admin": raise HTTPException(status_code=403, detail="Admin access required") from auth.database import SessionLocal as _SL from auth.rule_models import ClassificationCategory _db = _SL() try: existing = _db.query(ClassificationCategory).filter_by(name=payload.name).first() if existing: raise HTTPException(status_code=409, detail=f"Category '{payload.name}' already exists") cat = ClassificationCategory( name=payload.name, sheet_name=payload.sheet_name, sub_column=payload.sub_column, created_by=current_user.username, updated_by=current_user.username, ) _db.add(cat) _db.commit() _db.refresh(cat) return cat.to_dict() except HTTPException: raise except Exception as e: _db.rollback() raise HTTPException(status_code=500, detail=str(e)) finally: _db.close() @router.patch("/categories/{category_id}") async def update_category( category_id: str, payload: CategoryUpdateRequest, current_user: User = Depends(get_current_user), ): """Update a category's name, sheet mapping, or active state. Admin only.""" if current_user.role != "admin": raise HTTPException(status_code=403, detail="Admin access required") from auth.database import SessionLocal as _SL from auth.rule_models import ClassificationCategory _db = _SL() try: cat = _db.query(ClassificationCategory).filter_by(id=category_id).first() if not cat: raise HTTPException(status_code=404, detail="Category not found") if payload.name is not None: clash = _db.query(ClassificationCategory).filter( ClassificationCategory.name == payload.name, ClassificationCategory.id != category_id, ).first() if clash: raise HTTPException(status_code=409, detail=f"Category name '{payload.name}' already in use") cat.name = payload.name if payload.sheet_name is not None: cat.sheet_name = payload.sheet_name if payload.sub_column is not None: cat.sub_column = payload.sub_column if payload.is_active is not None: cat.is_active = payload.is_active cat.updated_by = current_user.username _db.commit() _db.refresh(cat) return cat.to_dict() except HTTPException: raise except Exception as e: _db.rollback() raise HTTPException(status_code=500, detail=str(e)) finally: _db.close() @router.delete("/categories/{category_id}") async def delete_category( category_id: str, current_user: User = Depends(get_current_user), ): """Soft-delete a category. Admin only.""" if current_user.role != "admin": raise HTTPException(status_code=403, detail="Admin access required") from auth.database import SessionLocal as _SL from auth.rule_models import ClassificationCategory _db = _SL() try: cat = _db.query(ClassificationCategory).filter_by(id=category_id).first() if not cat: raise HTTPException(status_code=404, detail="Category not found") cat.is_active = False cat.updated_by = current_user.username _db.commit() return {"ok": True} except HTTPException: raise except Exception as e: _db.rollback() raise HTTPException(status_code=500, detail=str(e)) finally: _db.close() @router.post("/reset-pipeline") async def admin_reset_pipeline(current_user: User = Depends(get_current_user)): """Force-reinitialise the classification pipeline. Admin only.""" if current_user.role != "admin": raise HTTPException(status_code=403, detail="Admin only") pipeline = reset_classification_pipeline() llm_ok = ( pipeline._llm_engine is not None and pipeline._llm_engine.enabled ) return { "status": "reset", "engines": [e.name for e in pipeline.engines], "llm_enabled": llm_ok, } @router.get("/llm-status") async def admin_llm_status(current_user: User = Depends(get_current_user)): """Return LLM engine health info. Admin only.""" if current_user.role != "admin": raise HTTPException(status_code=403, detail="Admin only") pipeline = get_classification_pipeline() llm = pipeline._llm_engine key_set = bool(os.environ.get("GOOGLE_API_KEY")) key_looks_valid = os.environ.get("GOOGLE_API_KEY", "").startswith("AIzaSy") return { "llm_engine_present": llm is not None, "llm_enabled": llm.enabled if llm else False, "llm_search_fallback_enabled": ( llm is not None and llm._fallback_config is not None ), "active_engines": [e.name for e in pipeline.engines], "google_api_key_set": key_set, "google_api_key_format_ok": key_looks_valid, }