backend/app/main.py

"""FastAPI app — the API surface for the dashboard.

Covers the full retail back-office story: pluggable OCR (MiniCPM / Cohere / LlamaParse),
document categorization, the hybrid IDP pipeline (HITL resume), web automation, an
actual SQLite + vector RAG datastore, business KPIs / observability, prompt management,
and an admin dashboard. Runs offline by default; upgrades when keys/deps/endpoints exist.
"""
from __future__ import annotations

import json
import shutil
import tempfile
from pathlib import Path

from fastapi import Depends, FastAPI, HTTPException, Request, UploadFile
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from pydantic import BaseModel

from .auth import _check, make_auth_middleware
from .browser import fill_order, run_browser_agent
from .caching import SemanticCache
from .categories import list_categories
from .config import get_settings
from .db import Database
from .metrics import MetricsStore
from .observability import (
    business_kpis,
    install_log_handler,
    list_snapshots,
    publish_offline_snapshot,
    recent_logs,
)
from .ocr.backends import build_ocr_registry
from .pipeline import process_document, resume_run
from .providers import build_registry
from .providers.pricing import PRICES
from .rag import KB
from .rag_store import VectorStore
from .router import ModelRouter

settings = get_settings()
registry = build_registry(settings)
metrics = MetricsStore(settings.metrics_db_path)
semantic_cache = SemanticCache(
    threshold=settings.semantic_cache_threshold, enabled=settings.enable_semantic_cache
)
router = ModelRouter(registry, settings, metrics, semantic_cache)
ocr_registry = build_ocr_registry(settings)
db = Database(settings.app_db_path)
rag_store = VectorStore(settings.rag_db_path)
install_log_handler()
db.audit("service_start", actor="system",
         detail={"ocr_backend": settings.ocr_backend, "tier": registry.capabilities()["active_tier"]})

app = FastAPI(title="Aperture — Open-Source Agentic Automation", version="0.2.0")


@app.middleware("http")
async def _no_cache_html(request, call_next):
    resp = await call_next(request)
    if resp.headers.get("content-type", "").startswith("text/html"):
        resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
    return resp


app.middleware("http")(make_auth_middleware(settings.auth_user, settings.auth_pass))
app.add_middleware(
    CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"],
)

DOC_EXTS = (".pdf", ".png", ".jpg", ".jpeg", ".tif", ".tiff")


# --- request models -----------------------------------------------------------
class ResumeRequest(BaseModel):
    corrected: dict


class OrderFillRequest(BaseModel):
    order: dict
    base_url: str | None = None


class AgentRequest(BaseModel):
    goal: str
    scenario: str = "scrape_orders"
    order: dict | None = None
    base_url: str | None = None


class PromptUpdate(BaseModel):
    content: str


class ErpChatRequest(BaseModel):
    question: str
    use_llm: bool = True


# --- helpers ------------------------------------------------------------------
def require_admin(request: Request) -> bool:
    if not _check(request.headers.get("authorization"), settings.admin_user, settings.admin_pass):
        raise HTTPException(403, "admin credentials required")
    return True


def _actor(request: Request) -> str:
    try:
        import base64
        decoded = base64.b64decode(
            (request.headers.get("authorization") or "Basic ").split(" ", 1)[1]).decode()
        return decoded.split(":", 1)[0] or "anonymous"
    except Exception:
        return "anonymous"


def _sample_path(sample_id: str) -> Path:
    d = settings.evals_dataset_dir
    for ext in DOC_EXTS:
        p = d / f"{sample_id}{ext}"
        if p.exists():
            return p
    raise HTTPException(404, f"sample '{sample_id}' not found (run scripts/generate_samples.py)")


def _list_samples() -> list[dict]:
    from .categories import category_for_doc_type
    d = settings.evals_dataset_dir
    out = []
    for gt in sorted(d.glob("*.gt.json")):
        meta = json.loads(gt.read_text()).get("_meta", {})
        sid = gt.name[: -len(".gt.json")]
        doc_file = next((f"{sid}{e}" for e in DOC_EXTS if (d / f"{sid}{e}").exists()), None)
        out.append({"id": sid, "file": doc_file,
                    "category": category_for_doc_type(meta.get("doc_type")), **meta})
    return out


def _run_pipeline(path, *, doc_id, source="upload", channel=None, difficulty=None,
                  ocr_backend=None, forced_doc_type=None, mode="live"):
    return process_document(
        path, router=router, settings=settings, metrics=metrics,
        ocr_registry=ocr_registry, db=db, rag_store=rag_store,
        doc_id=doc_id, source=source, channel=channel, difficulty=difficulty,
        ocr_backend=ocr_backend, forced_doc_type=forced_doc_type, mode=mode)


# --- core routes --------------------------------------------------------------
@app.get("/health")
def health():
    return {"status": "ok", "version": app.version}


@app.get("/api/login")
def login():
    return {"ok": True, "user": settings.auth_user}


@app.get("/api/capabilities")
def capabilities():
    caps = registry.capabilities()
    caps["semantic_cache_stats"] = semantic_cache.stats()
    caps["rag"] = {"backend": KB.backend(), "records": len(KB.records),
                   "vector_store": rag_store.info()}
    caps["ocr"] = {**caps.get("ocr", {}), "registry": ocr_registry.info()}
    caps["categories"] = list_categories()
    caps["mode"] = settings.mode
    from .models_registry import model_catalog
    mc = model_catalog(settings)
    caps["models"] = {"max_params_b": mc["max_params_b"], "count": mc["count"],
                      "available": mc["available"], "labs": [l["lab"] for l in mc["labs"]],
                      "reasoning_capable": mc.get("reasoning_capable", [])}
    try:
        from .erp import get_warehouse
        caps["erp"] = {"enabled": True, "tables": get_warehouse(settings).table_counts()}
    except Exception as e:  # never let ERP wiring break capabilities
        caps["erp"] = {"enabled": False, "error": str(e)}
    return caps


@app.get("/api/categories")
def categories():
    return {"categories": list_categories(), "counts": db.category_counts()}


@app.get("/api/ocr/backends")
def ocr_backends():
    return ocr_registry.info()


_ocr_report_cache: dict = {}


@app.get("/api/ocr/test-report")
def ocr_test_report(refresh: bool = False):
    """Run each AVAILABLE OCR backend against real scanned samples and report whether
    it actually extracted the expected text. Cached; pass ?refresh=1 to re-run."""
    from .ocr.backends.healthcheck import run_ocr_backend_tests
    if refresh or not _ocr_report_cache.get("report"):
        _ocr_report_cache["report"] = run_ocr_backend_tests(settings, ocr_registry)
        db.audit("ocr_self_test", detail={
            "functional": _ocr_report_cache["report"]["functional_backends"]})
    return _ocr_report_cache["report"]


@app.get("/api/models")
def models():
    """Enabled small models (≤32B) from OpenBMB, Cohere, Black Forest Labs."""
    from .models_registry import model_catalog
    return model_catalog(settings)


@app.get("/api/ocr/quality-report")
def ocr_quality_report(refresh: bool = False, request: Request = None):
    """OCR output-quality (CER/WER) + document-analysis (field accuracy) per backend.
    Serves the committed/published report; ?refresh=1 re-runs it (admin only)."""
    import json as _json
    if refresh:
        require_admin(request)
        from .ocr.quality import run_ocr_quality
        rep = run_ocr_quality(settings, ocr_registry, router, metrics, db=db, rag_store=rag_store)
        (settings.writable_dir / "ocr_quality_report.json").write_text(_json.dumps(rep))
        db.audit("ocr_quality_published", actor=_actor(request),
                 detail={"best_ocr": rep["best_ocr_quality"]})
        return rep
    for p in (settings.writable_dir / "ocr_quality_report.json",
              settings.eval_report_committed.parent / "ocr_quality_report.json"):
        if p.exists():
            return _json.loads(p.read_text())
    return JSONResponse({"available": False,
                         "message": "run `python scripts/ocr_quality.py`"}, status_code=200)


# --- ERP DocIQ (NLQ / analytics / summary / reasons over the ERP knowledgebase) ---
@app.get("/api/erp/schema")
def erp_schema():
    from .erp import get_warehouse
    from .erp.data import ERP_SCHEMA_DOC, EXAMPLE_QUESTIONS
    wh = get_warehouse(settings)
    return {"schema_doc": ERP_SCHEMA_DOC, "tables": wh.table_counts(),
            "examples": EXAMPLE_QUESTIONS}


@app.get("/api/erp/reports")
def erp_reports():
    """A few canned ERP reports (real data) the chatbot can summarize/explain."""
    from .erp.chat import (_q_spend_by_month, _q_spend_by_category, _q_top_vendors,
                           _q_late_vendors, _q_return_reasons)
    from .erp import get_warehouse
    wh = get_warehouse(settings)
    out = {}
    for name, fn in [("spend_by_month", _q_spend_by_month), ("spend_by_category", _q_spend_by_category),
                     ("top_vendors", _q_top_vendors), ("late_vendors", _q_late_vendors),
                     ("return_reasons", _q_return_reasons)]:
        sql, cols, rows, ans = fn(wh)
        out[name] = {"columns": cols, "rows": rows, "headline": ans, "sql": sql}
    return out


@app.post("/api/erp/chat")
def erp_chat(req: ErpChatRequest, request: Request = None):
    """Ask the ERP DocIQ chatbot: NLQ→SQL, analytics, summary, or 'why' reasoning."""
    from .erp.chat import ErpChat
    from .erp import get_warehouse
    chat = ErpChat(settings, router=router, warehouse=get_warehouse(settings),
                   metrics=metrics, db=db)
    return chat.answer(req.question, use_llm=req.use_llm, run_id=f"erp-{_actor(request)}")


@app.get("/api/erp/finetune-report")
def erp_finetune_report():
    """Latest fine-tune run (offline domain-adaptation demo + MiniCPM LoRA recipe)."""
    import json as _json
    from .config import BACKEND_DIR as _BD
    for p in (settings.writable_dir / "erp_finetune_report.json",
              _BD / "finetune" / "erp_finetune_report.json"):
        if p.exists():
            return _json.loads(p.read_text())
    return JSONResponse({"available": False,
                         "message": "run `python scripts/finetune_erp.py`"}, status_code=200)


@app.get("/api/samples")
def samples():
    return {"samples": _list_samples()}


@app.get("/api/samples/{sample_id}/file")
def sample_file(sample_id: str):
    return FileResponse(_sample_path(sample_id))


@app.post("/api/process")
async def process(request: Request, file: UploadFile | None = None,
                  sample_id: str | None = None, ocr_backend: str | None = None,
                  category: str | None = None):
    """Process an uploaded document OR a named sample. `ocr_backend` selects the OCR
    engine (auto|minicpm|cohere|llamaparse|tesseract|easyocr|sidecar)."""
    actor = _actor(request)
    if sample_id:
        path = _sample_path(sample_id)
        meta = next((s for s in _list_samples() if s["id"] == sample_id), {})
        db.audit("process_sample", actor=actor, detail={"sample_id": sample_id, "ocr_backend": ocr_backend})
        return _run_pipeline(path, doc_id=sample_id, source="sample",
                             channel=meta.get("channel"), difficulty=meta.get("difficulty"),
                             ocr_backend=ocr_backend)
    if file is None:
        raise HTTPException(400, "provide either an uploaded file or sample_id")
    suffix = Path(file.filename or "upload").suffix or ".pdf"
    tmp = Path(tempfile.mkdtemp()) / f"upload{suffix}"
    with tmp.open("wb") as f:
        shutil.copyfileobj(file.file, f)
    db.audit("process_upload", actor=actor,
             detail={"filename": file.filename, "ocr_backend": ocr_backend})
    return _run_pipeline(tmp, doc_id=Path(file.filename or "upload").stem, source="upload",
                         ocr_backend=ocr_backend)


# --- documents (DB-backed) ----------------------------------------------------
@app.get("/api/documents")
def documents(category: str | None = None, limit: int = 100):
    return {"documents": db.list_documents(category=category, limit=limit),
            "counts": db.category_counts()}


@app.get("/api/documents/{run_id}")
def document_detail(run_id: str):
    d = db.get_document(run_id) or metrics.get_run(run_id)
    if not d:
        raise HTTPException(404, "document not found")
    return d


# --- RAG search ---------------------------------------------------------------
@app.get("/api/search")
def search(q: str, k: int = 5, collection: str | None = None):
    return {"query": q, "backend": rag_store.backend,
            "results": rag_store.search(q, k=k, collection=collection)}


@app.get("/api/kb")
def kb_list():
    return {"backend": KB.backend(), "records": KB.records}


@app.get("/api/kb/search")
def kb_search(q: str, k: int = 3):
    return {"query": q, "backend": KB.backend(), "matches": KB.retrieve(q, k=k)}


# --- runs / HITL --------------------------------------------------------------
@app.get("/api/runs")
def runs(limit: int = 25):
    return {"runs": metrics.recent_runs(limit)}


@app.get("/api/runs/{run_id}")
def run_detail(run_id: str):
    r = metrics.get_run(run_id)
    if r is None:
        raise HTTPException(404, "run not found")
    return r


@app.post("/api/runs/{run_id}/resume")
def resume(run_id: str, req: ResumeRequest, request: Request):
    try:
        out = resume_run(run_id, req.corrected, metrics=metrics)
        db.audit("hitl_resume", run_id=run_id, actor=_actor(request),
                 detail={"posted": out.get("result", {}).get("posted")})
        return out
    except KeyError:
        raise HTTPException(404, "run not found")


# --- metrics / cost -----------------------------------------------------------
@app.get("/api/metrics/summary")
def metrics_summary():
    s = metrics.summary()
    s["semantic_cache"] = semantic_cache.stats()
    s["capabilities"] = registry.capabilities()
    return s


@app.post("/api/metrics/reset")
def metrics_reset(request: Request, _: bool = Depends(require_admin)):
    metrics.reset()
    db.audit("metrics_reset", actor=_actor(request))
    return {"status": "reset"}


@app.get("/api/cost-model")
def cost_model():
    return {
        "prices_per_million": {
            k: {"input": v.input, "cached_read": v.cached_read, "output": v.output}
            for k, v in PRICES.items()
        },
        "stacked_reductions": [
            {"strategy": "Prompt caching (static system prompt)", "pct": 31},
            {"strategy": "Model routing (cheap/local for simple tasks)", "pct": 19},
            {"strategy": "Self-hosting embeddings + classification", "pct": 11},
            {"strategy": "History compaction", "pct": 6},
            {"strategy": "Tool-definition trimming", "pct": 3},
            {"strategy": "Loop caps + budget guardrails", "pct": 2},
        ],
        "roi_example": {
            "annual_volume": 40000,
            "uipath_annual_low": 100000, "uipath_annual_high": 250000,
            "aperture_cost_per_doc_low": 0.004, "aperture_cost_per_doc_high": 0.012,
        },
        "live": metrics.summary(),
    }


# --- web automation -----------------------------------------------------------
@app.post("/api/automate/order-fill")
def automate_order_fill(req: OrderFillRequest):
    return fill_order(req.order, base_url=req.base_url or settings.demo_portal_url,
                      headless=settings.playwright_headless)


@app.post("/api/automate/agent")
def automate_agent(req: AgentRequest):
    return run_browser_agent(
        req.goal, router=router, settings=settings, metrics=metrics,
        scenario=req.scenario, order=req.order,
        base_url=req.base_url or settings.demo_portal_url,
        headless=settings.playwright_headless,
    )


# --- evals --------------------------------------------------------------------
@app.get("/api/evals")
def evals_report():
    for p in (settings.eval_report_writable, settings.eval_report_committed):
        if p.exists():
            return {"available": True, **json.loads(p.read_text())}
    return JSONResponse(
        {"available": False, "message": "run `python -m evals.run` to generate a report"},
        status_code=200)


@app.post("/api/evals/run")
def evals_run(policy: str | None = None):
    from evals.run import run_suite
    return {"available": True, **run_suite(policy=policy)}


# --- ADMIN observability (role-gated) -----------------------------------------
@app.get("/api/admin/kpis")
def admin_kpis(_: bool = Depends(require_admin)):
    return {"kpis": business_kpis(db, metrics), "inference": metrics.summary(),
            "category_counts": db.category_counts()}


@app.get("/api/admin/audit")
def admin_audit(limit: int = 100, _: bool = Depends(require_admin)):
    return {"audit": db.recent_audit(limit)}


@app.get("/api/admin/logs")
def admin_logs(limit: int = 200, level: str | None = None, _: bool = Depends(require_admin)):
    return {"logs": recent_logs(limit, level)}


@app.get("/api/admin/secrets")
def admin_secrets(_: bool = Depends(require_admin)):
    """Report SECRET STATUS only — never the values (masked)."""
    def mask(v): return ("set ••••" + v[-4:]) if v else "— not set"
    return {"secrets": [
        {"name": "ANTHROPIC_API_KEY", "status": mask(settings.anthropic_api_key)},
        {"name": "GEMINI_API_KEY", "status": mask(settings.gemini_api_key)},
        {"name": "MINICPM_API_KEY", "status": mask(settings.minicpm_api_key)},
        {"name": "MINICPM_BASE_URL", "status": settings.minicpm_base_url or "— not set"},
        {"name": "LLAMA_CLOUD_API_KEY", "status": mask(settings.llama_cloud_api_key)},
        {"name": "APERTURE_PASS", "status": "set ••••"},
    ], "note": "Values are never returned. Use Vault / cloud secret manager in production."}


@app.get("/api/admin/auth")
def admin_auth(_: bool = Depends(require_admin)):
    return {"scheme": "HTTP Basic over TLS", "api_user": settings.auth_user,
            "admin_user": settings.admin_user,
            "roles": ["user", "admin"],
            "note": "Production: SSO/OIDC + RBAC; this demo uses Basic auth."}


@app.get("/api/admin/prompts")
def admin_prompts(_: bool = Depends(require_admin)):
    return {"prompts": db.list_prompts()}


@app.get("/api/admin/prompts/{name}")
def admin_prompt_get(name: str, _: bool = Depends(require_admin)):
    p = db.get_prompt(name)
    if not p:
        raise HTTPException(404, "prompt not found")
    return {**p, "history": db.prompt_history(name)}


@app.put("/api/admin/prompts/{name}")
def admin_prompt_set(name: str, body: PromptUpdate, request: Request,
                     _: bool = Depends(require_admin)):
    if db.get_prompt(name) is None:
        raise HTTPException(404, "unknown prompt")
    actor = _actor(request)
    out = db.set_prompt(name, body.content, actor=actor)
    db.audit("prompt_updated", actor=actor, detail={"name": name, "version": out["version"]})
    return out


@app.post("/api/admin/publish")
def admin_publish(request: Request, _: bool = Depends(require_admin)):
    out = publish_offline_snapshot(db, metrics, settings.writable_dir / "metrics_snapshots")
    db.audit("metrics_published", actor=_actor(request), detail={"path": out["path"]})
    return out


@app.get("/api/admin/snapshots")
def admin_snapshots(_: bool = Depends(require_admin)):
    return {"snapshots": list_snapshots(settings.writable_dir / "metrics_snapshots")}


# --- static demo portal -------------------------------------------------------
if settings.demo_portal_dir.exists():
    app.mount("/portal", StaticFiles(directory=str(settings.demo_portal_dir), html=True),
              name="portal")

# --- serve the built SPA (mounted LAST) ---------------------------------------
if settings.frontend_dist_dir.exists():
    app.mount("/", StaticFiles(directory=str(settings.frontend_dist_dir), html=True),
              name="spa")