Upload app.py

app.py

@@ -2,20 +2,24 @@ import os
 import time
 import logging
 import json
+import uuid
+import hashlib
+from datetime import datetime
+from typing import Dict, Any, Optional, List
+
 from fastapi import FastAPI, Request, HTTPException, Depends, status
 from fastapi.exceptions import RequestValidationError
 from fastapi.responses import JSONResponse
 from fastapi.security import APIKeyHeader
-from typing import Dict, Any, Optional, List
-from datetime import datetime
+from starlette.responses import Response
 
 # LangGraph and Model Imports
 from langgraph.checkpoint.memory import MemorySaver
 from langgraph.checkpoint.base import BaseCheckpointSaver
 from agent import create_honeypot_graph
 from models import (
-    HoneypotRequest, HoneypotResponse, 
-    AgentState, ExtractedIntelligence, Message, EngagementMetrics
+    HoneypotRequest, HoneypotResponse,
+    AgentState, ExtractedIntelligence
 )
 
 # --- Logging Configuration ---
@@ -23,107 +27,293 @@ logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 # --- Global Debug Store ---
-# This will store the last 50 requests and responses in memory for easy debugging
+# Stores the last N request/response summaries in memory for easy debugging
 DEBUG_LOGS: List[Dict[str, Any]] = []
-MAX_DEBUG_LOGS = 50
+MAX_DEBUG_LOGS = int(os.environ.get("MAX_DEBUG_LOGS", "50"))
+
+# How much of a non-JSON body to preview (chars). Kept small to avoid log bloat.
+MAX_BODY_PREVIEW_CHARS = int(os.environ.get("MAX_BODY_PREVIEW_CHARS", "4000"))
+
+# --- Configuration ---
+API_KEY_NAME = "x-api-key"
+API_KEY = os.environ.get("HONEYPOT_API_KEY", "sk_test_123456789")
+api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=False)
+
+# --- Helpers (safe logging) ---
+def mask_api_key(value: Optional[str]) -> str:
+    """Mask API key while still letting us debug if it was sent."""
+    if not value:
+        return ""
+    v = value.strip()
+    if len(v) <= 8:
+        return "***"
+    return f"{v[:3]}...{v[-4:]}"
+
+def safe_headers(headers: Dict[str, str]) -> Dict[str, str]:
+    """Return headers with sensitive values masked."""
+    out: Dict[str, str] = {}
+    for k, v in headers.items():
+        if k.lower() == API_KEY_NAME:
+            out[k] = mask_api_key(v)
+        else:
+            out[k] = v
+    return out
+
+def summarize_body(raw: bytes, content_type: str) -> Dict[str, Any]:
+    """
+    Safe body summary for debugging endpoint testers:
+    - parsed JSON if possible
+    - else a small preview + sha256 hash + length
+    """
+    if not raw:
+        return {"raw_len": 0, "sha256": None, "parsed": None, "preview": None}
+
+    sha = hashlib.sha256(raw).hexdigest()
+    length = len(raw)
+
+    parsed = None
+    ct = (content_type or "").lower()
+    if "application/json" in ct:
+        try:
+            parsed = json.loads(raw.decode("utf-8", errors="replace"))
+        except Exception:
+            parsed = None
+
+    # For text-only testers, preview is often the fastest way to spot issues.
+    preview = raw.decode("utf-8", errors="replace")
+    if len(preview) > MAX_BODY_PREVIEW_CHARS:
+        preview = preview[:MAX_BODY_PREVIEW_CHARS] + "...(truncated)"
 
-def add_debug_log(direction: str, path: str, method: str, headers: Dict[str, str], body: Any):
+    return {
+        "raw_len": length,
+        "sha256": sha,
+        "parsed": parsed,
+        # If parsed JSON exists, preview is usually redundant.
+        "preview": None if parsed is not None else preview,
+    }
+
+def add_debug_log(
+    direction: str,
+    path: str,
+    method: str,
+    headers: Dict[str, str],
+    body: Any,
+    *,
+    request_id: Optional[str] = None,
+    status_code: Optional[int] = None,
+    query_params: Optional[Dict[str, str]] = None,
+    client: Optional[Dict[str, Any]] = None,
+    extra: Optional[Dict[str, Any]] = None
+) -> None:
     log_entry = {
         "timestamp": datetime.now().isoformat(),
+        "request_id": request_id,
         "direction": direction,
         "path": path,
         "method": method,
-        "headers": {k: v for k, v in headers.items() if k.lower() != "x-api-key"}, # Hide key for safety
-        "body": body
+        "status_code": status_code,
+        "query_params": query_params or {},
+        "client": client or {},
+        "headers": safe_headers(headers),
+        "body": body,
+        "extra": extra or {},
     }
     DEBUG_LOGS.insert(0, log_entry)
     if len(DEBUG_LOGS) > MAX_DEBUG_LOGS:
         DEBUG_LOGS.pop()
 
-# --- Configuration ---
-API_KEY_NAME = "x-api-key"
-API_KEY = os.environ.get("HONEYPOT_API_KEY", "sk_test_123456789")
-api_key_header = APIKeyHeader(name=API_KEY_NAME, auto_error=False)
-
 # --- Initialization ---
 app = FastAPI(
     title="Agentic Honey-Pot API - Super Debug Mode",
-    description="REST API for Scam Detection with enhanced logging.",
-    version="1.3.0"
+    description="REST API for Scam Detection with enhanced request/response logging for endpoint testers.",
+    version="1.4.0"
 )
 
 # Initialize LangGraph Checkpointer
 checkpointer: BaseCheckpointSaver = MemorySaver()
 honeypot_app = create_honeypot_graph(checkpointer)
 
-# --- Middleware for Global Logging ---
+# --- Middleware for Global Request/Response Forensics ---
 @app.middleware("http")
 async def log_requests(request: Request, call_next):
+    # Correlation ID (use tester-provided one if present)
+    request_id = request.headers.get("x-request-id") or str(uuid.uuid4())
+
     path = request.url.path
     method = request.method
     headers = dict(request.headers)
-    
-    # Capture request body
-    body = None
-    if method == "POST":
-        try:
-            raw_body = await request.body()
-            body = json.loads(raw_body)
-        except:
-            body = "Could not parse body as JSON"
-            
-    add_debug_log("INCOMING", path, method, headers, body)
-    logger.info(f"INCOMING {method} {path} | Body: {body}")
+    query_params = dict(request.query_params)
+
+    client_host = request.client.host if request.client else None
+    client_port = request.client.port if request.client else None
+
+    content_type = headers.get("content-type", "")
+    content_length = headers.get("content-length")
+
+    # Read raw body (FastAPI caches it, so downstream can still access it)
+    raw_body = await request.body()
+    body_summary = summarize_body(raw_body, content_type)
+
+    incoming_payload = {
+        "content_type": content_type,
+        "content_length": content_length,
+        "body_summary": body_summary,
+    }
+
+    add_debug_log(
+        "INCOMING",
+        path,
+        method,
+        headers,
+        incoming_payload,
+        request_id=request_id,
+        query_params=query_params,
+        client={"host": client_host, "port": client_port},
+    )
+
+    logger.info(
+        f"[{request_id}] INCOMING {method} {path} "
+        f"qp={query_params} ct={content_type} cl={content_length} "
+        f"client={client_host}:{client_port} "
+        f"api_key_present={API_KEY_NAME in {k.lower() for k in headers.keys()}} "
+        f"api_key_masked={mask_api_key(headers.get(API_KEY_NAME))}"
+    )
 
     start_time = time.time()
+
+    # Call downstream
     response = await call_next(request)
     process_time = time.time() - start_time
 
-    # Capture response body (this is a bit tricky in FastAPI middleware)
-    # For simplicity, we'll log the status code here and log the actual body in the endpoint
-    logger.info(f"OUTGOING {method} {path} | Status: {response.status_code} | Time: {process_time:.4f}s")
-    
-    return response
+    # Capture response body (consume iterator, then rebuild response)
+    resp_body_bytes = b""
+    async for chunk in response.body_iterator:
+        resp_body_bytes += chunk
+
+    new_response = Response(
+        content=resp_body_bytes,
+        status_code=response.status_code,
+        headers=dict(response.headers),
+        media_type=response.media_type,
+    )
+
+    resp_content_type = new_response.headers.get("content-type", "")
+    resp_summary = summarize_body(resp_body_bytes, resp_content_type)
+
+    outgoing_payload = {
+        "content_type": resp_content_type,
+        "body_summary": resp_summary,
+        "time_seconds": round(process_time, 4),
+    }
 
-# --- Exception Handler for Diagnostic Logging ---
+    add_debug_log(
+        "OUTGOING",
+        path,
+        method,
+        headers={},  # response headers usually not needed in debug store
+        body=outgoing_payload,
+        request_id=request_id,
+        status_code=new_response.status_code,
+        query_params=query_params,
+        client={"host": client_host, "port": client_port},
+    )
+
+    logger.info(
+        f"[{request_id}] OUTGOING {method} {path} "
+        f"status={new_response.status_code} time={process_time:.4f}s"
+    )
+
+    return new_response
+
+# --- Exception Handlers for Diagnostic Logging ---
 @app.exception_handler(RequestValidationError)
 async def validation_exception_handler(request: Request, exc: RequestValidationError):
-    body = await request.body()
-    try:
-        payload = json.loads(body)
-    except:
-        payload = body.decode()
-    
+    headers = dict(request.headers)
+    raw_body = await request.body()
+    content_type = headers.get("content-type", "")
+    body_summary = summarize_body(raw_body, content_type)
+
     error_detail = exc.errors()
-    logger.error(f"422 Unprocessable Entity Error! Payload: {payload} | Errors: {error_detail}")
-    
+    logger.error(
+        f"422 Validation Error! path={request.url.path} errors={error_detail} "
+        f"received_body_len={body_summary.get('raw_len')}"
+    )
+
     response_body = {
         "detail": error_detail,
-        "received_body": payload,
-        "message": "Validation failed. Check /debug/logs for details."
+        "received_body": body_summary.get("parsed") or body_summary.get("preview"),
+        "message": "Validation failed. Check /debug/logs for request_id-correlated details."
     }
-    add_debug_log("OUTGOING_ERROR", request.url.path, request.method, {}, response_body)
-    
+
+    add_debug_log(
+        "OUTGOING_ERROR",
+        request.url.path,
+        request.method,
+        headers,
+        {"error": response_body, "received_body_summary": body_summary},
+        request_id=request.headers.get("x-request-id"),
+        status_code=422,
+        query_params=dict(request.query_params),
+        client={"host": request.client.host if request.client else None},
+    )
+
     return JSONResponse(status_code=422, content=response_body)
 
-# --- Dependency for API Key Validation ---
-async def get_api_key(api_key_header: str = Depends(api_key_header)):
-    if api_key_header is None or api_key_header != API_KEY:
+@app.exception_handler(HTTPException)
+async def http_exception_handler(request: Request, exc: HTTPException):
+    # Log 401/404/etc with enough context to debug endpoint testers.
+    headers = dict(request.headers)
+    raw_body = await request.body()
+    content_type = headers.get("content-type", "")
+    body_summary = summarize_body(raw_body, content_type)
+
+    response_body = {"status": "error", "message": exc.detail}
+
+    logger.warning(
+        f"HTTPException status={exc.status_code} path={request.url.path} "
+        f"api_key_masked={mask_api_key(headers.get(API_KEY_NAME))}"
+    )
+
+    add_debug_log(
+        "OUTGOING_ERROR",
+        request.url.path,
+        request.method,
+        headers,
+        {"error": response_body, "received_body_summary": body_summary},
+        request_id=request.headers.get("x-request-id"),
+        status_code=exc.status_code,
+        query_params=dict(request.query_params),
+        client={"host": request.client.host if request.client else None},
+    )
+
+    return JSONResponse(status_code=exc.status_code, content=response_body)
+
+# --- Dependency for API Key Validation (with explicit logs) ---
+async def get_api_key(api_key_value: Optional[str] = Depends(api_key_header)):
+    if not api_key_value:
+        logger.warning("AUTH FAIL: missing x-api-key header")
         raise HTTPException(
             status_code=status.HTTP_401_UNAUTHORIZED,
-            detail="Invalid API Key or missing 'x-api-key' header.",
+            detail="Missing 'x-api-key' header.",
         )
-    return api_key_header
 
-# --- API Endpoints ---
+    if api_key_value != API_KEY:
+        logger.warning(
+            f"AUTH FAIL: invalid x-api-key provided={mask_api_key(api_key_value)} "
+            f"expected={mask_api_key(API_KEY)}"
+        )
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid API Key.",
+        )
 
+    return api_key_value
+
+# --- API Endpoints ---
 @app.get("/debug/logs")
 async def get_debug_logs():
-    """Endpoint to view the last 50 requests and responses."""
-    return {
-        "count": len(DEBUG_LOGS),
-        "logs": DEBUG_LOGS
-    }
+    """Endpoint to view the last N request/response summaries."""
+    return {"count": len(DEBUG_LOGS), "logs": DEBUG_LOGS}
 
 @app.post("/api/honeypot-detection", response_model=HoneypotResponse)
 async def honeypot_detection(
@@ -132,7 +322,7 @@ async def honeypot_detection(
 ) -> Dict[str, Any]:
     session_id = request_data.sessionId
     config = {"configurable": {"thread_id": session_id}}
-    
+
     checkpoint = honeypot_app.get_state(config)
     start_time = time.time()
 
@@ -144,7 +334,7 @@ async def honeypot_detection(
         current_state_dict.setdefault("conversationHistory", [])
         current_state_dict.setdefault("totalMessagesExchanged", 0)
         current_state_dict.setdefault("sessionId", session_id)
-        
+
         current_state = AgentState(**current_state_dict)
         current_state["conversationHistory"].append(request_data.message)
         current_state["totalMessagesExchanged"] += 1
@@ -167,7 +357,7 @@ async def honeypot_detection(
         final_state_dict = honeypot_app.invoke(input_state, config=config)
         final_state = AgentState(**final_state_dict)
         engagement_duration = int(time.time() - start_time)
-        
+
         response_content = {
             "status": "success",
             "scamDetected": final_state["scamDetected"],
@@ -178,24 +368,31 @@ async def honeypot_detection(
             "extractedIntelligence": final_state["extractedIntelligence"].model_dump(),
             "agentNotes": final_state["agentNotes"]
         }
-        
-        # Log the successful response body
-        add_debug_log("OUTGOING_SUCCESS", "/api/honeypot-detection", "POST", {}, response_content)
-        
+
+        # Keep endpoint-level success log too (useful if middleware is disabled later)
+        add_debug_log(
+            "OUTGOING_SUCCESS_ENDPOINT",
+            "/api/honeypot-detection",
+            "POST",
+            headers={},
+            body=response_content
+        )
+
         return response_content
 
     except Exception as e:
         error_msg = f"Internal Error: {str(e)}"
-        logger.error(error_msg)
-        add_debug_log("OUTGOING_ERROR", "/api/honeypot-detection", "POST", {}, {"error": error_msg})
+        logger.exception(error_msg)
+        add_debug_log("OUTGOING_ERROR_ENDPOINT", "/api/honeypot-detection", "POST", {}, {"error": error_msg})
         raise HTTPException(status_code=500, detail=error_msg)
 
 @app.get("/")
 async def root():
     return {
         "message": "Agentic Honey-Pot API is running.",
-        "endpoints": {
-            "detection": "/api/honeypot-detection",
-            "debug_logs": "/debug/logs"
+        "endpoints": {"detection": "/api/honeypot-detection", "debug_logs": "/debug/logs"},
+        "debug": {
+            "max_debug_logs": MAX_DEBUG_LOGS,
+            "max_body_preview_chars": MAX_BODY_PREVIEW_CHARS
         }
     }