"""Author RAG — A/B Testing API. Allows authors to test different bot welcome messages, response styles, and CTA button text to maximize conversions. Routes: GET /api/admin/{slug}/ab-tests → list active tests POST /api/admin/{slug}/ab-tests → create new test GET /api/admin/{slug}/ab-tests/{id} → test results + significance PUT /api/admin/{slug}/ab-tests/{id}/winner → pick winning variant DELETE /api/admin/{slug}/ab-tests/{id} → stop test Variant assignment uses consistent hashing on visitor_id so the same visitor always sees the same variant within a test. """ import hashlib from datetime import datetime, timezone from typing import Literal from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel, Field from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.dependencies import get_current_author, get_db router = APIRouter() # ── Schemas ─────────────────────────────────────────────────────────────────── class ABTestCreate(BaseModel): name: str = Field(..., max_length=200) test_type: Literal["welcome_message", "response_style", "cta_text", "bot_name"] variant_a: str = Field(..., max_length=500) variant_b: str = Field(..., max_length=500) class WinnerPick(BaseModel): winner: Literal["a", "b"] # ── In-memory store (Phase 2 — replace with DB table in Phase 3) ────────────── # Keyed by f"{author_id}:{test_id}" _tests: dict[str, dict] = {} def _test_id() -> str: from app.models.base import generate_uuid return generate_uuid()[:8] # ── Variant Assignment ──────────────────────────────────────────────────────── def assign_variant(visitor_id: str, test_id: str) -> Literal["a", "b"]: """Deterministically assign visitor to variant A or B. Uses SHA256 so the same visitor always gets the same variant for the lifetime of the test (no flickering). """ digest = hashlib.sha256(f"{visitor_id}:{test_id}".encode()).hexdigest() return "a" if int(digest[0], 16) < 8 else "b" # ── Routes ──────────────────────────────────────────────────────────────────── @router.get("/{slug}/ab-tests") async def list_ab_tests( slug: str, author=Depends(get_current_author), ): """List all A/B tests for this author.""" if author.id != slug: raise HTTPException(403, "Forbidden") tests = [t for k, t in _tests.items() if k.startswith(f"{slug}:")] return {"tests": tests} @router.post("/{slug}/ab-tests", status_code=201) async def create_ab_test( slug: str, body: ABTestCreate, author=Depends(get_current_author), ): """Create a new A/B test. Only one active test per type per author.""" if author.id != slug: raise HTTPException(403, "Forbidden") # Check for duplicate active test of same type for k, t in _tests.items(): if k.startswith(f"{slug}:") and t["test_type"] == body.test_type and t["status"] == "active": raise HTTPException(409, f"An active {body.test_type} test already exists. Stop it first.") tid = _test_id() test = { "id": tid, "author_id": slug, "name": body.name, "test_type": body.test_type, "variant_a": body.variant_a, "variant_b": body.variant_b, "impressions_a": 0, "impressions_b": 0, "conversions_a": 0, "conversions_b": 0, "status": "active", "winner": None, "created_at": datetime.now(timezone.utc).isoformat(), "ended_at": None, } _tests[f"{slug}:{tid}"] = test return test @router.get("/{slug}/ab-tests/{test_id}") async def get_ab_test( slug: str, test_id: str, author=Depends(get_current_author), ): """Get test results including conversion rates and statistical significance.""" if author.id != slug: raise HTTPException(403, "Forbidden") test = _tests.get(f"{slug}:{test_id}") if not test: raise HTTPException(404, "Test not found") # Compute conversion rates cr_a = test["conversions_a"] / test["impressions_a"] if test["impressions_a"] else 0 cr_b = test["conversions_b"] / test["impressions_b"] if test["impressions_b"] else 0 lift = ((cr_b - cr_a) / cr_a * 100) if cr_a > 0 else 0 # Minimum detectable effect check (simplified) min_samples = 100 has_significance = ( test["impressions_a"] >= min_samples and test["impressions_b"] >= min_samples and abs(cr_b - cr_a) > 0.02 ) return { **test, "conversion_rate_a": round(cr_a, 4), "conversion_rate_b": round(cr_b, 4), "lift_pct": round(lift, 1), "is_significant": has_significance, "recommendation": ( "Declare winner — B is significantly better" if has_significance and cr_b > cr_a else "Declare winner — A is significantly better" if has_significance and cr_a > cr_b else "Collect more data (need 100+ impressions per variant)" ), } @router.put("/{slug}/ab-tests/{test_id}/winner") async def pick_winner( slug: str, test_id: str, body: WinnerPick, author=Depends(get_current_author), ): """Pick the winning variant and apply it to the author's live bot config.""" if author.id != slug: raise HTTPException(403, "Forbidden") test = _tests.get(f"{slug}:{test_id}") if not test: raise HTTPException(404, "Test not found") winning_value = test[f"variant_{body.winner}"] test["winner"] = body.winner test["status"] = "completed" test["ended_at"] = datetime.now(timezone.utc).isoformat() # TODO: Apply the winning variant to the author's User config # e.g. if test_type == "welcome_message": user.welcome_message = winning_value return { "winner": body.winner, "winning_value": winning_value, "message": f"Variant {body.winner.upper()} is now live on your chatbot.", } @router.delete("/{slug}/ab-tests/{test_id}", status_code=200) async def stop_ab_test( slug: str, test_id: str, author=Depends(get_current_author), ): """Stop a running test without picking a winner.""" if author.id != slug: raise HTTPException(403, "Forbidden") test = _tests.get(f"{slug}:{test_id}") if not test: raise HTTPException(404, "Test not found") test["status"] = "stopped" test["ended_at"] = datetime.now(timezone.utc).isoformat() return {"message": "Test stopped. No winner applied."} # ── Internal: Record Impression/Conversion ──────────────────────────────────── def record_impression(author_id: str, test_type: str, variant: str) -> None: """Called by the chat pipeline when a visitor sees a variant.""" for k, t in _tests.items(): if (k.startswith(f"{author_id}:") and t["test_type"] == test_type and t["status"] == "active"): t[f"impressions_{variant}"] += 1 break def record_conversion(author_id: str, test_type: str, variant: str) -> None: """Called when a visitor clicks a buy link after seeing a variant.""" for k, t in _tests.items(): if (k.startswith(f"{author_id}:") and t["test_type"] == test_type and t["status"] == "active"): t[f"conversions_{variant}"] += 1 break