services/retention_engine.py

"""
services/retention_engine.py   [NEW — this is the Coach's actual brain]

Takes the real retention curve + the real timestamped transcript and produces
FACTS, not opinions:

    retention curve  ──┐
                       ├─► cliff detection ─► align to transcript ─► drop_off_moments
    transcript      ───┘                                             (timestamp, %, quote)

Everything numeric in the final report is computed HERE. The LLM is only ever
allowed to write prose about facts we hand it. That is the difference between
"the AI thinks people left around the middle" and "37% of your audience left at
1:42, right after you said 'but before we get into that, let me tell you about
today's sponsor'".
"""

from __future__ import annotations

from statistics import median
from typing import Optional

from services.transcript_service import Transcript, fmt_ts

# CTA / retention vocabulary — used for deterministic CTA + hook scoring
CTA_WORDS = ("subscribe", "hit the bell", "like this video", "smash that", "comment below",
             "let me know in the comments", "check the link", "link in the description",
             "join the channel", "next video", "watch this one", "follow me", "sign up")
STALL_WORDS = ("before we get into", "but first", "quick word from", "today's sponsor",
               "sponsored by", "don't forget to", "make sure to subscribe",
               "let me introduce myself", "welcome back to my channel", "hey guys welcome back",
               "in this video i'm going to", "so yeah", "um", "anyway")


# ── curve utilities ─────────────────────────────────────────────────────────

def build_curve(retention: list[dict], duration_sec: int) -> list[dict]:
    """
    Attach real wall-clock seconds to each retention sample, and NORMALISE.

    audienceWatchRatio is NOT a clean 0..1 fraction. YouTube returns it relative
    to the average view, so the opening bucket frequently comes back as 1.1–1.3
    (i.e. "129%"). Charting that raw produces "129% still watching", which is
    nonsense to a creator and makes the LLM assert impossible facts.

    Fix: rebase the whole curve so t=0 == 100% of the people who actually started,
    then clamp to [0, 100]. Every downstream number ("48.9% remaining", "15.6% of
    the audience left") is now a real, honest share of the opening audience.

    We keep the raw value in `watch_raw` for debugging, but nothing renders it.
    """
    pts = []
    for r in retention:
        ratio = float(r.get("ratio", 0))
        pts.append({
            "t": round(ratio * duration_sec, 1),
            "ratio": ratio,
            "watch_raw": float(r.get("watch_ratio", 0)),
            "relative": r.get("relative"),
        })
    pts.sort(key=lambda p: p["ratio"])

    # rebase to the first sample (the true 100% — everyone who pressed play)
    base = next((p["watch_raw"] for p in pts if p["watch_raw"] > 0), 1.0) or 1.0
    for p in pts:
        p["watch"] = max(0.0, min(1.0, p["watch_raw"] / base))
    return pts


def bucket_size(curve: list[dict]) -> float:
    """
    THE RESOLUTION LIMIT, stated out loud.

    YouTube does not hand back a per-second retention curve. It hands back
    `elapsedVideoTimeRatio` — 101 samples at 1% steps. On a 30-minute video that
    is an ~18-SECOND BUCKET. On a 51-minute video, ~31 seconds.

    So "viewers left at 0:18" is a lie we must never tell. The truth is "viewers
    left somewhere between 0:00 and 0:18". Everything downstream — the label, the
    chart marker, the transcript window we quote from — has to respect that, or
    the Coach will confidently quote the wrong sentence.
    """
    if len(curve) < 2:
        return 0.0
    gaps = [b["t"] - a["t"] for a, b in zip(curve, curve[1:]) if b["t"] > a["t"]]
    return round(median(gaps), 2) if gaps else 0.0


def watch_at(curve: list[dict], seconds: float) -> Optional[float]:
    """Interpolated audience remaining (0–1) at a given second."""
    if not curve:
        return None
    if seconds <= curve[0]["t"]:
        return curve[0]["watch"]
    for a, b in zip(curve, curve[1:]):
        if a["t"] <= seconds <= b["t"]:
            span = (b["t"] - a["t"]) or 1
            f = (seconds - a["t"]) / span
            return a["watch"] + (b["watch"] - a["watch"]) * f
    return curve[-1]["watch"]


# ── cliff detection ─────────────────────────────────────────────────────────

def detect_cliffs(curve: list[dict], duration_sec: int, max_moments: int = 5) -> list[dict]:
    """
    A "cliff" is a step where the audience falls FASTER THAN THIS VIDEO'S OWN
    baseline decay. Every video loses viewers; we only care about the moments
    where it loses them abnormally fast — those are caused by something the
    creator did.

    Method:
      1. per-step loss  = watch[i-1] - watch[i]
      2. baseline       = median positive loss (this video's natural decay)
      3. a step is a cliff if loss >= max(2 percentage points, 2.2 × baseline)
      4. consecutive cliff steps are merged into one moment
      5. keep the biggest `max_moments`
    """
    if len(curve) < 5:
        return []

    losses = []
    for a, b in zip(curve, curve[1:]):
        losses.append(max(0.0, a["watch"] - b["watch"]))
    base = median([l for l in losses if l > 0]) or 0.005
    threshold = max(0.02, base * 2.2)

    # The instant bounce in the FIRST bucket is not a "cliff" — it's the opening
    # loss, and it's reported separately as opening_retention_pct. If we leave it
    # in, it's so large it eats all `max_moments` slots and buries the real
    # mid-video drops the creator can actually act on (the exact bug in the
    # screenshot: four "cliffs" all inside 0:00–0:07). Skip index 0.
    flagged = [i for i, l in enumerate(losses) if l >= threshold and i >= 1]
    if not flagged:
        return []

    # merge runs of consecutive indexes
    runs, run = [], [flagged[0]]
    for i in flagged[1:]:
        if i == run[-1] + 1:
            run.append(i)
        else:
            runs.append(run)
            run = [i]
    runs.append(run)

    moments = []
    for run in runs:
        i0, i1 = run[0], run[-1] + 1
        before = curve[i0]["watch"]
        after = curve[i1]["watch"]
        lost = before - after
        if lost <= 0:
            continue
        steepest = max(losses[i] for i in run)
        moments.append({
            "t_start": curve[i0]["t"],
            "t_end": curve[i1]["t"],
            "retention_before": round(before * 100, 1),
            "retention_after": round(after * 100, 1),
            "drop_pct": round(lost * 100, 1),                       # percentage points of the ORIGINAL audience
            "share_of_audience_lost": round((lost / before) * 100, 1) if before else 0,
            # how much steeper the worst step was than this video's own natural decay
            "severity": min(50.0, round(steepest / base, 1)),
        })

    moments.sort(key=lambda m: m["drop_pct"], reverse=True)
    return moments[:max_moments]


def detect_replays(curve: list[dict], min_gain: float = 0.008, limit: int = 3) -> list[dict]:
    """Retention going UP = people rewinding. These are the moments that WORKED."""
    out = []
    for a, b in zip(curve, curve[1:]):
        gain = b["watch"] - a["watch"]
        if gain >= min_gain:
            out.append({"t": a["t"], "gain_pct": round(gain * 100, 1)})
    out.sort(key=lambda x: x["gain_pct"], reverse=True)
    return out[:limit]


# ── deterministic scoring (no LLM anywhere near these) ──────────────────────

def _interp(x: float, points: list[tuple[float, float]]) -> int:
    """Piecewise-linear map with clamping."""
    if x <= points[0][0]:
        return int(points[0][1])
    if x >= points[-1][0]:
        return int(points[-1][1])
    for (x0, y0), (x1, y1) in zip(points, points[1:]):
        if x0 <= x <= x1:
            f = (x - x0) / ((x1 - x0) or 1)
            return int(round(y0 + (y1 - y0) * f))
    return 50


def hook_score(curve: list[dict], duration_sec: int, is_short: bool) -> tuple[int, dict]:
    """
    YouTube's own rule of thumb: if you still have ~70% of viewers at 30s
    (or ~60% at the 15s mark of a Short), the hook worked.
    """
    mark = 15 if is_short else 30
    if duration_sec <= mark * 1.5:
        mark = max(5, int(duration_sec * 0.25))
    w = watch_at(curve, mark)
    if w is None:
        return 0, {}
    score = _interp(w, [(0.25, 5), (0.40, 25), (0.55, 45), (0.65, 60),
                        (0.72, 75), (0.82, 88), (0.92, 97)])
    return score, {
        "checkpoint_sec": mark,
        "retention_at_checkpoint": round(w * 100, 1),
        "benchmark": 70 if not is_short else 60,
        "verdict": "above benchmark" if w * 100 >= (60 if is_short else 70) else "below benchmark",
    }


def retention_score(avg_view_pct: float, relative: Optional[float]) -> tuple[int, dict]:
    s = _interp(avg_view_pct, [(10, 10), (20, 30), (30, 48), (40, 62),
                               (50, 76), (60, 88), (75, 97)])
    detail = {"avg_view_percentage": avg_view_pct}
    if relative is not None:
        detail["vs_similar_videos"] = round(relative * 100)
        # relativeRetentionPerformance 0.5 = median; nudge the score toward reality
        s = int(round(s * 0.75 + _interp(relative, [(0.2, 20), (0.5, 60), (0.8, 90), (1.0, 98)]) * 0.25))
    return max(0, min(100, s)), detail


def engagement_score(views: int, likes: int, comments: int) -> tuple[int, dict]:
    rate = ((likes + comments) / views) if views else 0
    s = _interp(rate, [(0.0, 0), (0.01, 30), (0.02, 50), (0.035, 72), (0.05, 88), (0.08, 98)])
    return s, {"engagement_rate_pct": round(rate * 100, 2),
               "like_to_view_ratio": round((likes / views * 100), 2) if views else 0}


def cta_score(transcript: Transcript, curve: list[dict], duration_sec: int,
              views: int, subs_gained: int) -> tuple[int, dict]:
    tail = transcript.window(max(0, duration_sec - 60), duration_sec, max_chars=1200).lower() if transcript.is_speech else ""
    said_cta = [w for w in CTA_WORDS if w in tail]
    end_watch = watch_at(curve, duration_sec * 0.95) or 0
    conv = (subs_gained / views) if views else 0

    s = 0
    s += 40 if said_cta else 0
    s += _interp(end_watch, [(0.05, 0), (0.15, 12), (0.30, 25), (0.45, 33)])
    s += _interp(conv, [(0.0, 0), (0.002, 8), (0.005, 16), (0.01, 27)])
    return max(0, min(100, int(s))), {
        "cta_detected": bool(said_cta),
        "cta_phrases_used": said_cta[:3],
        "audience_still_present_at_cta_pct": round(end_watch * 100, 1),
        "subscriber_conversion_pct": round(conv * 100, 3),
    }


def risk_level(hook: int, cliffs: list[dict]) -> str:
    severe = sum(1 for c in cliffs if c["drop_pct"] >= 8)
    if hook < 45 or severe >= 3:
        return "High"
    if hook < 65 or severe >= 1:
        return "Medium"
    return "Low"


def verdict(hook: int, ret: int, eng: int) -> str:
    avg = (hook * 0.4 + ret * 0.4 + eng * 0.2)
    if avg >= 70:
        return "Strong"
    if avg >= 48:
        return "Average"
    return "Needs Work"


# ── the fusion step: numbers + words ────────────────────────────────────────

def align(moments: list[dict], transcript: Transcript, bucket: float = 0.0) -> list[dict]:
    """
    THE core requirement: "Match time to text."

    The old version quoted transcript.at(t_start, before=7, after=3) — a 10-second
    window hanging off the START edge of an 18–31 second bucket. On a long video
    that quotes a sentence the viewer never even reached before leaving.

    The audience fell somewhere BETWEEN t_start and t_end. So the cause is:
      • what was being said DURING that whole span  → transcript_quote
      • what set it up, in the ~15s before it       → said_just_before
    Both windows are now derived from the real bucket, not a magic number.
    """
    out = []
    for m in moments:
        t0 = m["t_start"]
        t1 = m.get("t_end", t0) or t0
        if t1 <= t0:
            t1 = t0 + max(bucket, 1.0)

        speech = transcript.is_speech
        # everything spoken across the fall itself
        quote = transcript.window(t0, t1, max_chars=420) if speech else ""
        # the run-up: one bucket (min 12s) before the fall began
        lead = max(bucket, 12.0)
        lead_in = transcript.window(max(0.0, t0 - lead), t0, max_chars=300) if speech else ""

        # chapters are a real time→text map but NOT words the creator said —
        # they go in `section`, never in `transcript_quote`.
        section = transcript.section_at(t0) if transcript.kind == "chapters" else ""
        stalls = [w for w in STALL_WORDS if w in (lead_in + " " + quote + " " + section).lower()]

        out.append({
            **m,
            "t_end": round(t1, 1),
            # HONEST LABELS. A cliff is a span, not an instant.
            "timestamp": fmt_ts(t0),
            "timestamp_end": fmt_ts(t1),
            "window_label": f"{fmt_ts(t0)}–{fmt_ts(t1)}",
            "precision_sec": round(bucket, 1),
            "transcript_quote": quote,
            "said_just_before": lead_in,
            "section": section,
            "pattern_flags": stalls[:2],   # sponsor read, "but first", rambling intro…
        })
    return out


def analyse(curve_raw: list[dict], transcript: Transcript, *, duration_sec: int,
            views: int, likes: int, comments: int, avg_view_pct: float,
            avg_view_duration: int, subs_gained: int, is_short: bool) -> dict:
    """Everything the Coach knows, before a single token of LLM is generated."""
    curve = build_curve(curve_raw, duration_sec)
    bucket = bucket_size(curve)
    cliffs = align(detect_cliffs(curve, duration_sec), transcript, bucket)
    replays = detect_replays(curve)

    rel_vals = [p["relative"] for p in curve if p.get("relative") is not None]
    relative = (sum(rel_vals) / len(rel_vals)) if rel_vals else None

    h, h_detail = hook_score(curve, duration_sec, is_short)
    r, r_detail = retention_score(avg_view_pct, relative)
    e, e_detail = engagement_score(views, likes, comments)
    c, c_detail = cta_score(transcript, curve, duration_sec, views, subs_gained)

    for rp in replays:
        rp["timestamp"] = fmt_ts(rp["t"])
        rp["transcript_quote"] = transcript.at(rp["t"], before=4, after=6) if transcript.is_speech else ""

    # audienceWatchRatio SHOULD be ~1.0 at t=0. When it isn't, the very first
    # bucket already contains people who bounced instantly — the true opening
    # loss is worse than any cliff we can name. Say so rather than swallow it.
    opening = round(curve[0]["watch"] * 100, 1) if curve else 0.0

    return {
        "duration_sec": duration_sec,
        "bucket_sec": bucket,
        "opening_retention_pct": opening,
        "curve": [{"t": p["t"], "pct": round(p["watch"] * 100, 1)} for p in curve],
        "drop_off_moments": cliffs,
        "replay_moments": replays,
        "scores": {
            "hook_score": h, "retention_score": r,
            "engagement_score": e, "cta_strength": c,
            "retention_risk": risk_level(h, cliffs),
        },
        "evidence": {
            "hook": h_detail, "retention": r_detail,
            "engagement": e_detail, "cta": c_detail,
            "avg_view_duration_sec": avg_view_duration,
            "watched_to_end_pct": round((watch_at(curve, duration_sec * 0.98) or 0) * 100, 1),
        },
        "overall_verdict": verdict(h, r, e),
    }