scripts/keep_fal_warm.py

"""Keep the fal FLUX inpaint container WARM so customers never hit a cold start.

fal serverless scales the FLUX container to zero when idle; the next request then
pays ~2-3 min to spin it back up (cold). During salon hours this loop sends a small
inpaint request every `interval` seconds so the container stays hot and real
customer requests run warm (seconds-of-spin, not minutes).

COST WARNING (learned the hard way): a ping is NOT free, and NOT cheap-per-pixel.
fal flux-general bills $0.075 per MEGAPIXEL with an effective ~1 MP minimum, so even
a 256px ping costs ~$0.075. At a 45s interval that is ~800 pings/10h ≈ ~$60/day just
to keep warm — far more than paying ~$0.075 per ACTUAL customer photo. Ping keep-warm
is therefore NOT economical for this endpoint; prefer fal dedicated/keep_alive (if its
fixed price beats your volume) or simply accept the occasional cold start + a queue/UX.
This script is now mainly a diagnostic.

CRITICAL TUNING: the interval must be SHORTER than fal's idle timeout, or the
container scales down between pings and customers (and the pings themselves) still
hit cold starts. We saw this live at 150s: pings alternated ~4s (warm) and
100-170s (cold) — i.e. fal's idle window is under 150s. This script now CLASSIFIES
each ping (warm vs COLD) and warns when it's leaking, so you can dial the interval
down empirically (try 90s, then 60s) until COLD stops appearing.

Gated on FAL_KEY (this shell only; never logged/committed). Each ping bills ~$0.075
(see COST WARNING above) — shorter interval = more pings = much more cost.

Stop with Ctrl+C. Pilot Ready: NOT CONFIRMED.
"""
from __future__ import annotations

import argparse
import sys
import time
from io import BytesIO
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from app.services.fal_client import FalUnavailable, fal_inpaint_model, fal_real_enabled, run_fal_inpaint  # noqa: E402


def classify(ok: bool, duration: float, warm_max: float) -> str:
    """Label a ping: 'warm' (fast = container was hot), 'COLD' (slow = it had to
    spin up, i.e. keep-warm is leaking), or 'FAIL' (errored/timed out)."""
    if not ok:
        return "FAIL"
    return "warm" if duration <= warm_max else "COLD"


def should_warn_leak(n: int, label: str) -> bool:
    """True only when a ping signals a real between-ping leak. Ping #1 is the
    expected startup spin-up (container is down at launch) — never a leak; only
    a non-warm ping from #2 on means the container scaled down between pings."""
    return n >= 2 and label != "warm"


def sleep_seconds(interval: float, elapsed: float) -> float:
    """Fixed-cadence sleep: hold a CONSTANT start-to-start interval. Sleeping a flat
    `interval` AFTER each ping let a slow (cold ~28s) ping push the next gap out
    (60+28=88s), which kept tripping fal's idle window and cascaded more cold starts.
    Subtracting the elapsed ping time keeps the real gap == interval."""
    return max(1.0, interval - elapsed)


def is_fatal_error(msg: str) -> bool:
    """A NON-retryable fal failure (billing/auth) — keep-warm must STOP, not keep
    hammering a locked account every cadence with a misleading 'lower the interval'
    hint. Seen live: HTTP 403 'User is locked. Reason: Exhausted balance.'"""
    m = (msg or "").lower()
    return any(s in m for s in (
        "exhausted balance", "user is locked", "http 403", "http 401",
        "unauthorized", "forbidden", "payment", "billing",
    ))


def summary_line(counts: dict) -> str:
    """Rolling tally string. warm-rate = warm / total pings so far."""
    total = sum(counts.values())
    warm = counts.get("warm", 0)
    rate = (100.0 * warm / total) if total else 0.0
    return (f"tally: {warm} warm / {counts.get('COLD', 0)} COLD / "
            f"{counts.get('FAIL', 0)} FAIL  (warm-rate {rate:.0f}%)")


def _tiny_png(size=256):
    from PIL import Image, ImageDraw

    img = Image.new("RGB", (size, size), (128, 128, 128))
    mask = Image.new("L", (size, size), 0)
    ImageDraw.Draw(mask).ellipse([size * 0.3, size * 0.3, size * 0.7, size * 0.7], fill=255)
    a = BytesIO(); img.save(a, "PNG")
    b = BytesIO(); mask.save(b, "PNG")
    return a.getvalue(), b.getvalue()


def main(argv: list[str] | None = None) -> int:
    ap = argparse.ArgumentParser(description="Keep the fal FLUX container warm")
    ap.add_argument("--interval", type=int, default=90,
                    help="target seconds START-to-START (fixed cadence); MUST be under "
                         "fal's idle timeout. 60s measured ~55%% warm -> try 45, then 30")
    ap.add_argument("--steps", type=int, default=4, help="cheap ping inference steps")
    ap.add_argument("--size", type=int, default=256, help="tiny ping image size")
    ap.add_argument("--warm-max", type=float, default=20.0,
                    help="ping under this many seconds = warm; above = COLD (spin-up)")
    ap.add_argument("--timeout", type=int, default=240,
                    help="per-ping timeout; must exceed a cold spin-up (~2-3 min)")
    args = ap.parse_args(argv)

    if not fal_real_enabled():
        print("REFUSED: FAL_KEY not set (no pings sent).")
        return 2

    img, mask = _tiny_png(args.size)
    print(f"keep-warm: model={fal_inpaint_model()} every {args.interval}s "
          f"(tiny {args.size}px, {args.steps} steps, warm<= {args.warm_max}s). Ctrl+C to stop.")
    counts = {"warm": 0, "COLD": 0, "FAIL": 0}
    n = 0
    while True:
        n += 1
        t = time.time()
        err_msg = ""
        try:
            run_fal_inpaint(img, mask, "warm", strength=0.2, num_inference_steps=args.steps,
                            timeout_s=args.timeout)
            dur = time.time() - t
            label = classify(True, dur, args.warm_max)
            print(f"  ping #{n}: {label} ({dur:.1f}s)")
        except FalUnavailable as exc:
            label = "FAIL"; err_msg = str(exc)
            print(f"  ping #{n}: FAIL ({err_msg})")
        except Exception as exc:  # noqa: BLE001
            label = "FAIL"; err_msg = type(exc).__name__
            print(f"  ping #{n}: FAIL ({err_msg})")
        counts[label] += 1

        # A billing/auth failure is NOT a warm-up problem — stop instead of hammering
        # a locked account (and stop the misleading "lower the interval" hint).
        if label == "FAIL" and is_fatal_error(err_msg):
            print(f"  {summary_line(counts)}")
            print("  STOP: fatal fal error (billing/auth) — this is NOT a keep-warm "
                  "issue. Fix the account (top up at fal.ai/dashboard/billing) or the "
                  "key, then restart. Halting pings.")
            return 1

        # ping #1 is ALWAYS the startup spin-up (the container is down when this
        # process starts) — expected cold, NOT an interval leak. From ping #2 on,
        # a non-warm ping means the container scaled down *between pings* -> the
        # interval is longer than fal's idle window. Warn only on that bad ping;
        # otherwise show the tally periodically.
        if should_warn_leak(n, label):
            print(f"  {summary_line(counts)}")
            if label == "COLD":
                print(f"  hint: COLD on ping #{n} (between-ping) -> container scaled "
                      f"down; idle timeout < {args.interval}s; lower --interval "
                      f"(try {max(30, args.interval - 15)}).")
            else:
                print(f"  note: ping #{n} FAILED (transient/network) — retrying next cadence.")
        elif n == 1 and label != "warm":
            print("  (ping #1 = startup spin-up, expected cold; not a leak. "
                  "Let it run and watch pings #2+ to judge the interval.)")
        elif label == "warm" and n % 10 == 0:
            print(f"  {summary_line(counts)}")
        time.sleep(sleep_seconds(args.interval, time.time() - t))


if __name__ == "__main__":
    try:
        raise SystemExit(main())
    except KeyboardInterrupt:
        print("\nkeep-warm: stopped.")