Consistent LLM retry + DOCUMENTS token safety valve

- Add rag.invoke_with_retry(): single retry path for every LLM call
(chat, recommender, coach/eval, helpers). Retries transient failures
(5xx/network/rate) up to llm_max_retries with backoff, fails fast on
permanent errors (bad key/model), re-raises when spent so each site's
fallback still runs. Routes all 6 prior raw .invoke() call sites through it.
- Wire max_docs_block_chars into _build_block via _within_budget(): BCA Life
docs always kept; competitor docs dropped largest-first only when over
budget. Default is generous so today's catalog is unchanged.
- Tests: budget disabled/under/over + home-never-dropped; retry
recover/exhaust/fail-fast. Full suite 25 passed.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app/config.py

@@ -18,6 +18,18 @@ class Settings(BaseSettings):
     openrouter_coach_model: str = ""
     openrouter_eval_model: str = ""
 
+    # Soft budget for the concatenated DOCUMENTS block that gets stuffed into
+    # every prompt. Our own (BCA Life) products are always kept in full for
+    # accuracy; competitor docs are dropped (largest first) once the block
+    # would exceed this, so token cost stays bounded as the catalog grows.
+    # Default is generous — the current catalog is well under it, so nothing
+    # changes today; this is a safety valve for later. ~4 chars ≈ 1 token, so
+    # 120k chars ≈ 30k tokens.
+    max_docs_block_chars: int = 120_000
+
+    # How many times to retry a transient LLM error before falling back.
+    llm_max_retries: int = 2
+
     def coach_model_name(self) -> str:
         return self.openrouter_coach_model.strip() or self.openrouter_chat_model
 

app/rag.py

@@ -9,6 +9,7 @@ Design:
   lets DeepSeek's automatic prompt cache kick in (~10x cheaper, faster).
 """
 import re
+import time
 import logging
 import threading
 from typing import Dict, List, Optional, Tuple
@@ -126,6 +127,38 @@ def get_helper_llm() -> ChatOpenAI:
     return _helper_llm
 
 
+# Errors that won't fix themselves on a retry — bad key, malformed request. We
+# fail these fast to the caller's fallback instead of burning the retry budget.
+_PERMANENT_ERR_RX = re.compile(
+    r"\b(400|401|403|invalid.?api.?key|authenticat|unauthorized|no.?such.?model)\b",
+    re.IGNORECASE,
+)
+
+
+def invoke_with_retry(llm, messages):
+    """Invoke an LLM, retrying transient failures before giving up.
+
+    OpenRouter calls fail intermittently on network blips, provider 5xx, and
+    rate limits; a couple of retries with a short backoff turns most of those
+    into a success instead of a user-facing fallback. Every LLM call in the app
+    (chat, recommender, coach/eval, helpers) goes through here so the retry
+    behaviour is consistent. Permanent errors are re-raised immediately, and the
+    last error is re-raised once the budget (settings.llm_max_retries) is spent —
+    each call site keeps its own try/except fallback."""
+    attempts = max(1, settings.llm_max_retries + 1)
+    last_err: Optional[Exception] = None
+    for i in range(attempts):
+        try:
+            return llm.invoke(messages)
+        except Exception as e:  # noqa: BLE001 — provider errors are opaque
+            last_err = e
+            if _PERMANENT_ERR_RX.search(str(e)) or i == attempts - 1:
+                break
+            log.warning("LLM invoke failed (attempt %d/%d): %s", i + 1, attempts, e)
+            time.sleep(0.5 * (i + 1))
+    raise last_err
+
+
 # -----------------------------------------------------------------------------
 # In-memory document store
 # -----------------------------------------------------------------------------
@@ -148,6 +181,40 @@ def _invalidate_block() -> None:
     _DOCS_BLOCK = None
 
 
+def _insurer_of(d: Dict[str, str]) -> str:
+    return d.get("insurer") or "Lainnya"
+
+
+def _within_budget(items: List[Dict[str, str]], budget: int) -> List[Dict[str, str]]:
+    """Enforce the soft DOCUMENTS char budget (settings.max_docs_block_chars).
+
+    Our own (BCA Life) products are always kept in full for accuracy; competitor
+    docs are dropped largest-first until the concatenated text fits. Order is
+    preserved for the caller. A non-positive budget disables the cap. The
+    default budget is generous, so a normal catalog keeps everything and the
+    prompt prefix stays byte-identical (cache-friendly)."""
+    if not budget or budget <= 0:
+        return items
+    total = sum(len(d.get("text") or "") for d in items)
+    if total <= budget:
+        return items
+    over = total
+    dropped: set = set()
+    competitors = [d for d in items if _insurer_of(d) != HOME_INSURER]
+    for d in sorted(competitors, key=lambda x: len(x.get("text") or ""), reverse=True):
+        if total <= budget:
+            break
+        total -= len(d.get("text") or "")
+        dropped.add(id(d))
+    if dropped:
+        log.warning(
+            "DOCUMENTS block over budget (%d > %d chars); dropped %d competitor "
+            "doc(s) largest-first to fit (BCA Life docs always kept)",
+            over, budget, len(dropped),
+        )
+    return [d for d in items if id(d) not in dropped]
+
+
 def _build_block() -> str:
     """Render the full DOCUMENTS block, grouped by insurer so the model can tell
     our products from competitors. Home insurer first, then others A→Z; stable
@@ -156,8 +223,10 @@ def _build_block() -> str:
     if not items:
         return "(belum ada dokumen produk yang diunggah)"
 
+    items = _within_budget(items, settings.max_docs_block_chars)
+
     def insurer_of(d):
-        return d.get("insurer") or "Lainnya"
+        return _insurer_of(d)
 
     groups: Dict[str, List[Dict[str, str]]] = {}
     for d in items:
@@ -377,7 +446,7 @@ def rewrite_standalone(question: str, history: List[Dict[str, str]]) -> str:
         f"{h['role'].upper()}: {h['content']}" for h in history[-4:]
     )
     try:
-        out = get_helper_llm().invoke([
+        out = invoke_with_retry(get_helper_llm(), [
             SystemMessage(content=_REWRITE_SYSTEM),
             HumanMessage(content=(
                 f"Conversation so far:\n{convo}\n\n"
@@ -490,7 +559,7 @@ def _sources_listing() -> List[Dict]:
 
 
 def _generate(system: str, user: str, *, llm=None) -> str:
-    out = (llm or get_llm()).invoke([
+    out = invoke_with_retry(llm or get_llm(), [
         cached_system(system),
         HumanMessage(content=user),
     ])

app/recommender.py

@@ -269,7 +269,7 @@ def recommend(profile: Dict) -> Dict:
 Berikan rekomendasi JSON sesuai format yang diminta."""
 
     try:
-        out = rag.get_strict_llm().invoke([
+        out = rag.invoke_with_retry(rag.get_strict_llm(), [
             SystemMessage(content=system),
             HumanMessage(content=user_msg),
         ])
@@ -466,7 +466,7 @@ def _compare_llm(bca_text: str, comp_text: str, comp_insurer: str, mode: str) ->
     )
 
     try:
-        out = rag.get_strict_llm().invoke([
+        out = rag.invoke_with_retry(rag.get_strict_llm(), [
             SystemMessage(content=COMPARE_SYSTEM),
             HumanMessage(content=user_msg),
         ])

app/training.py

@@ -176,7 +176,7 @@ def reply(session_id: str, fa_message: str) -> Dict:
     # filler/acknowledgement turns skip the second LLM call entirely.
     def _customer() -> str:
         try:
-            out = rag.get_llm().invoke(msgs)
+            out = rag.invoke_with_retry(rag.get_llm(), msgs)
             return _clean_reply((out.content or "").strip())
         except Exception as e:
             log.exception("customer reply failed: %s", e)
@@ -356,7 +356,7 @@ def _coach_turn(
             f"FA menjawab: \"{fa_message}\"\n\n"
             "Nilai pesan FA itu. Output JSON."
         )
-        out = rag.get_coach_llm().invoke([
+        out = rag.invoke_with_retry(rag.get_coach_llm(), [
             rag.cached_system(_COACH_SYSTEM),
             HumanMessage(content=user),
         ])
@@ -477,7 +477,7 @@ Kasih evaluasi JSON sesuai format yang diminta."""
     last_error = ""
     for attempt in range(3):
         try:
-            out = rag.get_eval_llm().invoke([
+            out = rag.invoke_with_retry(rag.get_eval_llm(), [
                 rag.cached_system(EVAL_SYSTEM),
                 HumanMessage(content=user_block),
             ])

tests/test_rag_helpers.py

@@ -32,3 +32,81 @@ def test_is_no_answer():
     assert rag._is_no_answer("NO_ANSWER") is True
     assert rag._is_no_answer("") is True
     assert rag._is_no_answer("Produk ini memberikan manfaat proteksi jiwa.") is False
+
+
+# --- DOCUMENTS block soft budget (safety valve) ------------------------------
+
+def _doc(insurer, n):
+    return {"name": f"{insurer}-{n}", "type": "txt", "insurer": insurer, "text": "x" * n}
+
+
+def test_within_budget_disabled_keeps_all():
+    items = [_doc("BCA Life", 100), _doc("Prudential", 100)]
+    assert rag._within_budget(items, 0) == items
+
+
+def test_within_budget_under_budget_keeps_all():
+    items = [_doc("BCA Life", 100), _doc("Prudential", 100)]
+    assert rag._within_budget(items, 10_000) == items
+
+
+def test_within_budget_drops_largest_competitor_first():
+    home = _doc("BCA Life", 500)
+    small_comp = _doc("AIA", 100)
+    big_comp = _doc("Prudential", 900)
+    kept = rag._within_budget([home, small_comp, big_comp], 700)
+    # Home is never dropped; the largest competitor goes first.
+    assert home in kept
+    assert big_comp not in kept
+    assert small_comp in kept
+
+
+def test_within_budget_never_drops_home_even_if_over():
+    home = _doc("BCA Life", 5_000)
+    comp = _doc("Prudential", 100)
+    kept = rag._within_budget([home, comp], 1_000)
+    assert home in kept  # home kept even though it alone exceeds the budget
+    assert comp not in kept
+
+
+# --- LLM retry helper --------------------------------------------------------
+
+class _FlakyLLM:
+    """Fails `fail_times` invokes with `exc`, then returns `ok`."""
+    def __init__(self, fail_times, exc, ok="ok"):
+        self.fail_times = fail_times
+        self.exc = exc
+        self.ok = ok
+        self.calls = 0
+
+    def invoke(self, messages):
+        self.calls += 1
+        if self.calls <= self.fail_times:
+            raise self.exc
+        return self.ok
+
+
+def test_invoke_with_retry_recovers_from_transient(monkeypatch):
+    monkeypatch.setattr(rag.time, "sleep", lambda *_: None)
+    monkeypatch.setattr(rag.settings, "llm_max_retries", 2)
+    llm = _FlakyLLM(fail_times=1, exc=RuntimeError("503 upstream"))
+    assert rag.invoke_with_retry(llm, []) == "ok"
+    assert llm.calls == 2
+
+
+def test_invoke_with_retry_exhausts_budget(monkeypatch):
+    monkeypatch.setattr(rag.time, "sleep", lambda *_: None)
+    monkeypatch.setattr(rag.settings, "llm_max_retries", 2)
+    llm = _FlakyLLM(fail_times=99, exc=RuntimeError("network"))
+    with pytest.raises(RuntimeError):
+        rag.invoke_with_retry(llm, [])
+    assert llm.calls == 3  # 1 initial + 2 retries
+
+
+def test_invoke_with_retry_fails_fast_on_permanent(monkeypatch):
+    monkeypatch.setattr(rag.time, "sleep", lambda *_: None)
+    monkeypatch.setattr(rag.settings, "llm_max_retries", 2)
+    llm = _FlakyLLM(fail_times=99, exc=RuntimeError("401 invalid api key"))
+    with pytest.raises(RuntimeError):
+        rag.invoke_with_retry(llm, [])
+    assert llm.calls == 1  # no retry on a permanent error