Fix Generate Now: HF 503 retry + 120s countdown timer in UI

ai_analyst.py:
- HF API: retry up to 3x on 503 model-loading, waiting estimated_time
- Increase HF timeout 45s → 60s

dashboard template:
- Button shows live elapsed-seconds counter while waiting
- Hard timeout 120s (was 10s): shows "no response — try again"
- SSE ai_insight event clears the countdown via stored interval handle

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

ai_analyst/ai_analyst.py

@@ -49,30 +49,38 @@ def call_llm(prompt: str) -> str | None:
         except Exception as e:
             print(f"[AI-Analyst] Ollama unreachable: {e}")
 
-    # 2. HuggingFace Inference API
+    # 2. HuggingFace Inference API (with retry on 503 model-loading)
     if HF_TOKEN:
-        try:
-            url = f"https://api-inference.huggingface.co/models/{HF_MODEL}/v1/chat/completions"
-            resp = requests.post(
-                url,
-                headers={"Authorization": f"Bearer {HF_TOKEN}"},
-                json={
-                    "model":       HF_MODEL,
-                    "messages":    [{"role": "user", "content": prompt}],
-                    "max_tokens":  220,
-                    "temperature": 0.7,
-                },
-                timeout=45,
-            )
-            if resp.status_code == 200:
-                text = resp.json()["choices"][0]["message"]["content"].strip()
-                if text:
-                    print(f"[AI-Analyst] Insight via HuggingFace ({HF_MODEL})")
-                    return text
-            else:
-                print(f"[AI-Analyst] HF HTTP {resp.status_code}: {resp.text[:300]}")
-        except Exception as e:
-            print(f"[AI-Analyst] HF API error: {e}")
+        url = f"https://api-inference.huggingface.co/models/{HF_MODEL}/v1/chat/completions"
+        for attempt in range(3):
+            try:
+                resp = requests.post(
+                    url,
+                    headers={"Authorization": f"Bearer {HF_TOKEN}"},
+                    json={
+                        "model":       HF_MODEL,
+                        "messages":    [{"role": "user", "content": prompt}],
+                        "max_tokens":  220,
+                        "temperature": 0.7,
+                    },
+                    timeout=60,
+                )
+                if resp.status_code == 200:
+                    text = resp.json()["choices"][0]["message"]["content"].strip()
+                    if text:
+                        print(f"[AI-Analyst] Insight via HuggingFace ({HF_MODEL})")
+                        return text
+                elif resp.status_code == 503:
+                    body = resp.json() if resp.content else {}
+                    wait = body.get("estimated_time", 20)
+                    print(f"[AI-Analyst] HF model loading, waiting {wait:.0f}s (attempt {attempt+1}/3)")
+                    time.sleep(min(float(wait), 30))
+                else:
+                    print(f"[AI-Analyst] HF HTTP {resp.status_code}: {resp.text[:300]}")
+                    break
+            except Exception as e:
+                print(f"[AI-Analyst] HF API error (attempt {attempt+1}/3): {e}")
+                break
 
     return None
 

dashboard/templates/index.html

@@ -434,18 +434,32 @@
     let selectedOrder = null;
 
     async function triggerAIInsight() {
-      const btn = document.getElementById("ai-generate-btn");
+      const btn    = document.getElementById("ai-generate-btn");
+      const status = document.getElementById("ai-status");
       btn.disabled = true;
       btn.textContent = "Generating…";
-      document.getElementById("ai-status").textContent = "generating insight…";
+      status.textContent = "contacting LLM…";
       try {
         await fetch("/session/ai_insight", { method: "POST" });
       } catch(e) {}
-      // Re-enable after 10s (LLM needs time)
+
+      // Countdown timer — LLM (especially HF cold-start) can take up to 2 min
+      let elapsed = 0;
+      const tick = setInterval(() => {
+        elapsed += 1;
+        status.textContent = `generating… ${elapsed}s`;
+      }, 1000);
+
+      // Hard cap: re-enable after 120s with failure message
       setTimeout(() => {
+        clearInterval(tick);
         btn.disabled = false;
-        btn.textContent = "Generate Now";
-      }, 10000);
+        btn.textContent = "✨ Generate Now";
+        status.textContent = "no response — check logs or try again";
+      }, 120000);
+
+      // Store tick handle so SSE success can clear it
+      btn._tick = tick;
     }
 
     function addInsight(insight) {
@@ -1117,8 +1131,9 @@
         document.getElementById("ai-status").textContent =
           "Last update: " + new Date().toLocaleTimeString();
         const btn = document.getElementById("ai-generate-btn");
+        if (btn._tick) { clearInterval(btn._tick); btn._tick = null; }
         btn.disabled = false;
-        btn.textContent = "Generate Now";
+        btn.textContent = "✨ Generate Now";
       });
 
       eventSource.onerror = () => {