feat: numbered citations [1][2][3], follow-up questions, focus modes

- LLM prompt now outputs [1][2][3] numbered citations instead of verbose inline sources
- Each source gets a citation_index attached before being sent to frontend
- LLM generates 3 follow-up questions at end of every answer (FOLLOW_UP: q1 | q2 | q3)
- Parser strips FOLLOW_UP block from answer and returns as follow_up_questions array
- Both execute_chat and execute_stream updated consistently

Version: 2.7

src/core/use_cases/rag_chat_use_case.py

@@ -704,6 +704,22 @@ JSON:"""
         #             context_text = f"{trend_text}\n\nRetrieved Search Context:\n{context_text}"
         #     except: pass
 
+        # ── Build numbered source index for citations ─────────────────────────
+        # Each source gets a number [1], [2], [3]... so the LLM can cite by number
+        source_index_lines = ""
+        for idx, doc in enumerate(final_sources, 1):
+            meta = doc.get("metadata", {})
+            source_name = (
+                meta.get("source") or meta.get("title") or doc.get("source") or "Unknown"
+            )
+            search_lang = meta.get("_search_lang", "en")
+            if search_lang and search_lang != "en":
+                lang_label = SUPPORTED_LANGUAGES.get(search_lang, search_lang.upper())
+                source_label = f"{source_name} ({lang_label})"
+            else:
+                source_label = source_name
+            source_index_lines += f"[{idx}] {source_label}\n"
+
         prompt = f"""You are ARKI AI, a real-time news assistant. Today's date is {datetime.utcnow().strftime("%B %d, %Y")}.
 
 STRICT RULES — READ CAREFULLY BEFORE ANSWERING:
@@ -717,7 +733,7 @@ STEP 2 — EVALUATE THE SOURCES:
 Read the News Context below and determine:
 
 A) DIRECT MATCH — Sources directly answer the question:
-   → Provide a comprehensive answer with citations
+   → Provide a comprehensive answer with numbered citations
    → Synthesize information from multiple sources
    → Use numbered points with **bold** headlines
 
@@ -725,7 +741,6 @@ B) RELATED INFORMATION — Sources have related but not exact information:
    → Acknowledge what you found: "I found articles about [related topic]"
    → Explain the gap: "but not specifically about [exact query]"
    → Provide the related information anyway (it may still be helpful)
-   → Suggest: "Would you like to know about [related topic] instead?"
 
 C) NO RELEVANT INFORMATION — Sources are completely unrelated:
    → Say clearly: "I couldn't find relevant news on that topic in today's feed."
@@ -733,12 +748,17 @@ C) NO RELEVANT INFORMATION — Sources are completely unrelated:
 
 STEP 3 — ANSWER RULES:
 1. Use ONLY facts from the News Context below. NEVER use training data or general knowledge.
-2. CITATIONS: After EVERY fact, add inline citation: "— Source: name" using the exact name from the [Source:] tag.
+2. CITATIONS: After EVERY fact, cite using the source NUMBER like [1] or [2][3]. Use the Source Index below to match names to numbers.
 3. Prioritize high-authority sources (BBC, Reuters, Al Jazeera, The Guardian) over others.
-4. Non-English articles — translate content to English, note language: "— Source: Al Jazeera (Arabic)".
+4. Non-English articles — translate content to English in your answer.
 5. Always respond in English. No hedging. No "based on my knowledge."
 6. Be helpful and flexible — if exact match not found, offer related information.
+7. At the END of your answer, on a new line, write exactly:
+   FOLLOW_UP: question1 | question2 | question3
+   These must be 3 short, specific follow-up questions the user might want to ask next, based on your answer.
 
+Source Index:
+{source_index_lines}
 News Context (from live multilingual database):
 {context_text}
 
@@ -749,7 +769,17 @@ User Question: {request.query}
 
 Answer:"""
         
-        answer = self.llm.generate(prompt)
+        raw_answer = self.llm.generate(prompt)
+
+        # ── Parse follow-up questions out of the answer ───────────────────────
+        follow_up_questions: List[str] = []
+        answer = raw_answer
+        if "FOLLOW_UP:" in raw_answer:
+            parts = raw_answer.split("FOLLOW_UP:", 1)
+            answer = parts[0].strip()
+            follow_up_raw = parts[1].strip()
+            follow_up_questions = [q.strip() for q in follow_up_raw.split("|") if q.strip()][:3]
+
         retrieved_ids = [str(doc.get("doc_id")) for doc in final_sources]
         
         self.chat_history_db.save_interaction(session_id, request.query, answer, retrieved_ids)
@@ -759,10 +789,15 @@ Answer:"""
             doc.get("source_type") == "live" or doc.get("is_live") 
             for doc in final_sources
         )
-        
+
+        # ── Attach citation index to each source for frontend rendering ───────
+        for idx, doc in enumerate(final_sources, 1):
+            doc["citation_index"] = idx
+
         result = {
             "answer": answer,
             "sources": final_sources,
+            "follow_up_questions": follow_up_questions,
             "session_id": session_id,
             "debug": {
                 "search_query": request.query,
@@ -798,6 +833,22 @@ Answer:"""
             request.query, request.top_k, request.source_filter, request.language_filter, getattr(request, 'days_back', None)
         )
 
+        # ── Build numbered source index for citations ─────────────────────────
+        source_index_lines = ""
+        for idx, doc in enumerate(final_sources, 1):
+            meta = doc.get("metadata", {})
+            source_name = (
+                meta.get("source") or meta.get("title") or doc.get("source") or "Unknown"
+            )
+            search_lang = meta.get("_search_lang", "en")
+            if search_lang and search_lang != "en":
+                lang_label = SUPPORTED_LANGUAGES.get(search_lang, search_lang.upper())
+                source_label = f"{source_name} ({lang_label})"
+            else:
+                source_label = source_name
+            source_index_lines += f"[{idx}] {source_label}\n"
+            doc["citation_index"] = idx
+
         prompt_stream = f"""You are ARKI AI, a real-time news assistant. Today's date is {datetime.utcnow().strftime("%B %d, %Y")}.
 
 STRICT RULES — READ CAREFULLY BEFORE ANSWERING:
@@ -811,7 +862,7 @@ STEP 2 — EVALUATE THE SOURCES:
 Read the News Context below and determine:
 
 A) DIRECT MATCH — Sources directly answer the question:
-   → Provide a comprehensive answer with citations
+   → Provide a comprehensive answer with numbered citations
    → Synthesize information from multiple sources
    → Use numbered points with **bold** headlines
 
@@ -819,7 +870,6 @@ B) RELATED INFORMATION — Sources have related but not exact information:
    → Acknowledge what you found: "I found articles about [related topic]"
    → Explain the gap: "but not specifically about [exact query]"
    → Provide the related information anyway (it may still be helpful)
-   → Suggest: "Would you like to know about [related topic] instead?"
 
 C) NO RELEVANT INFORMATION — Sources are completely unrelated:
    → Say clearly: "I couldn't find relevant news on that topic in today's feed."
@@ -827,12 +877,17 @@ C) NO RELEVANT INFORMATION — Sources are completely unrelated:
 
 STEP 3 — ANSWER RULES:
 1. Use ONLY facts from the News Context below. NEVER use training data or general knowledge.
-2. CITATIONS: After EVERY fact, add inline citation: "— Source: name" using the exact name from the [Source:] tag.
+2. CITATIONS: After EVERY fact, cite using the source NUMBER like [1] or [2][3]. Use the Source Index below to match names to numbers.
 3. Prioritize high-authority sources (BBC, Reuters, Al Jazeera, The Guardian) over others.
-4. Non-English articles — translate content to English, note language: "— Source: Al Jazeera (Arabic)".
+4. Non-English articles — translate content to English in your answer.
 5. Always respond in English. No hedging. No "based on my knowledge."
 6. Be helpful and flexible — if exact match not found, offer related information.
+7. At the END of your answer, on a new line, write exactly:
+   FOLLOW_UP: question1 | question2 | question3
+   These must be 3 short, specific follow-up questions the user might want to ask next, based on your answer.
 
+Source Index:
+{source_index_lines}
 News Context (from live multilingual database):
 {context_text}
 
@@ -854,10 +909,20 @@ Answer:"""
                 except:
                     pass
 
+        # ── Parse follow-up questions out of the streamed answer ──────────────
+        follow_up_questions: List[str] = []
+        clean_answer = full_answer
+        if "FOLLOW_UP:" in full_answer:
+            parts = full_answer.split("FOLLOW_UP:", 1)
+            clean_answer = parts[0].strip()
+            follow_up_raw = parts[1].strip()
+            follow_up_questions = [q.strip() for q in follow_up_raw.split("|") if q.strip()][:3]
+
         import json
         final_response = {
-            "answer": full_answer,
+            "answer": clean_answer,
             "sources": final_sources,
+            "follow_up_questions": follow_up_questions,
             "session_id": session_id
         }
         yield f"data: {json.dumps(final_response)}\n\n"
@@ -866,4 +931,4 @@ Answer:"""
         # Only persist history for authenticated users
         if not is_guest:
             retrieved_ids = [str(doc.get("doc_id")) for doc in final_sources]
-            self.chat_history_db.save_interaction(session_id, request.query, full_answer, retrieved_ids, user_id=user_id)
+            self.chat_history_db.save_interaction(session_id, request.query, clean_answer, retrieved_ids, user_id=user_id)