Update api/clare_core.py

api/clare_core.py

@@ -52,10 +52,12 @@ def _count_text_tokens(text: str, model: str = "") -> int:
 
 
 def _count_messages_tokens(messages: List[Dict[str, str]], model: str = "") -> int:
-    # engineering approximation for chat messages overhead
+    """
+    Approximation for chat messages overhead.
+    """
     total = 0
     for m in messages or []:
-        total += 4
+        total += 4  # role/content wrappers
         total += _count_text_tokens(str(m.get("role", "")), model=model)
         total += _count_text_tokens(str(m.get("content", "")), model=model)
     total += 2
@@ -89,6 +91,10 @@ def _truncate_to_tokens(text: str, max_tokens: int, model: str = "") -> str:
     return enc.decode(ids[:max_tokens])
 
 
+def model_name_or_default(x: str) -> str:
+    return (x or "").strip() or DEFAULT_MODEL
+
+
 # ----------------------------
 # Speed knobs (HARD LIMITS)
 # ----------------------------
@@ -404,7 +410,8 @@ def _log_prompt_token_breakdown(
         "system_tokens": _count_text_tokens(system_prompt, model=model_name),
         "rag_tokens": _count_text_tokens(rag_context or "", model=model_name),
         "history_tokens": sum(
-            _count_text_tokens(u or "", model=model_name) + _count_text_tokens(a or "", model=model_name)
+            _count_text_tokens(u or "", model=model_name)
+            + _count_text_tokens(a or "", model=model_name)
             for u, a in (trimmed_history or [])
         ),
         "user_tokens": _count_text_tokens(user_message or "", model=model_name),
@@ -412,7 +419,7 @@ def _log_prompt_token_breakdown(
         "history_turns_kept": len(trimmed_history or []),
         "max_rag_tokens_in_prompt": MAX_RAG_TOKENS_IN_PROMPT,
         "max_output_tokens": DEFAULT_MAX_OUTPUT_TOKENS,
-        "model": model_name or DEFAULT_MODEL,
+        "model": model_name,
     }
     print("[LLM_PROMPT_TOKENS] " + json.dumps(stats, ensure_ascii=False))
     return stats
@@ -426,118 +433,105 @@ def safe_chat_completion_profiled(
     op: str = "chat",
     temperature: float = 0.5,
     max_tokens: Optional[int] = None,
+    timeout: int = 20,
 ) -> Tuple[str, Dict]:
     """
-    Returns:
-      - answer text
-      - profiling dict {ttft_ms, llm_total_ms, gen_ms, output_tokens_est, tokens_per_sec_est, streaming_used}
+    Streaming-based call to measure TTFT and tokens/sec (estimated).
+    Returns: (text, prof)
+    prof includes:
+      model, llm_total_ms, ttft_ms, gen_ms, output_tokens_est, tokens_per_sec_est, streaming_used, max_tokens
     """
-    preferred_model = model_name or DEFAULT_MODEL
+    t0 = time.perf_counter()
+
+    preferred_model = model_name_or_default(model_name)
     max_tokens = int(max_tokens or DEFAULT_MAX_OUTPUT_TOKENS)
 
+    used_model = preferred_model
     last_error: Optional[Exception] = None
 
     for attempt in range(2):
-        current_model = preferred_model if attempt == 0 else DEFAULT_MODEL
-
-        # 1) Try streaming for real TTFT
-        t0 = time.perf_counter()
+        used_model = preferred_model if attempt == 0 else DEFAULT_MODEL
         try:
-            resp = client.chat.completions.create(
-                model=current_model,
+            first_token_ms: Optional[float] = None
+            text_parts: List[str] = []
+            output_chars = 0
+
+            stream = client.chat.completions.create(
+                model=used_model,
                 messages=messages,
                 temperature=temperature,
                 max_tokens=max_tokens,
-                timeout=20,
                 stream=True,
+                timeout=timeout,
             )
 
-            first_token_t = None
-            out_parts: List[str] = []
-            for event in resp:
-                # OpenAI-style: event.choices[0].delta.content
+            for chunk in stream:
+                if first_token_ms is None:
+                    first_token_ms = (time.perf_counter() - t0) * 1000.0
+
+                delta = None
                 try:
-                    delta = event.choices[0].delta.content  # type: ignore
+                    delta = chunk.choices[0].delta
                 except Exception:
                     delta = None
-                if not delta:
-                    continue
-                if first_token_t is None:
-                    first_token_t = time.perf_counter()
-                out_parts.append(delta)
-
-            t_end = time.perf_counter()
-            answer = "".join(out_parts)
-
-            ttft_ms = None if first_token_t is None else (first_token_t - t0) * 1000.0
-            total_ms = (t_end - t0) * 1000.0
-            gen_ms = None if first_token_t is None else (t_end - first_token_t) * 1000.0
-            out_tokens = _count_text_tokens(answer, model=current_model)
-            tokens_per_sec = None
-            if gen_ms and gen_ms > 0:
-                tokens_per_sec = out_tokens / (gen_ms / 1000.0)
+
+                piece = ""
+                if delta is not None:
+                    piece = getattr(delta, "content", "") or ""
+                else:
+                    try:
+                        piece = chunk.choices[0].message.content or ""
+                    except Exception:
+                        piece = ""
+
+                if piece:
+                    text_parts.append(piece)
+                    output_chars += len(piece)
+
+            full_text = "".join(text_parts)
+            llm_total_ms = (time.perf_counter() - t0) * 1000.0
+            ttft_ms = float(first_token_ms or llm_total_ms)
+            gen_ms = max(0.0, llm_total_ms - ttft_ms)
+
+            # output tokens est (rough)
+            if re.search(r"[\u4e00-\u9fff]", full_text or ""):
+                output_tokens_est = int(output_chars / 2.0) if output_chars else 0
+            else:
+                output_tokens_est = int(output_chars / 4.0) if output_chars else 0
+
+            tokens_per_sec_est = (
+                (output_tokens_est / (gen_ms / 1000.0)) if gen_ms > 1 else None
+            )
 
             prof = {
+                "model": used_model,
                 "streaming_used": True,
+                "max_tokens": max_tokens,
+                "output_tokens_est": output_tokens_est,
+                "tokens_per_sec_est": tokens_per_sec_est,
                 "ttft_ms": ttft_ms,
-                "llm_total_ms": total_ms,
                 "gen_ms": gen_ms,
-                "output_tokens_est": out_tokens,
-                "tokens_per_sec_est": tokens_per_sec,
-                "model": current_model,
-                "max_tokens": max_tokens,
+                "llm_total_ms": llm_total_ms,
             }
-            print("[LLM_PROFILING] " + json.dumps(prof, ensure_ascii=False))
-            return answer, prof
+            return full_text, prof
 
         except Exception as e:
             last_error = e
-            # fall through to non-stream fallback below
-
-        # 2) Non-stream fallback (TTFT not available; approximate)
-        try:
-            t0 = time.perf_counter()
-            resp2 = client.chat.completions.create(
-                model=current_model,
-                messages=messages,
-                temperature=temperature,
-                max_tokens=max_tokens,
-                timeout=20,
-            )
-            t_end = time.perf_counter()
-            answer = resp2.choices[0].message.content or ""
-
-            total_ms = (t_end - t0) * 1000.0
-            out_tokens = _count_text_tokens(answer, model=current_model)
-            tokens_per_sec = None
-            if total_ms > 0:
-                tokens_per_sec = out_tokens / (total_ms / 1000.0)
-
-            prof = {
-                "streaming_used": False,
-                "ttft_ms": None,  # not measurable without stream
-                "llm_total_ms": total_ms,
-                "gen_ms": None,
-                "output_tokens_est": out_tokens,
-                "tokens_per_sec_est": tokens_per_sec,
-                "model": current_model,
-                "max_tokens": max_tokens,
-                "note": "non-stream fallback; ttft_ms unavailable",
-            }
-            print("[LLM_PROFILING] " + json.dumps(prof, ensure_ascii=False))
-            return answer, prof
-
-        except Exception as e:
             print(
-                f"[safe_chat_completion_profiled][{op}] attempt {attempt+1} "
-                f"failed with model={current_model}: {repr(e)}"
+                f"[safe_chat_completion_profiled][{op}] attempt {attempt+1} failed: {repr(e)}"
             )
-            last_error = e
-            if current_model == DEFAULT_MODEL or attempt == 1:
+            if attempt == 1:
                 break
 
-    return build_error_message(last_error or Exception("unknown error"), lang, op), {
-        "streaming_used": False,
+    return build_error_message(last_error or Exception("unknown"), lang, op), {
+        "model": used_model,
+        "streaming_used": True,
+        "max_tokens": max_tokens,
+        "output_tokens_est": 0,
+        "tokens_per_sec_est": None,
+        "ttft_ms": None,
+        "gen_ms": None,
+        "llm_total_ms": (time.perf_counter() - t0) * 1000.0,
         "error": repr(last_error) if last_error else "unknown",
     }
 
@@ -552,7 +546,10 @@ def build_messages(
     weaknesses: Optional[List[str]],
     cognitive_state: Optional[Dict[str, int]],
     rag_context: Optional[str] = None,
+    model_name: str = "",
 ) -> List[Dict[str, str]]:
+    model_for_count = model_name_or_default(model_name)
+
     messages: List[Dict[str, str]] = [
         {"role": "system", "content": CLARE_SYSTEM_PROMPT}
     ]
@@ -593,9 +590,7 @@ def build_messages(
         messages.append(
             {
                 "role": "system",
-                "content": (
-                    "Student struggles (recent). Be extra clear on these: " + weak_text
-                ),
+                "content": "Student struggles (recent). Be extra clear on these: " + weak_text,
             }
         )
 
@@ -606,18 +601,14 @@ def build_messages(
             messages.append(
                 {
                     "role": "system",
-                    "content": (
-                        "Student under HIGH cognitive load. Use simpler language and shorter steps."
-                    ),
+                    "content": "Student under HIGH cognitive load. Use simpler language and shorter steps.",
                 }
             )
         elif mastery >= 2 and mastery >= confusion + 1:
             messages.append(
                 {
                     "role": "system",
-                    "content": (
-                        "Student comfortable. You may go slightly deeper and add a follow-up question."
-                    ),
+                    "content": "Student comfortable. You may go slightly deeper and add a follow-up question.",
                 }
             )
 
@@ -631,25 +622,20 @@ def build_messages(
         weaknesses=weaknesses,
         cognitive_state=cognitive_state,
     )
-    messages.append(
-        {
-            "role": "system",
-            "content": "Session memory: " + session_memory_text,
-        }
-    )
+    messages.append({"role": "system", "content": "Session memory: " + session_memory_text})
 
-    # RAG context: enforce token cap here (in addition to rag_engine caps)
+    # RAG context: enforce token cap here
     rag_text_for_prompt = ""
     if rag_context:
         rag_text_for_prompt = _truncate_to_tokens(
-            rag_context, max_tokens=MAX_RAG_TOKENS_IN_PROMPT, model=model_name_or_default(DEFAULT_MODEL)
+            rag_context,
+            max_tokens=MAX_RAG_TOKENS_IN_PROMPT,
+            model=model_for_count,
         )
         messages.append(
             {
                 "role": "system",
-                "content": (
-                    "Relevant excerpts (use as primary grounding):\n\n" + rag_text_for_prompt
-                ),
+                "content": "Relevant excerpts (use as primary grounding):\n\n" + rag_text_for_prompt,
             }
         )
 
@@ -669,16 +655,12 @@ def build_messages(
         rag_context=rag_text_for_prompt,
         trimmed_history=trimmed_history,
         user_message=user_message,
-        model_name=(DEFAULT_MODEL or ""),
+        model_name=model_for_count,
     )
 
     return messages
 
 
-def model_name_or_default(x: str) -> str:
-    return x or DEFAULT_MODEL
-
-
 @traceable(run_type="chain", name="chat_with_clare")
 def chat_with_clare(
     message: str,
@@ -691,7 +673,13 @@ def chat_with_clare(
     weaknesses: Optional[List[str]],
     cognitive_state: Optional[Dict[str, int]],
     rag_context: Optional[str] = None,
-) -> Tuple[str, List[Tuple[str, str]]]:
+) -> Tuple[str, List[Tuple[str, str]], Dict]:
+    """
+    Returns:
+      answer: str
+      history: List[(user, assistant)]
+      llm_stats: Dict (TTFT + tokens/sec est + prompt token breakdown printed in logs)
+    """
     try:
         set_run_metadata(
             learning_mode=learning_mode,
@@ -711,29 +699,41 @@ def chat_with_clare(
         weaknesses=weaknesses,
         cognitive_state=cognitive_state,
         rag_context=rag_context,
+        model_name=model_name,
     )
-    answer, prof = safe_chat_completion_profiled(...)
+
+    # IMPORTANT: pass messages + lang (fixes your HTTP 500)
+    answer, prof = safe_chat_completion_profiled(
+        model_name=model_name,
+        messages=messages,
+        lang=language_preference,
+        op="chat",
+        temperature=0.5,
+        max_tokens=DEFAULT_MAX_OUTPUT_TOKENS,
+        timeout=20,
+    )
+
     history = history + [(message, answer)]
-    
+
     llm_stats = {
-      "llm_profile": {
-          "model": prof.get("model"),
-          "streaming_used": prof.get("streaming_used"),
-          "max_tokens": prof.get("max_tokens"),
-          "output_tokens_est": prof.get("output_tokens_est"),
-          "tokens_per_sec_est": prof.get("tokens_per_sec_est"),
-      },
-      "marks_ms": {
-          "llm_first_token": prof.get("ttft_ms"),
-          "llm_done": prof.get("llm_total_ms"),
-      },
-      "segments_ms": {
-          "llm_ttft_ms": prof.get("ttft_ms"),
-          "llm_gen_ms": prof.get("gen_ms"),
-          "llm_done": prof.get("llm_total_ms"),
-      },
+        "llm_profile": {
+            "model": prof.get("model"),
+            "streaming_used": prof.get("streaming_used"),
+            "max_tokens": prof.get("max_tokens"),
+            "output_tokens_est": prof.get("output_tokens_est"),
+            "tokens_per_sec_est": prof.get("tokens_per_sec_est"),
+        },
+        "marks_ms": {
+            "llm_first_token": prof.get("ttft_ms"),
+            "llm_done": prof.get("llm_total_ms"),
+        },
+        "segments_ms": {
+            "llm_ttft_ms": prof.get("ttft_ms"),
+            "llm_gen_ms": prof.get("gen_ms"),
+            "llm_done": prof.get("llm_total_ms"),
+        },
     }
-    
+
     return answer, history, llm_stats
 
 
@@ -785,9 +785,7 @@ def summarize_conversation(
         {"role": "system", "content": CLARE_SYSTEM_PROMPT},
         {
             "role": "system",
-            "content": (
-                "Produce a concept-only summary. Use bullet points. No off-topic text."
-            ),
+            "content": "Produce a concept-only summary. Use bullet points. No off-topic text.",
         },
         {"role": "system", "content": f"Course topics: {topics_text}"},
         {"role": "system", "content": f"Student difficulties: {weakness_text}"},
@@ -805,5 +803,6 @@ def summarize_conversation(
         op="summary",
         temperature=0.4,
         max_tokens=DEFAULT_MAX_OUTPUT_TOKENS,
+        timeout=20,
     )
     return summary_text