Update api/server.py

api/server.py

@@ -58,9 +58,13 @@ WARMUP_STARTED = False
 CLARE_ENABLE_WARMUP = os.getenv("CLARE_ENABLE_WARMUP", "1").strip() == "1"
 CLARE_WARMUP_BLOCK_READY = os.getenv("CLARE_WARMUP_BLOCK_READY", "0").strip() == "1"
 
+# Dataset logging (create_example)
 CLARE_ENABLE_LANGSMITH_LOG = os.getenv("CLARE_ENABLE_LANGSMITH_LOG", "0").strip() == "1"
 CLARE_LANGSMITH_ASYNC = os.getenv("CLARE_LANGSMITH_ASYNC", "1").strip() == "1"
 
+# Feedback logging (create_feedback -> attach to run_id)
+CLARE_ENABLE_LANGSMITH_FEEDBACK = os.getenv("CLARE_ENABLE_LANGSMITH_FEEDBACK", "1").strip() == "1"
+
 # ----------------------------
 # App
 # ----------------------------
@@ -172,6 +176,9 @@ if (Client is not None) and CLARE_ENABLE_LANGSMITH_LOG:
 
 
 def _log_event_to_langsmith(data: Dict[str, Any]):
+    """
+    Dataset logging: create_example into LS_DATASET_NAME
+    """
     if _ls_client is None:
         return
 
@@ -205,6 +212,50 @@ def _log_event_to_langsmith(data: Dict[str, Any]):
         _do()
 
 
+def _write_feedback_to_langsmith_run(
+    run_id: str,
+    rating: str,
+    comment: str = "",
+    tags: Optional[List[str]] = None,
+    metadata: Optional[Dict[str, Any]] = None,
+) -> bool:
+    """
+    Run-level feedback: create_feedback attached to a specific run_id.
+    This is separate from dataset create_example logging.
+    """
+    if not CLARE_ENABLE_LANGSMITH_FEEDBACK:
+        return False
+    if Client is None:
+        return False
+
+    rid = (run_id or "").strip()
+    if not rid:
+        return False
+
+    try:
+        ls = Client()
+        score = 1 if rating == "helpful" else 0
+
+        meta = metadata or {}
+        if tags is not None:
+            meta["tags"] = tags
+
+        if LS_PROJECT:
+            meta.setdefault("langsmith_project", LS_PROJECT)
+
+        ls.create_feedback(
+            run_id=rid,
+            key="ui_rating",
+            score=score,
+            comment=comment or "",
+            metadata=meta,
+        )
+        return True
+    except Exception as e:
+        print("[langsmith] create_feedback failed:", repr(e))
+        return False
+
+
 # ----------------------------
 # Health endpoints
 # ----------------------------
@@ -220,6 +271,7 @@ def health():
         "ready": bool(WARMUP_DONE) if CLARE_WARMUP_BLOCK_READY else True,
         "langsmith_enabled": bool(CLARE_ENABLE_LANGSMITH_LOG),
         "langsmith_async": bool(CLARE_LANGSMITH_ASYNC),
+        "langsmith_feedback_enabled": bool(CLARE_ENABLE_LANGSMITH_FEEDBACK),
         "ts": int(time.time()),
     }
 
@@ -264,9 +316,6 @@ MICRO_QUIZ_INSTRUCTION = (
 )
 
 
-
-
-
 # ----------------------------
 # Schemas
 # ----------------------------
@@ -282,6 +331,7 @@ class ChatReq(BaseModel):
     language_preference: str = "Auto"
     doc_type: str = "Syllabus"
 
+
 class QuizStartReq(BaseModel):
     user_id: str
     language_preference: str = "Auto"
@@ -307,6 +357,10 @@ class FeedbackReq(BaseModel):
 
     user_id: str
     rating: str  # "helpful" | "not_helpful"
+
+    # NEW: attach feedback to a specific LangSmith run
+    run_id: Optional[str] = None
+
     assistant_message_id: Optional[str] = None
 
     assistant_text: str
@@ -355,6 +409,7 @@ def chat(req: ChatReq):
             ),
             "refs": [],
             "latency_ms": 0.0,
+            "run_id": None,
         }
 
     t0 = time.time()
@@ -376,7 +431,7 @@ def chat(req: ChatReq):
     marks_ms["rag_retrieve_done"] = (time.time() - t0) * 1000.0
 
     try:
-        answer, new_history = chat_with_clare(
+        answer, new_history, run_id = chat_with_clare(
             message=msg,
             history=sess["history"],
             model_name=sess["model_name"],
@@ -441,6 +496,7 @@ def chat(req: ChatReq):
             "learning_mode": req.learning_mode,
             "doc_type": req.doc_type,
             "refs": refs,
+            "run_id": run_id,  # NEW: keep in dataset metadata for debugging
         }
     )
 
@@ -451,8 +507,10 @@ def chat(req: ChatReq):
         ),
         "refs": refs,
         "latency_ms": total_ms,
+        "run_id": run_id,  # NEW: FE attaches feedback to this run
     }
 
+
 @app.post("/api/quiz/start")
 def quiz_start(req: QuizStartReq):
     user_id = (req.user_id or "").strip()
@@ -475,7 +533,7 @@ def quiz_start(req: QuizStartReq):
     )
 
     try:
-        answer, new_history = chat_with_clare(
+        answer, new_history, run_id = chat_with_clare(
             message=quiz_instruction,
             history=sess["history"],              # 直接接在当前会话 history 后面
             model_name=sess["model_name"],
@@ -518,6 +576,7 @@ def quiz_start(req: QuizStartReq):
             "refs": refs,
             "rag_used_chunks_count": len(rag_used_chunks or []),
             "history_len": len(sess["history"]),
+            "run_id": run_id,  # NEW
         }
     )
 
@@ -528,6 +587,7 @@ def quiz_start(req: QuizStartReq):
         ),
         "refs": refs,
         "latency_ms": total_ms,
+        "run_id": run_id,  # NEW
     }
 
 
@@ -612,6 +672,7 @@ def api_feedback(req: FeedbackReq):
     tags = req.tags or []
     timestamp_ms = int(req.timestamp_ms or int(time.time() * 1000))
 
+    # 1) Dataset event log (what you already have)
     _log_event_to_langsmith(
         {
             "experiment_id": EXPERIMENT_ID,
@@ -622,6 +683,7 @@ def api_feedback(req: FeedbackReq):
             "timestamp_ms": timestamp_ms,
             "rating": rating,
             "assistant_message_id": req.assistant_message_id,
+            "run_id": req.run_id,  # NEW
 
             # Keep the Example readable:
             "question": user_text,            # what user asked (optional)
@@ -636,7 +698,28 @@ def api_feedback(req: FeedbackReq):
         }
     )
 
-    return {"ok": True}
+    # 2) Run-level feedback (attach to actual LangSmith run)
+    #    Only works when FE provides run_id and LangSmith credentials are configured.
+    wrote_run_feedback = False
+    if req.run_id:
+        wrote_run_feedback = _write_feedback_to_langsmith_run(
+            run_id=req.run_id,
+            rating=rating,
+            comment=comment,
+            tags=tags,
+            metadata={
+                "experiment_id": EXPERIMENT_ID,
+                "student_id": user_id,
+                "student_name": student_name,
+                "assistant_message_id": req.assistant_message_id,
+                "learning_mode": req.learning_mode,
+                "doc_type": req.doc_type,
+                "refs": refs,
+                "timestamp_ms": timestamp_ms,
+            },
+        )
+
+    return {"ok": True, "run_feedback_written": wrote_run_feedback}
 
 
 @app.post("/api/export")
@@ -698,4 +781,4 @@ def spa_fallback(full_path: str, request: Request):
     return JSONResponse(
         {"detail": "web/build not found. Build frontend first (web/build/index.html)."},
         status_code=500,
-    )
+    )