Update app/graph/nodes/evaluator.py

app/graph/nodes/evaluator.py

@@ -1,43 +1,109 @@
-# app/graph/nodes/evaluator.py
-from app.core.llm_engine import llm
-from app.core.prompts.evaluator_prompt import evaluator_prompt
-from langchain_core.output_parsers import StrOutputParser
-import json
-
-chain = evaluator_prompt | llm | StrOutputParser()
-
-
-def evaluator_node(state):
-    query = state.get("query")
-    answer = state.get("final_answer")
-    context = state.get("context", "")
-
-    try:
-        response = chain.invoke({
-                "query": query,
-                "answer": answer,
-                "context": context
-            })
-
-        # 🔥 clean response (important)
-        response = response.strip()
-
-        # sometimes model adds ```json
-        if response.startswith("```"):
-            response = response.replace("```json", "").replace("```", "").strip()
-
-        evaluation = json.loads(response)
-
-    except Exception as e:
-        print("EVALUATOR ERROR →", e)
-
-        evaluation = {
-            "relevance_score": 0.5,
-            "context_usage": 0.5,
-            "hallucination": True
-        }
-
-    return {
-        **state,
-        "evaluation": evaluation
-    }
+# # app/graph/nodes/evaluator.py
+# from app.core.llm_engine import llm
+# from app.core.prompts.evaluator_prompt import evaluator_prompt
+# from langchain_core.output_parsers import StrOutputParser
+# import json
+
+# chain = evaluator_prompt | llm | StrOutputParser()
+
+
+# def evaluator_node(state):
+#     query = state.get("query")
+#     answer = state.get("final_answer")
+#     context = state.get("context", "")
+
+#     try:
+#         response = chain.invoke({
+#                 "query": query,
+#                 "answer": answer,
+#                 "context": context
+#             })
+
+#         # 🔥 clean response (important)
+#         response = response.strip()
+
+#         # sometimes model adds ```json
+#         if response.startswith("```"):
+#             response = response.replace("```json", "").replace("```", "").strip()
+
+#         evaluation = json.loads(response)
+
+#     except Exception as e:
+#         print("EVALUATOR ERROR →", e)
+
+#         evaluation = {
+#             "relevance_score": 0.5,
+#             "context_usage": 0.5,
+#             "hallucination": True
+#         }
+
+#     return {
+#         **state,
+#         "evaluation": evaluation
+#     }
+
+
+
+
+
+
+
+
+# app/graph/nodes/evaluator.py
+from app.core.llm_engine import llm
+from app.core.prompts.evaluator_prompt import evaluator_prompt
+from langchain_core.output_parsers import StrOutputParser
+import json, re
+
+chain = evaluator_prompt | llm | StrOutputParser()
+
+
+def evaluator_node(state):
+    query = state.get("query")
+    answer = state.get("final_answer")
+    context = state.get("context", "")
+
+    try:
+
+        response = chain.invoke({
+            "query": query,
+            "answer": answer,
+            "context": context
+        }).strip()
+
+        # 🔥 remove markdown/code blocks
+        response = re.sub(r"```.*?```", "", response, flags=re.DOTALL).strip()
+
+        # 🔥 extract JSON only
+        match = re.search(r"\{.*\}", response, re.DOTALL)
+
+        if match:
+            response = match.group(0)
+
+        # 🔥 validate JSON start
+        if not response.startswith("{"):
+            raise ValueError("Invalid JSON from LLM")
+
+        evaluation = json.loads(response)
+
+        # 🔥 clamp values
+        evaluation = {
+            "relevance_score": min(max(evaluation.get("relevance_score", 0), 0), 1),
+            "context_usage": min(max(evaluation.get("context_usage", 0), 0), 1),
+            "hallucination": bool(evaluation.get("hallucination", True))
+        }
+
+    except Exception as e:
+        print("EVALUATOR ERROR →", e)
+
+        evaluation = {
+            "relevance_score": 0.5,
+            "context_usage": 0.5,
+            "hallucination": True
+        }
+
+    return {
+        **state,
+        "evaluation": evaluation
+    }
+