File size: 2,235 Bytes
b22c324 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 | from agentic_workflow.config import LLM_MODEL
from langchain_ollama import ChatOllama
import re
import json
from langchain_core.messages import HumanMessage, SystemMessage
llm = ChatOllama(model=LLM_MODEL, temperature=0)
def evaluate_response(query: str, response: str) -> dict:
"""
Scores the draft response and decides whether to retry.
Returns a dict with keys:
- score: int 1-10
- pass: bool → True means use this response
- reason: str → short explanation
- suggestion: str → how to improve (if not passing)
"""
system = """
You are a strict response quality evaluator. Given a user query and a draft response, return ONLY valid JSON:
- "score": integer 1-10 (10 = excellent)
- "pass": true if score >= 7, false otherwise
- "reason": one-sentence explanation
- "suggestion": if not passing, a short instruction to improve the response; else ""
Return ONLY the JSON object. No explanation.
"""
response_eval = llm.invoke([
SystemMessage(content=system),
HumanMessage(content=f"Query: {query}\n\nDraft response: {response}")
])
raw = response_eval.content.strip()
raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw).strip()
try:
parsed = json.loads(raw)
except json.JSONDecodeError:
parsed = {}
return {
"score": 5,
"pass": False,
"reason": "Evaluation failed",
"suggestion": "Retry with more detail",
**parsed
}
# result = evaluate_response(
# query="What is the refund policy?",
# response="According to the context provided earlier, the cancellation procedure involves a notice period of 30 days. The premium paid by you will be returned on a pro-rata basis or 25% of the annual premium, whichever is higher, will be retained. Any cancellation request sent after 30 days of commencement of the policy will be refunded on a pro-rata basis."
# )
# print("Test 1 (good response):", result)
# result = evaluate_response(
# query="What is the refund policy?",
# response="You can get a refund if you contact support."
# )
# print("Test 3 (vague response):", result) |