Removed Keyword Coverage Percentage
Browse files- agents/scorer.py +10 -14
agents/scorer.py
CHANGED
|
@@ -2,6 +2,7 @@
|
|
| 2 |
agents/scorer.py
|
| 3 |
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 4 |
ScorerAgent: Semantic, context-aware answer scoring β implements Agenda Item #4.
|
|
|
|
| 5 |
"""
|
| 6 |
|
| 7 |
import json
|
|
@@ -35,7 +36,7 @@ class ScorerAgent:
|
|
| 35 |
# ββ Step 1: Semantic Prompt Execution βββββββββββββββββββββββββββββββββ
|
| 36 |
prompt = self._build_prompt(answer_clip, question, industry, role_level, keywords)
|
| 37 |
|
| 38 |
-
# Call your core engine (now running Qwen 2.5
|
| 39 |
llm_response = self._ask(prompt, temperature=0.1, max_tokens=400)
|
| 40 |
|
| 41 |
# ββ Step 2: Parse and Extract Structured JSON Metrics βββββββββββββββββ
|
|
@@ -53,12 +54,11 @@ class ScorerAgent:
|
|
| 53 |
"numeric_score": float(parsed_data["score"]),
|
| 54 |
"hit_keywords": parsed_data["hit_keywords"],
|
| 55 |
"missed_keywords": parsed_data["missed_keywords"],
|
| 56 |
-
"coverage_pct":
|
| 57 |
"star_hint": star_hint,
|
| 58 |
}
|
| 59 |
|
| 60 |
-
|
| 61 |
-
# ββ Advanced Prompt Builder βββββββββββββββββββββββββββββββββββββββββββββββ
|
| 62 |
def _build_prompt(self, answer: str, question: str, industry: str,
|
| 63 |
role_level: str, keywords: list) -> str:
|
| 64 |
keywords_str = ", ".join(keywords) if keywords else "General Industry Domain"
|
|
@@ -83,6 +83,7 @@ Provide your evaluation in a raw, clean JSON format. Do not include any markdown
|
|
| 83 |
"weakness": "One clear sentence identifying the biggest structural or domain gap.",
|
| 84 |
"fix": "One clear sentence containing actionable advice to improve the response."
|
| 85 |
}}"""
|
|
|
|
| 86 |
# ββ Robust Safe JSON Parser βββββββββββββββββββββββββββββββββββββββββββββββ
|
| 87 |
def _parse_json_response(self, raw_text: str, original_keywords: list) -> dict:
|
| 88 |
"""Safely parses LLM output and provides bulletproof defaults if parsing fails."""
|
|
@@ -94,12 +95,8 @@ Provide your evaluation in a raw, clean JSON format. Do not include any markdown
|
|
| 94 |
score = int(data.get("score", 6))
|
| 95 |
hit_kw = data.get("hit_keywords", [])
|
| 96 |
missed_kw = data.get("missed_keywords", [])
|
| 97 |
-
|
| 98 |
-
# Re-verify keyword totals cleanly
|
| 99 |
-
total_kw = len(original_keywords) if original_keywords else 1
|
| 100 |
-
coverage_pct = round((len(hit_kw) / total_kw) * 100, 1)
|
| 101 |
|
| 102 |
-
# Reconstruct the UI display text string dynamically
|
| 103 |
feedback_str = (
|
| 104 |
f"Relevant: YES\n"
|
| 105 |
f"Score: {score}/10\n"
|
|
@@ -112,17 +109,16 @@ Provide your evaluation in a raw, clean JSON format. Do not include any markdown
|
|
| 112 |
"score": score,
|
| 113 |
"hit_keywords": hit_kw,
|
| 114 |
"missed_keywords": missed_kw,
|
| 115 |
-
"coverage_pct":
|
| 116 |
"feedback_str": feedback_str
|
| 117 |
}
|
| 118 |
except Exception:
|
| 119 |
# Safe production fallback configuration
|
| 120 |
-
total_kw = len(original_keywords) if original_keywords else 1
|
| 121 |
return {
|
| 122 |
"score": 6,
|
| 123 |
-
"hit_keywords": original_keywords[:1],
|
| 124 |
-
"missed_keywords": original_keywords[1:],
|
| 125 |
-
"coverage_pct":
|
| 126 |
"feedback_str": "Relevant: YES\nScore: 6/10\nStrength: Answer was recorded successfully.\nWeakness: Evaluation parsing exception occurred.\nFix: Try expanding your STAR answer metrics slightly."
|
| 127 |
}
|
| 128 |
|
|
|
|
| 2 |
agents/scorer.py
|
| 3 |
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 4 |
ScorerAgent: Semantic, context-aware answer scoring β implements Agenda Item #4.
|
| 5 |
+
Optimized to remove visibility of the coverage percentage metric entirely.
|
| 6 |
"""
|
| 7 |
|
| 8 |
import json
|
|
|
|
| 36 |
# ββ Step 1: Semantic Prompt Execution βββββββββββββββββββββββββββββββββ
|
| 37 |
prompt = self._build_prompt(answer_clip, question, industry, role_level, keywords)
|
| 38 |
|
| 39 |
+
# Call your core engine (now running Qwen 2.5 7B via Chat Completion)
|
| 40 |
llm_response = self._ask(prompt, temperature=0.1, max_tokens=400)
|
| 41 |
|
| 42 |
# ββ Step 2: Parse and Extract Structured JSON Metrics βββββββββββββββββ
|
|
|
|
| 54 |
"numeric_score": float(parsed_data["score"]),
|
| 55 |
"hit_keywords": parsed_data["hit_keywords"],
|
| 56 |
"missed_keywords": parsed_data["missed_keywords"],
|
| 57 |
+
"coverage_pct": 0.0, # π¨ FORCE ZERO TO STRIP PERCENT METRICS FROM THE GRADIO PAYLOAD
|
| 58 |
"star_hint": star_hint,
|
| 59 |
}
|
| 60 |
|
| 61 |
+
# ββ Advanced Prompt Builder βββββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 62 |
def _build_prompt(self, answer: str, question: str, industry: str,
|
| 63 |
role_level: str, keywords: list) -> str:
|
| 64 |
keywords_str = ", ".join(keywords) if keywords else "General Industry Domain"
|
|
|
|
| 83 |
"weakness": "One clear sentence identifying the biggest structural or domain gap.",
|
| 84 |
"fix": "One clear sentence containing actionable advice to improve the response."
|
| 85 |
}}"""
|
| 86 |
+
|
| 87 |
# ββ Robust Safe JSON Parser βββββββββββββββββββββββββββββββββββββββββββββββ
|
| 88 |
def _parse_json_response(self, raw_text: str, original_keywords: list) -> dict:
|
| 89 |
"""Safely parses LLM output and provides bulletproof defaults if parsing fails."""
|
|
|
|
| 95 |
score = int(data.get("score", 6))
|
| 96 |
hit_kw = data.get("hit_keywords", [])
|
| 97 |
missed_kw = data.get("missed_keywords", [])
|
|
|
|
|
|
|
|
|
|
|
|
|
| 98 |
|
| 99 |
+
# Reconstruct the UI display text string dynamically without keyword metric lines
|
| 100 |
feedback_str = (
|
| 101 |
f"Relevant: YES\n"
|
| 102 |
f"Score: {score}/10\n"
|
|
|
|
| 109 |
"score": score,
|
| 110 |
"hit_keywords": hit_kw,
|
| 111 |
"missed_keywords": missed_kw,
|
| 112 |
+
"coverage_pct": 0.0, # π¨ BYPASS THE MATH CALCULATION COMPLETELY
|
| 113 |
"feedback_str": feedback_str
|
| 114 |
}
|
| 115 |
except Exception:
|
| 116 |
# Safe production fallback configuration
|
|
|
|
| 117 |
return {
|
| 118 |
"score": 6,
|
| 119 |
+
"hit_keywords": original_keywords[:1] if original_keywords else [],
|
| 120 |
+
"missed_keywords": original_keywords[1:] if original_keywords else [],
|
| 121 |
+
"coverage_pct": 0.0, # π¨ BYPASS THE MATH CALCULATION COMPLETELY
|
| 122 |
"feedback_str": "Relevant: YES\nScore: 6/10\nStrength: Answer was recorded successfully.\nWeakness: Evaluation parsing exception occurred.\nFix: Try expanding your STAR answer metrics slightly."
|
| 123 |
}
|
| 124 |
|