Spaces:
Sleeping
Sleeping
File size: 3,033 Bytes
b2171fc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | import re
import json
import datetime
import pandas as pd
from src.api_clients import BACKENDS
def split_json_objects(s):
objs, depth, start = [], 0, None
for i, ch in enumerate(s):
if ch == "{":
if depth == 0:
start = i
depth += 1
elif ch == "}":
depth -= 1
if depth == 0 and start is not None:
objs.append(s[start:i+1])
return objs
def evaluate_with_judges(conversation, selected_models, variant, *weights_and_temp, prompt_template):
weights, temperature = list(weights_and_temp[:-1]), weights_and_temp[-1]
if not conversation.strip():
raise ValueError("Conversation input is empty.")
from src.conversation import structure_conversation
structured = structure_conversation(conversation)
system_msg = (
"You are Judge-Care-Lock, a rigorous evaluator of AI-therapist dialogues.\n"
"1. Use ONLY the transcript—quote it for every decision.\n"
"2. Apply the multi-layer rubric exactly; do NOT invent scales.\n"
"3. Return valid JSON matching the schema; no extra text."
)
user_prompt = prompt_template.replace("{CONVERSATION}", structured)
metrics_rows = []
comments_map = {}
tokens_map = {}
pros_map = {}
cons_map = {}
summary_map = {}
for model_name in selected_models:
fn = BACKENDS[model_name]
raw, toks = fn(system_msg, user_prompt, temperature)
tokens_map[model_name] = toks
clean = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw, flags=re.MULTILINE).strip()
objs = split_json_objects(clean)
if not objs:
raise ValueError(f"No valid JSON from {model_name}:\n{clean}")
try:
parsed = json.loads(objs[0])
except json.JSONDecodeError as e:
raise ValueError(f"Invalid JSON from {model_name}: {str(e)}")
row = {"Model": model_name}
total_score = 0.0
for idx, (m, data) in enumerate(parsed["metrics"].items()):
score = data.get("score", 0.0)
row[m] = score
total_score += score * weights[idx]
row["Total"] = round(total_score, 2)
metrics_rows.append(row)
comments_map[model_name] = parsed
pros_map[model_name] = parsed.get("positive", [])
cons_map[model_name] = parsed.get("negative", [])
summary_map[model_name] = parsed.get("summary", "")
ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"/tmp/carelock_eval_{ts}.json"
combined = {
"metrics_table": metrics_rows,
"parsed_per_model": comments_map,
"tokens_per_model": tokens_map,
"pros_per_model": pros_map,
"cons_per_model": cons_map,
"summary_per_model": summary_map
}
with open(filename, "w", encoding="utf-8") as f:
json.dump(combined, f, indent=2)
return (pd.DataFrame(metrics_rows), comments_map, tokens_map,
pros_map, cons_map, summary_map, filename) |