Upload 3 files

src/api_clients.py

@@ -0,0 +1,117 @@
+import os
+import anthropic
+import openai
+from openai import OpenAI
+from anthropic import Anthropic
+
+# ✅ Debug: log versions and proxy env vars
+print(f"Anthropic version: {anthropic.__version__}")
+print(f"OpenAI version: {openai.__version__}")
+print(f"Proxy env vars before cleanup: HTTP_PROXY={os.getenv('HTTP_PROXY')}, HTTPS_PROXY={os.getenv('HTTPS_PROXY')}, NO_PROXY={os.getenv('NO_PROXY')}")
+
+# ✅ Forcefully disable proxies for HF Spaces
+os.environ["HTTP_PROXY"] = ""
+os.environ["HTTPS_PROXY"] = ""
+os.environ["NO_PROXY"] = "*"
+
+# ✅ Patch any client constructors to ignore 'proxies'
+def safe_init(client_cls):
+    """Wrap client __init__ to strip 'proxies' kwarg injected by Spaces."""
+    orig_init = client_cls.__init__
+    def wrapped_init(self, *args, **kwargs):
+        if "proxies" in kwargs:
+            print(f"[Patch] Stripped unexpected 'proxies' from {client_cls.__name__}")
+            kwargs.pop("proxies", None)
+        return orig_init(self, *args, **kwargs)
+    client_cls.__init__ = wrapped_init
+
+# Apply patch to both clients
+safe_init(OpenAI)
+safe_init(Anthropic)
+
+def init_clients():
+    """Initialize API clients using HF Spaces Repository Secrets."""
+    try:
+        openai_key = os.getenv("OPENAI_API_KEY")
+        anthropic_key = os.getenv("ANTHROPIC_API_KEY")
+        deepseek_key = os.getenv("DEEPSEEK_API_KEY")
+
+        if not all([openai_key, anthropic_key, deepseek_key]):
+            raise ValueError("Missing one or more API keys in HF Spaces Repository Secrets.")
+
+        # ✅ Initialize OpenAI client
+        openai_client = OpenAI(api_key=openai_key)
+
+        # ✅ Initialize Anthropic client
+        anthropic_client = Anthropic(api_key=anthropic_key)
+
+        # ✅ Initialize DeepSeek client (via OpenAI interface)
+        deepseek_client = OpenAI(
+            api_key=deepseek_key,
+            base_url="https://api.deepseek.com/v1"
+        )
+
+        return openai_client, anthropic_client, deepseek_client
+
+    except Exception as e:
+        raise Exception(f"Failed to initialize API clients: {str(e)}")
+
+
+def gpt4_mini_backend(system_msg, user_prompt, temperature):
+    """Call GPT-4o Mini API."""
+    openai_client, _, _ = init_clients()
+    try:
+        r = openai_client.chat.completions.create(
+            model="gpt-4o-mini",
+            messages=[
+                {"role": "system", "content": system_msg},
+                {"role": "user", "content": user_prompt}
+            ],
+            temperature=temperature
+        )
+        return r.choices[0].message.content, r.usage.total_tokens
+    except Exception as e:
+        raise Exception(f"GPT-4o-mini error: {str(e)}")
+
+
+def anthropic_backend(system_msg, user_prompt, temperature):
+    """Call Anthropic Claude API."""
+    _, anthropic_client, _ = init_clients()
+    try:
+        r = anthropic_client.messages.create(
+            model="claude-3-5-sonnet-20241022",
+            system=system_msg,
+            messages=[{"role": "user", "content": user_prompt}],
+            max_tokens=2000,
+            temperature=temperature
+        )
+        text = r.content[0].text.strip()
+        toks = r.usage.input_tokens + r.usage.output_tokens
+        return text, toks
+    except Exception as e:
+        raise Exception(f"Anthropic error: {str(e)}")
+
+
+def deepseek_backend(system_msg, user_prompt, temperature):
+    """Call DeepSeek API."""
+    _, _, deepseek_client = init_clients()
+    try:
+        r = deepseek_client.chat.completions.create(
+            model="deepseek-chat",
+            messages=[
+                {"role": "system", "content": system_msg},
+                {"role": "user", "content": user_prompt}
+            ],
+            temperature=temperature
+        )
+        return r.choices[0].message.content, r.usage.total_tokens
+    except Exception as e:
+        raise Exception(f"DeepSeek error: {str(e)}")
+
+
+# ✅ Register backends
+BACKENDS = {
+    "GPT-4o Mini": gpt4_mini_backend,
+    "Claude 3.5 Sonnet": anthropic_backend,
+    "DeepSeek Chat": deepseek_backend
+}

src/conversation.py

@@ -0,0 +1,20 @@
+from src.api_clients import init_clients
+
+def structure_conversation(raw_text):
+    _, anthropic_client, _ = init_clients()
+    formatter_prompt = (
+        "Convert this dialogue into a turn-by-turn transcript where each line "
+        "starts with 'HUMAN:' or 'AI:'. Do not add any other commentary.\n\n"
+        + raw_text
+    )
+    try:
+        resp = anthropic_client.messages.create(
+            model="claude-3-5-sonnet-20241022",
+            system="You are a conversation formatter.",
+            messages=[{"role": "user", "content": formatter_prompt}],
+            max_tokens=1000,
+            temperature=0.0
+        )
+        return resp.content[0].text.strip()
+    except Exception as e:
+        raise Exception(f"Error in structuring conversation: {str(e)}")

src/evaluation.py

@@ -0,0 +1,82 @@
+import re
+import json
+import datetime
+import pandas as pd
+from src.api_clients import BACKENDS
+
+def split_json_objects(s):
+    objs, depth, start = [], 0, None
+    for i, ch in enumerate(s):
+        if ch == "{":
+            if depth == 0:
+                start = i
+            depth += 1
+        elif ch == "}":
+            depth -= 1
+            if depth == 0 and start is not None:
+                objs.append(s[start:i+1])
+    return objs
+
+def evaluate_with_judges(conversation, selected_models, variant, *weights_and_temp, prompt_template):
+    weights, temperature = list(weights_and_temp[:-1]), weights_and_temp[-1]
+    if not conversation.strip():
+        raise ValueError("Conversation input is empty.")
+    from src.conversation import structure_conversation
+    structured = structure_conversation(conversation)
+    system_msg = (
+        "You are Judge-Care-Lock, a rigorous evaluator of AI-therapist dialogues.\n"
+        "1. Use ONLY the transcript—quote it for every decision.\n"
+        "2. Apply the multi-layer rubric exactly; do NOT invent scales.\n"
+        "3. Return valid JSON matching the schema; no extra text."
+    )
+    user_prompt = prompt_template.replace("{CONVERSATION}", structured)
+
+    metrics_rows = []
+    comments_map = {}
+    tokens_map = {}
+    pros_map = {}
+    cons_map = {}
+    summary_map = {}
+
+    for model_name in selected_models:
+        fn = BACKENDS[model_name]
+        raw, toks = fn(system_msg, user_prompt, temperature)
+        tokens_map[model_name] = toks
+
+        clean = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw, flags=re.MULTILINE).strip()
+        objs = split_json_objects(clean)
+        if not objs:
+            raise ValueError(f"No valid JSON from {model_name}:\n{clean}")
+        try:
+            parsed = json.loads(objs[0])
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON from {model_name}: {str(e)}")
+
+        row = {"Model": model_name}
+        total_score = 0.0
+        for idx, (m, data) in enumerate(parsed["metrics"].items()):
+            score = data.get("score", 0.0)
+            row[m] = score
+            total_score += score * weights[idx]
+        row["Total"] = round(total_score, 2)
+        metrics_rows.append(row)
+        comments_map[model_name] = parsed
+        pros_map[model_name] = parsed.get("positive", [])
+        cons_map[model_name] = parsed.get("negative", [])
+        summary_map[model_name] = parsed.get("summary", "")
+
+    ts = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+    filename = f"/tmp/carelock_eval_{ts}.json"
+    combined = {
+        "metrics_table": metrics_rows,
+        "parsed_per_model": comments_map,
+        "tokens_per_model": tokens_map,
+        "pros_per_model": pros_map,
+        "cons_per_model": cons_map,
+        "summary_per_model": summary_map
+    }
+    with open(filename, "w", encoding="utf-8") as f:
+        json.dump(combined, f, indent=2)
+
+    return (pd.DataFrame(metrics_rows), comments_map, tokens_map,
+            pros_map, cons_map, summary_map, filename)