"""Verification judges: Claude / OpenAI (vision) and DeepSeek (text-only), sharing the specificity-scaled prompt from the paper benchmark.""" import base64 import io import json import time MODELS = { "Claude Sonnet 4.6": dict(provider="anthropic", model="claude-sonnet-4-6", vision=True, price=(3.0, 15.0), key_url="https://console.anthropic.com/settings/keys", key_label="Anthropic API Key"), "Claude Haiku 4.5": dict(provider="anthropic", model="claude-haiku-4-5-20251001", vision=True, price=(1.0, 5.0), key_url="https://console.anthropic.com/settings/keys", key_label="Anthropic API Key"), "OpenAI GPT-4o": dict(provider="openai", model="gpt-4o", vision=True, price=(2.5, 10.0), key_url="https://platform.openai.com/api-keys", key_label="OpenAI API Key"), "DeepSeek (text-only)": dict(provider="deepseek", model="deepseek-chat", vision=False, price=(0.27, 1.1), key_url="https://platform.deepseek.com/api_keys", key_label="DeepSeek API Key"), } PROMPT = """A researcher is searching for a published scientific figure with \ this query: "{query}" {materials_line} Caption: {caption} Judge the candidate against what the query actually specifies. If the query \ is broad or vague, any figure genuinely showing what it describes counts as \ a match; do not demand details the query never mentions. If the query \ specifies particulars (quantities, axes, plot type, what is compared), the \ figure must show them. Science match is the dominant factor: a figure on \ the wrong scientific topic should never score above 0.4.{sketch_line} Respond JSON only: {{"match": true|false, "confidence": 0.0-1.0, \ "what_is_plotted": "", "reason": ""}}""" M_VISION = "Below is the candidate figure image together with its caption." M_TEXT = "Only the figure's caption is available (no image)." M_SKETCH = ("\nThe researcher also provided a rough sketch of the figure " "they remember (first image); weigh structural similarity to it.") def _jpeg_b64(path_or_bytes, max_dim=1024): from PIL import Image if isinstance(path_or_bytes, (bytes, bytearray)): im = Image.open(io.BytesIO(path_or_bytes)) else: im = Image.open(path_or_bytes) im = im.convert("RGB") im.thumbnail((max_dim, max_dim)) buf = io.BytesIO() im.save(buf, format="JPEG", quality=80) return base64.standard_b64encode(buf.getvalue()).decode() def _parse(text): obj = json.loads(text[text.index("{"):text.rindex("}") + 1]) return {"match": bool(obj.get("match")), "confidence": float(obj.get("confidence", 0.0)), "what_is_plotted": str(obj.get("what_is_plotted", ""))[:300], "reason": str(obj.get("reason", ""))[:300]} class Judge: def __init__(self, model_label, api_key): cfg = MODELS[model_label] self.cfg = cfg self.calls = 0 if cfg["provider"] == "anthropic": from anthropic import Anthropic self.client = Anthropic(api_key=api_key) else: from openai import OpenAI base = "https://api.deepseek.com" \ if cfg["provider"] == "deepseek" else None self.client = OpenAI(api_key=api_key, base_url=base) def est_cost(self): i, o = self.cfg["price"] per = (1300 if self.cfg["vision"] else 500) / 1e6 * i + 80 / 1e6 * o return self.calls * per def judge(self, query, caption, image_path=None, sketch_bytes=None, retries=4): vision = self.cfg["vision"] and image_path is not None prompt = PROMPT.format( query=query, materials_line=M_VISION if vision else M_TEXT, caption=(caption or "")[:1500], sketch_line=M_SKETCH if (sketch_bytes and vision) else "") for attempt in range(retries): try: self.calls += 1 if self.cfg["provider"] == "anthropic": content = [] if sketch_bytes and vision: content.append({"type": "image", "source": { "type": "base64", "media_type": "image/jpeg", "data": _jpeg_b64(sketch_bytes)}}) if vision: content.append({"type": "image", "source": { "type": "base64", "media_type": "image/jpeg", "data": _jpeg_b64(image_path)}}) content.append({"type": "text", "text": prompt}) resp = self.client.messages.create( model=self.cfg["model"], max_tokens=300, messages=[{"role": "user", "content": content}]) return _parse(resp.content[0].text) else: if vision: content = [] if sketch_bytes: content.append({"type": "image_url", "image_url": { "url": "data:image/jpeg;base64," + _jpeg_b64(sketch_bytes)}}) content.append({"type": "image_url", "image_url": { "url": "data:image/jpeg;base64," + _jpeg_b64(image_path)}}) content.append({"type": "text", "text": prompt}) else: content = prompt resp = self.client.chat.completions.create( model=self.cfg["model"], max_tokens=300, messages=[{"role": "user", "content": content}]) return _parse(resp.choices[0].message.content) except Exception: if attempt == retries - 1: return None time.sleep(4 * (attempt + 1))