""" inference.py — Baseline inference script Uses OpenAI client with API_BASE_URL / MODEL_NAME / HF_TOKEN env vars. Must run in <20 min on 2 vCPU / 8GB. """ import os, sys, json sys.path.insert(0, "server") # so grader/environment are importable import requests from openai import OpenAI from server.environment import SensitivityAction API_BASE_URL = os.environ.get("API_BASE_URL", "https://api.openai.com/v1") MODEL_NAME = os.environ.get("MODEL_NAME", "gpt-4o-mini") HF_TOKEN = os.environ.get("HF_TOKEN", "") ENV_URL = os.environ.get("ENV_URL", "http://localhost:7860") client = OpenAI(api_key=HF_TOKEN, base_url=API_BASE_URL) def env_reset(): r = requests.post(f"{ENV_URL}/reset", timeout=30) r.raise_for_status() return r.json() def env_step(action: dict): r = requests.post(f"{ENV_URL}/step", json={"action": action}, timeout=30) r.raise_for_status() return r.json() def build_prompt(obs: dict) -> str: variants = "\n".join(f" [{i}] \"{v}\"" for i, v in enumerate(obs["variants"])) responses = "\n".join(f" Response [{i}]: \"{r}\"" for i, r in enumerate(obs["ai_responses"])) return f"""You are an expert AI evaluator specializing in prompt sensitivity analysis. Base Prompt: "{obs['base_prompt']}" Prompt Variants: {variants} AI Responses to each variant: {responses} Task: {obs['instruction']} Respond ONLY with a valid JSON object — no markdown, no extra text: {{ "verdict": "", "confidence": , "explanation": "", "sensitive_variant_index": }}""" def call_agent(obs: dict) -> dict: raw = client.chat.completions.create( model=MODEL_NAME, messages=[{"role": "user", "content": build_prompt(obs)}], temperature=0.0, ).choices[0].message.content.strip() if raw.startswith("```"): raw = raw.split("```")[1] if raw.startswith("json"): raw = raw[4:] return json.loads(raw.strip()) def run(num_episodes: int = 5): print(f"\n{'='*58}") print(f" NeuroHack — Task 1: Prompt Sensitivity Baseline") print(f" Model : {MODEL_NAME}") print(f" Env : {ENV_URL}") print(f"{'='*58}\n") scores = [] for ep in range(1, num_episodes + 1): print(f"── Episode {ep}/{num_episodes} ──────────────────────────") data = env_reset() obs = data["observation"] print(f" Base prompt : {obs['base_prompt']}") try: action_dict = call_agent(obs) except Exception as e: print(f" [AGENT ERROR] {e} — using fallback") action_dict = { "verdict": "stable", "confidence": 0.5, "explanation": "Agent error — fallback to stable verdict.", "sensitive_variant_index": None, } print(f" Verdict : {action_dict.get('verdict')}") print(f" Confidence : {action_dict.get('confidence')}") print(f" Explanation : {str(action_dict.get('explanation',''))[:80]}") result = env_step(action_dict) reward = result["reward"] info = result["info"] scores.append(reward) print(f" Reward : {reward}") print(f" GT verdict : {info.get('ground_truth')}") print(f" Breakdown : {info.get('breakdown')}\n") avg = round(sum(scores) / len(scores), 4) print(f"{'='*58}") print(f" Episodes : {num_episodes}") print(f" Scores : {scores}") print(f" Average : {avg}") print(f"{'='*58}\n") return scores, avg if __name__ == "__main__": run(num_episodes=5)