""" Task 1: Prompt Sensitivity Detection — stateless design for HF Spaces The scenario is embedded in the observation so step() works across replicas. """ from __future__ import annotations import random, sys, os, json sys.path.insert(0, os.path.dirname(__file__)) from typing import Optional, List from openenv.core import Environment, Action, Observation, State from grader import grade_sensitivity SCENARIOS = [ {"id":0,"base_prompt":"Summarize the benefits of exercise.","variants":["Summarize the benefits of exercise.","List the benefits of exercise.","What are the advantages of working out?"],"ai_responses":["Exercise improves cardiovascular health, boosts mood, and increases energy.","1. Better heart health 2. Improved mood 3. More energy","Working out can help you lose weight and feel happier."],"ground_truth":"sensitive","sensitive_index":2,"gt_explanation":"Third variant shifts focus to weight loss."}, {"id":1,"base_prompt":"What is the capital of France?","variants":["What is the capital of France?","Tell me the capital city of France.","Name the capital of France."],"ai_responses":["Paris","The capital of France is Paris.","Paris is the capital of France."],"ground_truth":"stable","sensitive_index":-1,"gt_explanation":"All variants say Paris."}, {"id":2,"base_prompt":"Explain recursion in programming.","variants":["Explain recursion in programming.","What is recursion in coding?","Describe recursion for a 5-year-old."],"ai_responses":["Recursion is when a function calls itself to solve a smaller version of the same problem.","Recursion: a function that calls itself until a base case is met.","Imagine a box inside a box inside a box, that is recursion!"],"ground_truth":"sensitive","sensitive_index":2,"gt_explanation":"Third variant uses childlike analogy not technical definition."}, {"id":3,"base_prompt":"How do I sort a list in Python?","variants":["How do I sort a list in Python?","What is the way to sort a list in Python?","Give me Python code to sort a list."],"ai_responses":["Use list.sort() for in-place sorting or sorted(list) to return a new sorted list.","You can sort a Python list using list.sort() or sorted().","my_list=[3,1,2]; my_list.sort(); print(my_list)"],"ground_truth":"sensitive","sensitive_index":2,"gt_explanation":"Third variant gives code only with no explanation."}, {"id":4,"base_prompt":"Who wrote Hamlet?","variants":["Who wrote Hamlet?","Who is the author of Hamlet?","Which playwright wrote Hamlet?"],"ai_responses":["William Shakespeare wrote Hamlet.","The author of Hamlet is William Shakespeare.","Hamlet was written by William Shakespeare."],"ground_truth":"stable","sensitive_index":-1,"gt_explanation":"All variants credit Shakespeare."}, {"id":5,"base_prompt":"What is machine learning?","variants":["What is machine learning?","Define machine learning.","Explain machine learning like I am a CEO."],"ai_responses":["Machine learning is a subset of AI where systems learn patterns from data.","Machine learning: algorithms that improve through experience.","ML lets software get smarter over time by analyzing data, giving you a competitive edge."],"ground_truth":"sensitive","sensitive_index":2,"gt_explanation":"Third variant produces a business-framing response."}, {"id":6,"base_prompt":"What is the boiling point of water?","variants":["What is the boiling point of water?","At what temperature does water boil?","When does water start boiling?"],"ai_responses":["Water boils at 100C (212F) at standard atmospheric pressure.","Water boils at 100 Celsius or 212 Fahrenheit.","Water starts boiling at 100C or 212F under normal conditions."],"ground_truth":"stable","sensitive_index":-1,"gt_explanation":"All variants agree on 100C / 212F."}, {"id":7,"base_prompt":"How do I reverse a string in Python?","variants":["How do I reverse a string in Python?","What is the Python way to reverse a string?","Reverse a string in Python, give me a one-liner."],"ai_responses":["You can reverse a string using slicing: my_string[::-1] or reversed().","In Python, string reversal is done with slicing: s[::-1].","s[::-1]"],"ground_truth":"sensitive","sensitive_index":2,"gt_explanation":"Third variant returns only a one-liner with zero explanation."}, ] SCENARIO_MAP = {s["id"]: s for s in SCENARIOS} class SensitivityAction(Action): verdict: str confidence: float explanation: str scenario_id: int = 0 # echoed back from observation sensitive_variant_index: Optional[int] = None class SensitivityObservation(Observation): task: str level: str episode_id: int scenario_id: int # key for stateless step() base_prompt: str variants: List[str] ai_responses: List[str] instruction: str class PromptSensitivityEnvironment(Environment): def __init__(self): self._episode_id = 0 self._done = False self._step_count = 0 def reset(self, **kwargs) -> SensitivityObservation: scenario = random.choice(SCENARIOS) self._episode_id += 1 self._done = False self._step_count = 0 self._current_scenario_id = scenario["id"] return self._obs(scenario) def step(self, action: SensitivityAction, **kwargs): if self._done: scenario = SCENARIO_MAP.get(action.scenario_id, SCENARIOS[0]) return self._obs(scenario), 0.0, True, {"error": "Episode done. Call reset()."} # Stateless: look up scenario from action.scenario_id scenario = SCENARIO_MAP.get(action.scenario_id, SCENARIO_MAP.get(self._current_scenario_id, SCENARIOS[0])) self._step_count += 1 reward, info = grade_sensitivity(action, scenario) self._done = True info.update({ "episode_id": self._episode_id, "step": self._step_count, "ground_truth": scenario["ground_truth"], "ground_truth_sensitive_index": scenario["sensitive_index"], "ground_truth_explanation": scenario["gt_explanation"], }) return self._obs(scenario), reward, self._done, info def state(self) -> State: return State() def _obs(self, scenario) -> SensitivityObservation: return SensitivityObservation( task="prompt_sensitivity_detection", level="easy", episode_id=self._episode_id, scenario_id=scenario["id"], base_prompt=scenario["base_prompt"], variants=scenario["variants"], ai_responses=scenario["ai_responses"], instruction=( "Analyze the AI responses to the prompt variants. " "Determine if the output is 'sensitive' (changes significantly), " "'stable' (consistent), or 'partial' (minor differences). " "If sensitive, set sensitive_variant_index to the variant index that caused inconsistency. " "Include scenario_id from this observation unchanged in your action." ), )