""" Test the actual HTTP response flow as the OpenEnv validator would see it. This simulates calling /step endpoint and checking JSON response. """ import json import sys from fastapi.testclient import TestClient from server.app import app client = TestClient(app) def test_http_flow(): """Test complete HTTP flow for reward responses.""" print("="*80) print("HTTP RESPONSE FLOW TEST") print("="*80) failures = [] # 1. Reset print("\n1. Calling /reset...") resp_reset = client.post("/reset") if resp_reset.status_code != 200: print(f" [FAIL] Reset failed: {resp_reset.status_code}") print(f" Response: {resp_reset.text}") return print(f" [OK] Reset successful") #2. Step with fixed config for task1 print("\n2. Calling /step with fixed config for task1_json...") step_payload = { "fixed_config": '{"name": "Alice", "age": 30, "email": "alice@example.com"}' } resp_step = client.post("/step", json=step_payload) if resp_step.status_code != 200: print(f" [FAIL] Step failed: {resp_step.status_code}") print(f" Response: {resp_step.text}") return print(f" [OK] Step returned 200") try: response_json = resp_step.json() print(f" JSON parsed successfully") print(f" Response keys: {list(response_json.keys())}") except json.JSONDecodeError as e: print(f" [FAIL] Failed to parse JSON: {e}") print(f" Raw response: {resp_step.text}") failures.append("JSON parse failed") return # 3. Check reward field if "reward" not in response_json: print(f" [FAIL] No 'reward' field in response") print(f" Response: {response_json}") failures.append("Missing reward field") return reward = response_json["reward"] print(f"\n3. Reward value in response: {reward}") print(f" Type: {type(reward).__name__}") print(f" Repr: {repr(reward)}") # 4. Validate reward try: reward_float = float(reward) print(f" Converted to float: {reward_float}") except (TypeError, ValueError) as e: print(f" [FAIL] Cannot convert to float: {e}") failures.append(f"Reward not numeric: {reward}") return # 5. Check bounds if reward_float == float('inf') or reward_float == float('-inf'): print(f" [FAIL] Reward is infinite") failures.append("Reward is infinite") elif reward_float != reward_float: # NaN print(f" [FAIL] Reward is NaN") failures.append("Reward is NaN") elif not (0.0 < reward_float < 1.0): print(f" [FAIL] Reward {reward_float} out of bounds (0, 1)") failures.append(f"Out of bounds: {reward_float}") else: print(f" [OK] Reward {reward_float} is in valid range (0, 1)") # 6. Check JSON serialization round-trip print(f"\n4. JSON serialization check...") json_str = json.dumps(response_json) print(f" JSON string: {json_str[:100]}...") reparsed = json.loads(json_str) reparsed_reward = reparsed.get("reward") print(f" After round-trip: {reparsed_reward}") print(f" Type: {type(reparsed_reward).__name__}") if reparsed_reward != reward: print(f" [WARN] Reward changed during round-trip: {reward} -> {reparsed_reward}") # Summary if failures: print(f"\n{'='*80}") print(f"[FAIL] Found {len(failures)} issues:") for f in failures: print(f" - {f}") return False else: print(f"\n[OK] All HTTP response checks passed") return True def test_all_tasks(): """Test /step with perfect configs for all 7 tasks.""" print("\n" + "="*80) print("TESTING ALL 7 TASKS VIA /step ENDPOINT") print("="*80) test_configs = { "task1_json": '{"name": "Alice", "age": 30, "email": "alice@example.com"}', "task2_yaml": "name: Alice\nage: 30\nemail: alice@example.com", "task3_dockerfile": "FROM python:3.9-slim\nRUN apt-get update\nEXPOSE 3000\nCMD [\"python\", \"app.py\"]", "task4_compose": "services:\n web:\n image: app\n ports:\n - '8080:3000'\n db:\n image: postgres", "task5_k8s": "apiVersion: v1\nkind: Pod\nmetadata:\n name: app-pod\nspec:\n containers:\n - name: app\n image: app:latest", "task6_github_actions": "on: push\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout\n - run: echo test", "task7_nginx": "server {\n listen 80;\n server_name example.com;\n location / {\n proxy_pass http://localhost:3000;\n }\n}", } all_passed = True # Reset once client.post("/reset") for task_id, config in test_configs.items(): print(f"\nTask: {task_id}") resp = client.post("/step", json={"fixed_config": config}) if resp.status_code != 200: print(f" [FAIL] Status {resp.status_code}: {resp.text[:100]}") all_passed = False continue try: data = resp.json() reward = data.get("reward") if reward is None: print(f" [FAIL] No reward in response") all_passed = False else: reward_float = float(reward) if 0.0 < reward_float < 1.0: print(f" [OK] Reward: {reward}") else: print(f" [FAIL] Out of bounds: {reward_float}") all_passed = False except Exception as e: print(f" [FAIL] Exception: {e}") all_passed = False return all_passed def main(): try: # Test single flow passed = test_http_flow() if passed is not False: # Test all tasks all_passed = test_all_tasks() print("\n" + "="*80) print("HTTP FLOW TEST COMPLETE") print("="*80) except Exception as e: print(f"\n[EXCEPTION] {type(e).__name__}: {e}") import traceback traceback.print_exc() sys.exit(1) if __name__ == "__main__": main()