""" Unit tests for server/grader.py Tests every reward component individually plus the range clamp. Run with: pytest tests/test_grader.py -v """ import importlib.util import sys import os import pytest import pandas as pd # Import grader directly (avoids server/__init__.py → openenv-core chain) _ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) _GRADER = os.path.join(_ROOT, "server", "grader.py") spec = importlib.util.spec_from_file_location("grader", _GRADER) _mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(_mod) REWARD_MAX = _mod.REWARD_MAX REWARD_MIN = _mod.REWARD_MIN compute_accuracy_reward = _mod.compute_accuracy_reward compute_efficiency_reward = _mod.compute_efficiency_reward compute_preservation_reward = _mod.compute_preservation_reward compute_process_reward = _mod.compute_process_reward compute_step_reward = _mod.compute_step_reward compute_total_reward = _mod.compute_total_reward compute_lightweight_score = _mod.compute_lightweight_score # ── Accuracy reward ─────────────────────────────────────────────────────────── class TestAccuracyReward: def test_improvement_positive(self): r = compute_accuracy_reward(0.70, 0.62, 0.62, 0.80) assert r > 0, f"Improvement should give positive reward, got {r}" def test_regression_negative(self): r = compute_accuracy_reward(0.60, 0.70, 0.62, 0.80) assert r < 0, f"Regression should give negative reward, got {r}" def test_no_change_zero(self): r = compute_accuracy_reward(0.65, 0.65, 0.62, 0.80) assert r == 0.0 def test_submit_success_bonus(self): r = compute_accuracy_reward(0.80, 0.75, 0.62, 0.80, is_submit=True) assert r > 0.5, f"Submit success should add bonus, got {r}" def test_submit_fail_partial_credit(self): """With strict grader: failing to hit target gives negative reward. baseline=0.62, target=0.80, current=0.71 → 50% of the way there. New grader: penalty = -0.40 * (1-0.5) = -0.20, minus regression = net negative. This is intentional — agents that fail to hit target are penalised. """ # baseline=0.62, target=0.80, current=0.71, previous=0.70 r = compute_accuracy_reward(0.71, 0.70, 0.62, 0.80, is_submit=True) # Reward has an improvement component (+0.025) but the failure penalty (-0.20) dominates # Net should be negative (strict grader — failing to hit target is punished) assert r < 0.30, f"Partial fail at submit: expected < 0.30, got {r}" # But not as bad as a complete failure (current == baseline) r_zero = compute_accuracy_reward(0.62, 0.62, 0.62, 0.80, is_submit=True) assert r > r_zero, f"Partial progress should be better than no progress: {r} vs {r_zero}" # ── Preservation reward ─────────────────────────────────────────────────────── class TestPreservationReward: def test_above_90_bonus(self): r = compute_preservation_reward(97, 100) assert r > 0 def test_below_90_zero_or_neg(self): r = compute_preservation_reward(85, 100) assert r <= 0.02 # at best neutral at 85% def test_below_50_catastrophic(self): r = compute_preservation_reward(40, 100) assert r <= -0.40, f"Expected catastrophic penalty, got {r}" def test_full_preservation(self): r = compute_preservation_reward(100, 100) assert r == 0.05 # ── Process reward ──────────────────────────────────────────────────────────── class TestProcessReward: def test_query_after_inspect_rewarded(self): history = ["inspect_dataset"] r = compute_process_reward(history, "query_cleaner") assert r > 0 def test_apply_without_query_penalized(self): history = ["inspect_dataset", "inspect_model"] r = compute_process_reward(history, "apply 1") assert r < 0 def test_apply_after_query_rewarded(self): history = ["inspect_dataset", "query_cleaner"] r = compute_process_reward(history, "apply 1") assert r > 0 def test_submit_without_validate_penalized(self): history = ["inspect_dataset", "query_cleaner", "apply 1"] r = compute_process_reward(history, "submit") assert r < 0 # ── Step reward ─────────────────────────────────────────────────────────────── class TestStepReward: def test_quality_improvement_positive(self): r = compute_step_reward("apply 1", quality_before=0.5, quality_after=0.7, rows_preserved_after=0.97) assert r > 0 def test_quality_degradation_negative(self): r = compute_step_reward("apply 1", quality_before=0.7, quality_after=0.4, rows_preserved_after=0.97) assert r < 0 def test_non_apply_zero(self): r = compute_step_reward("validate", quality_before=0.5, quality_after=0.7, rows_preserved_after=0.97) assert r == 0.0 def test_low_preservation_penalty(self): r = compute_step_reward("apply 1", quality_before=0.5, quality_after=0.6, rows_preserved_after=0.75) # Row preservation penalty should reduce reward r_without = compute_step_reward("apply 1", quality_before=0.5, quality_after=0.6, rows_preserved_after=0.97) assert r < r_without # ── Total reward range ──────────────────────────────────────────────────────── class TestRewardRange: def test_within_declared_range(self): """All reward combinations must stay within [-1.0, 1.0].""" test_cases = [ (0.5, 0.1, 0.05, 0.2, 0.1), (-0.8, -0.1, -0.4, -0.05, -0.2), (1.0, 0.2, 0.05, 0.2, 0.15), # might need clamping (-1.5, -0.2, -0.4, -0.05, -0.3), # definitely needs clamping ] for acc, proc, pres, eff, step in test_cases: r = compute_total_reward(acc, proc, pres, eff, step) assert REWARD_MIN <= r <= REWARD_MAX, ( f"Reward {r} out of [{REWARD_MIN}, {REWARD_MAX}] " f"for inputs acc={acc} proc={proc} pres={pres}" ) def test_clamping_applied(self): """Extreme inputs should be clamped, not crash.""" r = compute_total_reward(10.0, 5.0, 5.0) assert r == REWARD_MAX r = compute_total_reward(-10.0, -5.0, -5.0) assert r == REWARD_MIN # ── Lightweight quality score ───────────────────────────────────────────────── class TestLightweightScore: def _make_df(self, n_rows=10, n_missing=0, n_dups=0): """Create a minimal test dataframe.""" df = pd.DataFrame({ "feature_0": [float(i) for i in range(n_rows)], "target": [i % 2 for i in range(n_rows)], }) if n_missing: df.loc[:n_missing - 1, "feature_0"] = float("nan") if n_dups: df = pd.concat([df, df.iloc[:n_dups]], ignore_index=True) return df def test_clean_df_high_score(self): df = self._make_df() score = compute_lightweight_score(df, df.copy(), len(df), {"feature_0": {"expected_dtype": "float64"}}) assert score >= 0.80 def test_many_missing_low_score(self): df = self._make_df(n_missing=8) score = compute_lightweight_score(df, df.copy(), 10, {"feature_0": {"expected_dtype": "float64"}}, initial_missing=8) assert score < 0.70 def test_score_in_range(self): df = self._make_df(n_missing=3, n_dups=2) score = compute_lightweight_score(df, df.copy(), 10, {"feature_0": {"expected_dtype": "float64"}}, initial_missing=3) assert 0.0 <= score <= 1.0