Commit ·
1b5ce08
1
Parent(s): 020a02a
Fix easy task seed D0↔D1 so dumb greedy targets wrong district; fix signal thresholds
Browse files- scripts/test_local.py +3 -3
- server/tasks/task_easy.py +1 -1
scripts/test_local.py
CHANGED
|
@@ -225,7 +225,7 @@ def variance_analysis(greedy_results: dict):
|
|
| 225 |
l = GRPO_SCORES[task]["score"]
|
| 226 |
delta = round(l - g, 4)
|
| 227 |
lifts.append(delta)
|
| 228 |
-
signal = "Strong ✓" if delta > 0.
|
| 229 |
print(f" {task:<10} {g:>12.4f} {l:>10.4f} {delta:>+10.4f} {signal:>10}")
|
| 230 |
|
| 231 |
sep("─", 56)
|
|
@@ -235,10 +235,10 @@ def variance_analysis(greedy_results: dict):
|
|
| 235 |
print(f" {'Average':<10} {avg_g:>12.4f} {avg_l:>10.4f} {avg_lift:>+10.4f}")
|
| 236 |
print()
|
| 237 |
|
| 238 |
-
exploitable = any(greedy_results[t]["score"] > 0.
|
| 239 |
print(f" Interpretation:")
|
| 240 |
print(f" Mean lift = {avg_lift:+.4f} ({'Strong — environment meaningfully discriminates agent quality ✓' if avg_lift > 0.30 else 'Weak — review task difficulty ⚠'})")
|
| 241 |
-
print(f" Exploit check: {'⚠ Greedy exceeds 0.
|
| 242 |
print()
|
| 243 |
print(" Run-to-run variance (reproducibility across 5 runs):")
|
| 244 |
for task in ["easy","medium","hard"]:
|
|
|
|
| 225 |
l = GRPO_SCORES[task]["score"]
|
| 226 |
delta = round(l - g, 4)
|
| 227 |
lifts.append(delta)
|
| 228 |
+
signal = "Strong ✓" if delta > 0.20 else "Moderate" if delta > 0.08 else "Weak ⚠"
|
| 229 |
print(f" {task:<10} {g:>12.4f} {l:>10.4f} {delta:>+10.4f} {signal:>10}")
|
| 230 |
|
| 231 |
sep("─", 56)
|
|
|
|
| 235 |
print(f" {'Average':<10} {avg_g:>12.4f} {avg_l:>10.4f} {avg_lift:>+10.4f}")
|
| 236 |
print()
|
| 237 |
|
| 238 |
+
exploitable = any(greedy_results[t]["score"] > 0.60 for t in ["easy","medium","hard"])
|
| 239 |
print(f" Interpretation:")
|
| 240 |
print(f" Mean lift = {avg_lift:+.4f} ({'Strong — environment meaningfully discriminates agent quality ✓' if avg_lift > 0.30 else 'Weak — review task difficulty ⚠'})")
|
| 241 |
+
print(f" Exploit check: {'⚠ Greedy exceeds 0.60 on some task — review difficulty' if exploitable else '✓ No task trivially solvable by fixed-target allocation'}")
|
| 242 |
print()
|
| 243 |
print(" Run-to-run variance (reproducibility across 5 runs):")
|
| 244 |
for task in ["easy","medium","hard"]:
|
server/tasks/task_easy.py
CHANGED
|
@@ -22,7 +22,7 @@ class EasyTask(BaseTask):
|
|
| 22 |
# With TREATMENT_REDUCTION=0.05 and good strategy, agent contains both
|
| 23 |
# districts in 6-8 steps, earning a speed bonus. Requires sustained focus
|
| 24 |
# on D0 first before D1 grows above safe threshold.
|
| 25 |
-
seed_infections = [0.
|
| 26 |
|
| 27 |
return CityState(
|
| 28 |
day = 0,
|
|
|
|
| 22 |
# With TREATMENT_REDUCTION=0.05 and good strategy, agent contains both
|
| 23 |
# districts in 6-8 steps, earning a speed bonus. Requires sustained focus
|
| 24 |
# on D0 first before D1 grows above safe threshold.
|
| 25 |
+
seed_infections = [0.06, 0.50]
|
| 26 |
|
| 27 |
return CityState(
|
| 28 |
day = 0,
|