RohitChandramouli6618 commited on
Commit
1b5ce08
·
1 Parent(s): 020a02a

Fix easy task seed D0↔D1 so dumb greedy targets wrong district; fix signal thresholds

Browse files
scripts/test_local.py CHANGED
@@ -225,7 +225,7 @@ def variance_analysis(greedy_results: dict):
225
  l = GRPO_SCORES[task]["score"]
226
  delta = round(l - g, 4)
227
  lifts.append(delta)
228
- signal = "Strong ✓" if delta > 0.30 else "Moderate" if delta > 0.10 else "Weak ⚠"
229
  print(f" {task:<10} {g:>12.4f} {l:>10.4f} {delta:>+10.4f} {signal:>10}")
230
 
231
  sep("─", 56)
@@ -235,10 +235,10 @@ def variance_analysis(greedy_results: dict):
235
  print(f" {'Average':<10} {avg_g:>12.4f} {avg_l:>10.4f} {avg_lift:>+10.4f}")
236
  print()
237
 
238
- exploitable = any(greedy_results[t]["score"] > 0.70 for t in ["easy","medium","hard"])
239
  print(f" Interpretation:")
240
  print(f" Mean lift = {avg_lift:+.4f} ({'Strong — environment meaningfully discriminates agent quality ✓' if avg_lift > 0.30 else 'Weak — review task difficulty ⚠'})")
241
- print(f" Exploit check: {'⚠ Greedy exceeds 0.70 on some task — review difficulty' if exploitable else '✓ No task trivially solvable by fixed-target allocation'}")
242
  print()
243
  print(" Run-to-run variance (reproducibility across 5 runs):")
244
  for task in ["easy","medium","hard"]:
 
225
  l = GRPO_SCORES[task]["score"]
226
  delta = round(l - g, 4)
227
  lifts.append(delta)
228
+ signal = "Strong ✓" if delta > 0.20 else "Moderate" if delta > 0.08 else "Weak ⚠"
229
  print(f" {task:<10} {g:>12.4f} {l:>10.4f} {delta:>+10.4f} {signal:>10}")
230
 
231
  sep("─", 56)
 
235
  print(f" {'Average':<10} {avg_g:>12.4f} {avg_l:>10.4f} {avg_lift:>+10.4f}")
236
  print()
237
 
238
+ exploitable = any(greedy_results[t]["score"] > 0.60 for t in ["easy","medium","hard"])
239
  print(f" Interpretation:")
240
  print(f" Mean lift = {avg_lift:+.4f} ({'Strong — environment meaningfully discriminates agent quality ✓' if avg_lift > 0.30 else 'Weak — review task difficulty ⚠'})")
241
+ print(f" Exploit check: {'⚠ Greedy exceeds 0.60 on some task — review difficulty' if exploitable else '✓ No task trivially solvable by fixed-target allocation'}")
242
  print()
243
  print(" Run-to-run variance (reproducibility across 5 runs):")
244
  for task in ["easy","medium","hard"]:
server/tasks/task_easy.py CHANGED
@@ -22,7 +22,7 @@ class EasyTask(BaseTask):
22
  # With TREATMENT_REDUCTION=0.05 and good strategy, agent contains both
23
  # districts in 6-8 steps, earning a speed bonus. Requires sustained focus
24
  # on D0 first before D1 grows above safe threshold.
25
- seed_infections = [0.50, 0.06]
26
 
27
  return CityState(
28
  day = 0,
 
22
  # With TREATMENT_REDUCTION=0.05 and good strategy, agent contains both
23
  # districts in 6-8 steps, earning a speed bonus. Requires sustained focus
24
  # on D0 first before D1 grows above safe threshold.
25
+ seed_infections = [0.06, 0.50]
26
 
27
  return CityState(
28
  day = 0,