Spaces:
Sleeping
Sleeping
File size: 3,062 Bytes
9cfcf69 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | import matplotlib.pyplot as plt
import numpy as np
from core.revenue import revenue_factor
from training.reward_fn import (
PROPOSE_BASE_REWARD,
PROPOSE_FACTOR_WEIGHT,
ILLEGAL_ACTION_REWARD,
)
def simulate_rewards():
# Setup standard sector thresholds
demand = 100.0
critical = 40.0
surplus = 150.0
wastage = 250.0 # Engine uses demand * 2.5
allocations = np.linspace(0, 300, 500)
rewards = []
revenue_factors = []
for x in allocations:
rf = revenue_factor(x, critical, demand, surplus, wastage)
if rf is None:
# Critical failure
reward = ILLEGAL_ACTION_REWARD
rf_val = 0.0
else:
# Match training/reward_fn.py logic
reward = PROPOSE_BASE_REWARD + PROPOSE_FACTOR_WEIGHT * (rf - 1.0)
rf_val = rf
rewards.append(reward)
revenue_factors.append(rf_val)
# Plotting
fig, ax1 = plt.subplots(figsize=(12, 7))
color = 'tab:blue'
ax1.set_xlabel('Allocation Amount (Demand = 100)')
ax1.set_ylabel('Revenue Factor (Efficiency)', color=color)
ax1.plot(allocations, revenue_factors, color=color, linewidth=2, label='Revenue Factor')
ax1.tick_params(axis='y', labelcolor=color)
ax1.grid(True, alpha=0.3)
ax2 = ax1.twinx()
color = 'tab:red'
ax2.set_ylabel('GRPO Reward (Training Signal)', color=color)
ax2.plot(allocations, rewards, color=color, linewidth=3, linestyle='--', label='RL Reward')
ax2.tick_params(axis='y', labelcolor=color)
# Annotate key points
ax1.axvline(x=critical, color='black', linestyle=':', label='Critical Threshold')
ax1.axvline(x=demand, color='green', linestyle=':', label='Demand')
ax1.axvline(x=surplus, color='purple', linestyle=':', label='Surplus (Peak)')
plt.title('Reward Landscape: How the RL Agent sees the Economy')
fig.tight_layout()
output_path = "assets/results/reward_landscape.png"
plt.savefig(output_path)
print(f"Graph saved to {output_path}")
# Benchmark Summary
print("\n=== REWARD BENCHMARK SUMMARY ===")
print(f"{'Approach':<20} | {'Average Reward':<15}")
print("-" * 40)
# 1. Random Approach (0 to 250)
random_rewards = [r for r in rewards if r > ILLEGAL_ACTION_REWARD]
print(f"{'Random Agent':<20} | {np.mean(random_rewards):.4f}")
# 2. Under-allocator (fails critical)
print(f"{'Panic Under-alloc':<20} | {ILLEGAL_ACTION_REWARD:.4f}")
# 3. Demand-Only Agent
rf_demand = revenue_factor(100, critical, demand, surplus, wastage)
reward_demand = PROPOSE_BASE_REWARD + PROPOSE_FACTOR_WEIGHT * (rf_demand - 1.0)
print(f"{'Safe Agent (Demand)':<20} | {reward_demand:.4f}")
# 4. Optimal Agent (Surplus)
rf_surplus = revenue_factor(150, critical, demand, surplus, wastage)
reward_surplus = PROPOSE_BASE_REWARD + PROPOSE_FACTOR_WEIGHT * (rf_surplus - 1.0)
print(f"{'Optimal Agent':<20} | {reward_surplus:.4f}")
if __name__ == "__main__":
simulate_rewards()
|