""" Multi-Agent RL Simulation for Market-Aware Grid Topology Optimization Proof-of-Concept Implementation """ import numpy as np import pandas as pd import matplotlib.pyplot as plt import seaborn as sns from dataclasses import dataclass from typing import List, Dict, Tuple, Optional import gym from gym import spaces import torch import torch.nn as nn import torch.optim as optim from collections import deque import random import json from datetime import datetime, timedelta # Set random seeds for reproducibility np.random.seed(42) torch.manual_seed(42) @dataclass class GridState: """Represents current grid state""" bus_voltages: np.ndarray line_flows: np.ndarray line_limits: np.ndarray topology_matrix: np.ndarray congestion_status: np.ndarray timestamp: datetime @dataclass class MarketSignals: """Represents market trading signals""" p2p_trades: List[Dict] price_signals: np.ndarray surplus_areas: List[int] deficit_areas: List[int] trading_volume: float @dataclass class AgentAction: """Represents agent action""" action_type: str # 'topology_change', 'market_response', 'coordinate' action_value: np.ndarray confidence: float rationale: Dict class GridEnvironment(gym.Env): """Custom grid environment for multi-agent RL""" def __init__(self, n_buses=30, n_lines=41): super().__init__() self.n_buses = n_buses self.n_lines = n_lines # Initialize IEEE 30-bus system parameters self._initialize_grid_parameters() # Action and observation spaces self.action_space = spaces.MultiDiscrete([ n_lines + 1, # Topology actions (0=no action, 1-n=line switching) 3, # Market response (0=hold, 1=buy, 2=sell) 5 # Coordination level (0-4 priority) ]) self.observation_space = spaces.Box( low=-10, high=10, shape=(n_buses * 3 + n_lines * 2 + 10,), # Grid + market + temporal dtype=np.float32 ) self.current_step = 0 self.max_steps = 1000 def _initialize_grid_parameters(self): """Initialize IEEE 30-bus system parameters""" # Simplified IEEE 30-bus system self.bus_data = np.random.rand(self.n_buses, 6) # [Pd, Qd, Pg, Qg, Vmin, Vmax] self.line_data = np.random.rand(self.n_lines, 4) # [from, to, r, x] # Initialize topology (all lines initially connected) self.topology = np.ones(self.n_lines, dtype=int) # Line limits (MW) self.line_limits = np.random.uniform(50, 200, self.n_lines) # Base case power flow self.base_case = self._calculate_power_flow() def _calculate_power_flow(self): """Simplified DC power flow calculation""" # Build admittance matrix Ybus = np.zeros((self.n_buses, self.n_buses), dtype=complex) for i in range(self.n_lines): if self.topology[i] == 1: # Line is connected from_bus = int(self.line_data[i, 0]) to_bus = int(self.line_data[i, 1]) reactance = self.line_data[i, 3] # Add line admittance admittance = -1j / reactance Ybus[from_bus, to_bus] += admittance Ybus[to_bus, from_bus] += admittance Ybus[from_bus, from_bus] -= admittance Ybus[to_bus, to_bus] -= admittance # Simplified power flow (DC approximation) P_inj = self.bus_data[:, 2] - self.bus_data[:, 0] # Generation - Load # Solve for voltage angles try: # Remove slack bus (bus 0) Y_reduced = Ybus[1:, 1:] P_reduced = P_inj[1:] theta = np.linalg.solve(Y_reduced.imag, P_reduced) theta_full = np.concatenate([[0], theta]) # Calculate line flows line_flows = np.zeros(self.n_lines) for i in range(self.n_lines): if self.topology[i] == 1: from_bus = int(self.line_data[i, 0]) to_bus = int(self.line_data[i, 1]) reactance = self.line_data[i, 3] line_flows[i] = (theta_full[from_bus] - theta_full[to_bus]) / reactance return line_flows except np.linalg.LinAlgError: return np.zeros(self.n_lines) def reset(self): """Reset environment to initial state""" self.current_step = 0 self.topology = np.ones(self.n_lines, dtype=int) # Generate random market conditions self.market_signals = self._generate_market_signals() # Calculate initial grid state line_flows = self._calculate_power_flow() self.grid_state = GridState( bus_voltages=np.ones(self.n_buses), line_flows=line_flows, line_limits=self.line_limits, topology_matrix=self.topology.copy(), congestion_status=self._check_congestion(line_flows), timestamp=datetime.now() ) return self._get_observation() def _generate_market_signals(self): """Generate realistic market signals""" # P2P trading simulation n_trades = np.random.randint(10, 50) p2p_trades = [] for _ in range(n_trades): trade = { 'from_bus': np.random.randint(0, self.n_buses), 'to_bus': np.random.randint(0, self.n_buses), 'amount': np.random.uniform(1, 20), # MW 'price': np.random.uniform(20, 80), # $/MWh 'timestamp': datetime.now() } p2p_trades.append(trade) # Identify surplus/deficit areas net_injection = np.random.normal(0, 10, self.n_buses) surplus_areas = np.where(net_injection > 5)[0].tolist() deficit_areas = np.where(net_injection < -5)[0].tolist() # Price signals (LMP-like) - add time-based variation base_price = 50.0 time_factor = np.sin(self.current_step * 0.1) * 20 # Time-varying prices price_signals = np.random.uniform(20, 100, self.n_buses) + time_factor return MarketSignals( p2p_trades=p2p_trades, price_signals=price_signals, surplus_areas=surplus_areas, deficit_areas=deficit_areas, trading_volume=np.sum([t['amount'] for t in p2p_trades]) ) def _check_congestion(self, line_flows): """Check for line congestion""" congestion = np.zeros(self.n_lines) for i in range(self.n_lines): if abs(line_flows[i]) > 0.9 * self.line_limits[i]: congestion[i] = 1 return congestion def _get_observation(self): """Combine grid state and market signals into observation""" # Grid state features grid_features = np.concatenate([ self.grid_state.bus_voltages, # 30 features self.grid_state.line_flows / self.grid_state.line_limits, # 41 features self.grid_state.topology_matrix # 41 features ]) # Market features market_features = np.concatenate([ self.market_signals.price_signals / 100.0, # 30 features [len(self.market_signals.surplus_areas) / self.n_buses], # 1 feature [len(self.market_signals.deficit_areas) / self.n_buses], # 1 feature [self.market_signals.trading_volume / 1000.0], # 1 feature [self.current_step / self.max_steps], # 1 feature [np.sum(self.grid_state.congestion_status) / self.n_lines] # 1 feature ]) obs = np.concatenate([grid_features, market_features]).astype(np.float32) print(f"Observation shape: {obs.shape}") # Debug print return obs def step(self, actions): """Execute one time step""" self.current_step += 1 # Unpack actions from multiple agents topology_action = actions[0] market_action = actions[1] coordination_action = actions[2] # Execute topology action if 1 <= topology_action < self.n_lines + 1: line_to_switch = topology_action - 1 if 0 <= line_to_switch < self.n_lines: self.topology[line_to_switch] = 1 - self.topology[line_to_switch] # Recalculate power flow new_line_flows = self._calculate_power_flow() # Update grid state self.grid_state = GridState( bus_voltages=np.ones(self.n_buses), # Simplified line_flows=new_line_flows, line_limits=self.line_limits, topology_matrix=self.topology.copy(), congestion_status=self._check_congestion(new_line_flows), timestamp=datetime.now() ) # Calculate reward reward = self._calculate_reward(topology_action, market_action, coordination_action) # Check if episode is done done = self.current_step >= self.max_steps # Generate new market signals self.market_signals = self._generate_market_signals() return self._get_observation(), reward, done, {} def _calculate_reward(self, topology_action, market_action, coordination_action): """Calculate multi-objective reward""" # Congestion relief reward congestion_before = np.sum(self.grid_state.congestion_status) # Simulate next state for congestion comparison temp_topology = self.topology.copy() if 1 <= topology_action < self.n_lines: line_to_switch = topology_action - 1 temp_topology[line_to_switch] = 1 - temp_topology[line_to_switch] # Temporarily apply topology to check congestion old_topology = self.topology.copy() self.topology = temp_topology future_flows = self._calculate_power_flow() future_congestion = self._check_congestion(future_flows) self.topology = old_topology congestion_after = np.sum(future_congestion) congestion_reward = 10 * (congestion_before - congestion_after) # Market efficiency reward surplus_deficit_match = 0 for surplus_bus in self.market_signals.surplus_areas: for deficit_bus in self.market_signals.deficit_areas: # Check if topology action improves connectivity if self._improves_connectivity(surplus_bus, deficit_bus, topology_action): surplus_deficit_match += 1 market_reward = 5 * surplus_deficit_match # Coordination reward (penalize conflicting actions) coordination_reward = -2 * coordination_action if coordination_action > 2 else 0 # Switching cost penalty switching_penalty = -0.5 if topology_action > 0 else 0 total_reward = congestion_reward + market_reward + coordination_reward + switching_penalty return total_reward def _improves_connectivity(self, bus1, bus2, topology_action): """Check if topology action improves connectivity between two buses""" # Simplified connectivity check if topology_action == 0: return False line_to_switch = topology_action - 1 # Boundary check if line_to_switch < 0 or line_to_switch >= self.n_lines: return False from_bus = int(self.line_data[line_to_switch, 0]) to_bus = int(self.line_data[line_to_switch, 1]) # Check if this line helps connect the buses return (from_bus == bus1 and to_bus == bus2) or (from_bus == bus2 and to_bus == bus1) class MultiAgentNetwork(nn.Module): """Neural network for multi-agent decision making""" def __init__(self, input_dim, hidden_dim=256, n_agents=3): super().__init__() self.input_dim = input_dim self.hidden_dim = hidden_dim self.n_agents = n_agents # Shared feature extraction self.feature_extractor = nn.Sequential( nn.Linear(input_dim, hidden_dim), nn.ReLU(), nn.Linear(hidden_dim, hidden_dim), nn.ReLU(), nn.Linear(hidden_dim, hidden_dim // 2), nn.ReLU() ) # Agent-specific heads self.agent_heads = nn.ModuleList([ nn.Sequential( nn.Linear(hidden_dim // 2, hidden_dim // 4), nn.ReLU(), nn.Linear(hidden_dim // 4, 50) # Topology actions ), nn.Sequential( nn.Linear(hidden_dim // 2, hidden_dim // 4), nn.ReLU(), nn.Linear(hidden_dim // 4, 3) # Market actions ), nn.Sequential( nn.Linear(hidden_dim // 2, hidden_dim // 4), nn.ReLU(), nn.Linear(hidden_dim // 4, 5) # Coordination actions ) ]) # Value head for critic self.value_head = nn.Sequential( nn.Linear(hidden_dim // 2, hidden_dim // 4), nn.ReLU(), nn.Linear(hidden_dim // 4, 1) ) def forward(self, state): features = self.feature_extractor(state) # Agent actions actions = [] for head in self.agent_heads: action_logits = head(features) actions.append(action_logits) # Value estimate value = self.value_head(features) return actions, value class PPOAgent: """Proximal Policy Optimization agent""" def __init__(self, input_dim, lr=3e-4): self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") self.network = MultiAgentNetwork(input_dim).to(self.device) self.optimizer = optim.Adam(self.network.parameters(), lr=lr) # PPO hyperparameters self.clip_ratio = 0.2 self.entropy_coef = 0.01 self.value_coef = 0.5 self.gamma = 0.99 self.gae_lambda = 0.95 # Experience buffer self.buffer = [] def select_action(self, state): """Select actions using current policy""" state_tensor = torch.FloatTensor(state).unsqueeze(0).to(self.device) print(f"State tensor shape: {state_tensor.shape}") # Debug print print(f"Network input_dim: {self.network.input_dim}") # Debug print with torch.no_grad(): action_logits, value = self.network(state_tensor) actions = [] action_log_probs = [] for i, logits in enumerate(action_logits): if i == 0: # Topology agent action_probs = torch.softmax(logits, dim=-1) dist = torch.distributions.Categorical(action_probs) action = dist.sample() actions.append(action.item()) action_log_probs.append(dist.log_prob(action)) elif i == 1: # Market agent action_probs = torch.softmax(logits, dim=-1) dist = torch.distributions.Categorical(action_probs) action = dist.sample() actions.append(action.item()) action_log_probs.append(dist.log_prob(action)) else: # Coordination agent action_probs = torch.softmax(logits, dim=-1) dist = torch.distributions.Categorical(action_probs) action = dist.sample() actions.append(action.item()) action_log_probs.append(dist.log_prob(action)) return actions, torch.stack(action_log_probs), value.squeeze() def store_experience(self, state, actions, log_probs, value, reward, next_state, done): """Store experience in buffer""" self.buffer.append({ 'state': state, 'actions': actions, 'log_probs': log_probs, 'value': value, 'reward': reward, 'next_state': next_state, 'done': done }) def update(self, n_epochs=10, batch_size=64): """Update policy using PPO""" if len(self.buffer) < batch_size: return # Convert buffer to tensors states = torch.FloatTensor([exp['state'] for exp in self.buffer]).to(self.device) actions = torch.LongTensor([exp['actions'] for exp in self.buffer]).to(self.device) old_log_probs = torch.stack([exp['log_probs'] for exp in self.buffer]).to(self.device) old_values = torch.FloatTensor([exp['value'].item() for exp in self.buffer]).to(self.device) rewards = torch.FloatTensor([exp['reward'] for exp in self.buffer]).to(self.device) dones = torch.FloatTensor([exp['done'] for exp in self.buffer]).to(self.device) # Calculate advantages advantages = self._calculate_advantages(rewards, old_values, dones) returns = advantages + old_values # Normalize advantages advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8) # PPO update for epoch in range(n_epochs): # Random shuffle indices = torch.randperm(len(self.buffer)) for start in range(0, len(self.buffer), batch_size): end = start + batch_size batch_indices = indices[start:end] batch_states = states[batch_indices] batch_actions = actions[batch_indices] batch_old_log_probs = old_log_probs[batch_indices] batch_old_values = old_values[batch_indices] batch_advantages = advantages[batch_indices] batch_returns = returns[batch_indices] # Forward pass new_action_logits, new_values = self.network(batch_states) # Calculate loss for each agent total_policy_loss = 0 total_entropy_loss = 0 for i in range(3): # 3 agents if i == 0: # Topology agent new_action_probs = torch.softmax(new_action_logits[i], dim=-1) batch_agent_actions = batch_actions[:, i] new_log_probs = torch.distributions.Categorical(new_action_probs).log_prob(batch_agent_actions) old_log_probs_agent = batch_old_log_probs[:, i] elif i == 1: # Market agent new_action_probs = torch.softmax(new_action_logits[i], dim=-1) batch_agent_actions = batch_actions[:, i] new_log_probs = torch.distributions.Categorical(new_action_probs).log_prob(batch_agent_actions) old_log_probs_agent = batch_old_log_probs[:, i] else: # Coordination agent new_action_probs = torch.softmax(new_action_logits[i], dim=-1) batch_agent_actions = batch_actions[:, i] new_log_probs = torch.distributions.Categorical(new_action_probs).log_prob(batch_agent_actions) old_log_probs_agent = batch_old_log_probs[:, i] # PPO ratio ratio = torch.exp(new_log_probs - old_log_probs_agent) # Clipped surrogate objective surr1 = ratio * batch_advantages surr2 = torch.clamp(ratio, 1 - self.clip_ratio, 1 + self.clip_ratio) * batch_advantages policy_loss = -torch.min(surr1, surr2).mean() total_policy_loss += policy_loss # Entropy bonus entropy = -torch.sum(new_action_probs * torch.log(new_action_probs + 1e-8), dim=-1).mean() total_entropy_loss += -entropy # Value loss value_loss = F.mse_loss(new_values.squeeze(), batch_returns) # Total loss loss = total_policy_loss + self.value_coef * value_loss + self.entropy_coef * total_entropy_loss # Backward pass self.optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(self.network.parameters(), 0.5) self.optimizer.step() # Clear buffer self.buffer.clear() def _calculate_advantages(self, rewards, values, dones): """Calculate Generalized Advantage Estimation""" advantages = torch.zeros_like(rewards) last_advantage = 0 for t in reversed(range(len(rewards))): if t == len(rewards) - 1: next_value = 0 else: next_value = values[t + 1] delta = rewards[t] + self.gamma * next_value * (1 - dones[t]) - values[t] advantages[t] = delta + self.gamma * self.gae_lambda * (1 - dones[t]) * last_advantage last_advantage = advantages[t] return advantages def train_simulation(): """Main training loop""" # Initialize environment and agent env = GridEnvironment() input_dim = env.observation_space.shape[0] agent = PPOAgent(input_dim) # Training parameters n_episodes = 1000 max_steps_per_episode = 500 update_frequency = 10 # Tracking metrics episode_rewards = [] congestion_levels = [] switching_actions = [] print("Starting Multi-Agent RL Training for Market-Aware Grid Topology Optimization") print("=" * 80) for episode in range(n_episodes): state = env.reset() episode_reward = 0 episode_congestion = [] episode_switches = [] for step in range(max_steps_per_episode): # Select actions actions, log_probs, value = agent.select_action(state) # Take step next_state, reward, done, _ = env.step(actions) # Store experience agent.store_experience(state, actions, log_probs, value, reward, next_state, done) # Track metrics episode_reward += reward episode_congestion.append(np.sum(env.grid_state.congestion_status)) episode_switches.append(1 if actions[0] > 0 else 0) state = next_state if done: break # Update agent if episode % update_frequency == 0: agent.update() # Record episode metrics episode_rewards.append(episode_reward) congestion_levels.append(np.mean(episode_congestion)) switching_actions.append(np.sum(episode_switches)) # Print progress if episode % 50 == 0: avg_reward = np.mean(episode_rewards[-50:]) avg_congestion = np.mean(congestion_levels[-50:]) avg_switches = np.mean(switching_actions[-50:]) print(f"Episode {episode:4d} | Avg Reward: {avg_reward:8.2f} | " f"Avg Congestion: {avg_congestion:5.2f} | Avg Switches: {avg_switches:5.1f}") print("\nTraining completed!") # Plot results plot_training_results(episode_rewards, congestion_levels, switching_actions) return agent, env, episode_rewards, congestion_levels, switching_actions def plot_training_results(rewards, congestion, switches): """Plot training metrics""" fig, axes = plt.subplots(3, 1, figsize=(12, 10)) # Smooth the curves window = 50 rewards_smooth = np.convolve(rewards, np.ones(window)/window, mode='valid') congestion_smooth = np.convolve(congestion, np.ones(window)/window, mode='valid') switches_smooth = np.convolve(switches, np.ones(window)/window, mode='valid') # Episode rewards axes[0].plot(rewards_smooth) axes[0].set_title('Episode Rewards (Smoothed)') axes[0].set_ylabel('Reward') axes[0].grid(True) # Congestion levels axes[1].plot(congestion_smooth) axes[1].set_title('Average Congestion Levels (Smoothed)') axes[1].set_ylabel('Congested Lines') axes[1].grid(True) # Switching actions axes[2].plot(switches_smooth) axes[2].set_title('Topology Switching Frequency (Smoothed)') axes[2].set_ylabel('Switches per Episode') axes[2].set_xlabel('Episode') axes[2].grid(True) plt.tight_layout() plt.savefig('training_results.png', dpi=300, bbox_inches='tight') plt.show() def evaluate_agent(agent, env, n_episodes=100): """Evaluate trained agent""" print("\nEvaluating trained agent...") total_rewards = [] total_congestion = [] total_switches = [] for episode in range(n_episodes): state = env.reset() episode_reward = 0 episode_congestion = [] episode_switches = [] for step in range(500): # Fixed evaluation steps actions, _, _ = agent.select_action(state) next_state, reward, done, _ = env.step(actions) episode_reward += reward episode_congestion.append(np.sum(env.grid_state.congestion_status)) episode_switches.append(1 if actions[0] > 0 else 0) state = next_state if done: break total_rewards.append(episode_reward) total_congestion.append(np.mean(episode_congestion)) total_switches.append(np.sum(episode_switches)) print(f"Evaluation Results ({n_episodes} episodes):") print(f" Average Reward: {np.mean(total_rewards):.2f} ± {np.std(total_rewards):.2f}") print(f" Average Congestion: {np.mean(total_congestion):.2f} ± {np.std(total_congestion):.2f}") print(f" Average Switches: {np.mean(total_switches):.1f} ± {np.std(total_switches):.1f}") return total_rewards, total_congestion, total_switches def save_results(agent, rewards, congestion, switches): """Save training results and model""" timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") # Save model torch.save(agent.network.state_dict(), f'multi_agent_model_{timestamp}.pth') # Save training data results = { 'episode_rewards': rewards, 'congestion_levels': congestion, 'switching_actions': switches, 'training_timestamp': timestamp } with open(f'training_results_{timestamp}.json', 'w') as f: json.dump(results, f, indent=2) print(f"\nResults saved with timestamp: {timestamp}") if __name__ == "__main__": # Import PyTorch functional interface import torch.nn.functional as F # Train the multi-agent system agent, env, rewards, congestion, switches = train_simulation() # Evaluate the trained agent eval_rewards, eval_congestion, eval_switches = evaluate_agent(agent, env) # Save results save_results(agent, rewards, congestion, switches) print("\n" + "=" * 80) print("Multi-Agent RL Simulation Completed Successfully!") print("=" * 80) print("\nKey Achievements:") print("✅ Implemented market-aware topology optimization") print("✅ Multi-agent coordination (topology + market + coordination)") print("✅ Congestion management through proactive switching") print("✅ Integration of P2P trading signals") print("✅ Reward function balancing reliability and economics") print("\nNext Steps:") print("📋 Integrate with blockchain audit framework") print("📋 Add zero-knowledge proofs for privacy") print("📋 Scale to larger grid models") print("📋 Implement real-time market data feeds")