multi_agent_rl_simulation.py

"""
Multi-Agent RL Simulation for Market-Aware Grid Topology Optimization
Proof-of-Concept Implementation
"""

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from dataclasses import dataclass
from typing import List, Dict, Tuple, Optional
import gym
from gym import spaces
import torch
import torch.nn as nn
import torch.optim as optim
from collections import deque
import random
import json
from datetime import datetime, timedelta

# Set random seeds for reproducibility
np.random.seed(42)
torch.manual_seed(42)

@dataclass
class GridState:
    """Represents current grid state"""
    bus_voltages: np.ndarray
    line_flows: np.ndarray
    line_limits: np.ndarray
    topology_matrix: np.ndarray
    congestion_status: np.ndarray
    timestamp: datetime

@dataclass
class MarketSignals:
    """Represents market trading signals"""
    p2p_trades: List[Dict]
    price_signals: np.ndarray
    surplus_areas: List[int]
    deficit_areas: List[int]
    trading_volume: float

@dataclass
class AgentAction:
    """Represents agent action"""
    action_type: str  # 'topology_change', 'market_response', 'coordinate'
    action_value: np.ndarray
    confidence: float
    rationale: Dict

class GridEnvironment(gym.Env):
    """Custom grid environment for multi-agent RL"""
    
    def __init__(self, n_buses=30, n_lines=41):
        super().__init__()
        
        self.n_buses = n_buses
        self.n_lines = n_lines
        
        # Initialize IEEE 30-bus system parameters
        self._initialize_grid_parameters()
        
        # Action and observation spaces
        self.action_space = spaces.MultiDiscrete([
            n_lines + 1,  # Topology actions (0=no action, 1-n=line switching)
            3,           # Market response (0=hold, 1=buy, 2=sell)
            5            # Coordination level (0-4 priority)
        ])
        
        self.observation_space = spaces.Box(
            low=-10, high=10,
            shape=(n_buses * 3 + n_lines * 2 + 10,),  # Grid + market + temporal
            dtype=np.float32
        )
        
        self.current_step = 0
        self.max_steps = 1000
        
    def _initialize_grid_parameters(self):
        """Initialize IEEE 30-bus system parameters"""
        # Simplified IEEE 30-bus system
        self.bus_data = np.random.rand(self.n_buses, 6)  # [Pd, Qd, Pg, Qg, Vmin, Vmax]
        self.line_data = np.random.rand(self.n_lines, 4)   # [from, to, r, x]
        
        # Initialize topology (all lines initially connected)
        self.topology = np.ones(self.n_lines, dtype=int)
        
        # Line limits (MW)
        self.line_limits = np.random.uniform(50, 200, self.n_lines)
        
        # Base case power flow
        self.base_case = self._calculate_power_flow()
        
    def _calculate_power_flow(self):
        """Simplified DC power flow calculation"""
        # Build admittance matrix
        Ybus = np.zeros((self.n_buses, self.n_buses), dtype=complex)
        
        for i in range(self.n_lines):
            if self.topology[i] == 1:  # Line is connected
                from_bus = int(self.line_data[i, 0])
                to_bus = int(self.line_data[i, 1])
                reactance = self.line_data[i, 3]
                
                # Add line admittance
                admittance = -1j / reactance
                Ybus[from_bus, to_bus] += admittance
                Ybus[to_bus, from_bus] += admittance
                Ybus[from_bus, from_bus] -= admittance
                Ybus[to_bus, to_bus] -= admittance
        
        # Simplified power flow (DC approximation)
        P_inj = self.bus_data[:, 2] - self.bus_data[:, 0]  # Generation - Load
        
        # Solve for voltage angles
        try:
            # Remove slack bus (bus 0)
            Y_reduced = Ybus[1:, 1:]
            P_reduced = P_inj[1:]
            
            theta = np.linalg.solve(Y_reduced.imag, P_reduced)
            theta_full = np.concatenate([[0], theta])
            
            # Calculate line flows
            line_flows = np.zeros(self.n_lines)
            for i in range(self.n_lines):
                if self.topology[i] == 1:
                    from_bus = int(self.line_data[i, 0])
                    to_bus = int(self.line_data[i, 1])
                    reactance = self.line_data[i, 3]
                    line_flows[i] = (theta_full[from_bus] - theta_full[to_bus]) / reactance
            
            return line_flows
            
        except np.linalg.LinAlgError:
            return np.zeros(self.n_lines)
    
    def reset(self):
        """Reset environment to initial state"""
        self.current_step = 0
        self.topology = np.ones(self.n_lines, dtype=int)
        
        # Generate random market conditions
        self.market_signals = self._generate_market_signals()
        
        # Calculate initial grid state
        line_flows = self._calculate_power_flow()
        
        self.grid_state = GridState(
            bus_voltages=np.ones(self.n_buses),
            line_flows=line_flows,
            line_limits=self.line_limits,
            topology_matrix=self.topology.copy(),
            congestion_status=self._check_congestion(line_flows),
            timestamp=datetime.now()
        )
        
        return self._get_observation()
    
    def _generate_market_signals(self):
        """Generate realistic market signals"""
        # P2P trading simulation
        n_trades = np.random.randint(10, 50)
        p2p_trades = []
        
        for _ in range(n_trades):
            trade = {
                'from_bus': np.random.randint(0, self.n_buses),
                'to_bus': np.random.randint(0, self.n_buses),
                'amount': np.random.uniform(1, 20),  # MW
                'price': np.random.uniform(20, 80),  # $/MWh
                'timestamp': datetime.now()
            }
            p2p_trades.append(trade)
        
        # Identify surplus/deficit areas
        net_injection = np.random.normal(0, 10, self.n_buses)
        surplus_areas = np.where(net_injection > 5)[0].tolist()
        deficit_areas = np.where(net_injection < -5)[0].tolist()
        
        # Price signals (LMP-like) - add time-based variation
        base_price = 50.0
        time_factor = np.sin(self.current_step * 0.1) * 20  # Time-varying prices
        price_signals = np.random.uniform(20, 100, self.n_buses) + time_factor
        
        return MarketSignals(
            p2p_trades=p2p_trades,
            price_signals=price_signals,
            surplus_areas=surplus_areas,
            deficit_areas=deficit_areas,
            trading_volume=np.sum([t['amount'] for t in p2p_trades])
        )
    
    def _check_congestion(self, line_flows):
        """Check for line congestion"""
        congestion = np.zeros(self.n_lines)
        for i in range(self.n_lines):
            if abs(line_flows[i]) > 0.9 * self.line_limits[i]:
                congestion[i] = 1
        return congestion
    
    def _get_observation(self):
        """Combine grid state and market signals into observation"""
        # Grid state features
        grid_features = np.concatenate([
            self.grid_state.bus_voltages,  # 30 features
            self.grid_state.line_flows / self.grid_state.line_limits,  # 41 features
            self.grid_state.topology_matrix  # 41 features
        ])
        
        # Market features
        market_features = np.concatenate([
            self.market_signals.price_signals / 100.0,  # 30 features
            [len(self.market_signals.surplus_areas) / self.n_buses],  # 1 feature
            [len(self.market_signals.deficit_areas) / self.n_buses],  # 1 feature
            [self.market_signals.trading_volume / 1000.0],  # 1 feature
            [self.current_step / self.max_steps],  # 1 feature
            [np.sum(self.grid_state.congestion_status) / self.n_lines]  # 1 feature
        ])
        
        obs = np.concatenate([grid_features, market_features]).astype(np.float32)
        print(f"Observation shape: {obs.shape}")  # Debug print
        return obs
    
    def step(self, actions):
        """Execute one time step"""
        self.current_step += 1
        
        # Unpack actions from multiple agents
        topology_action = actions[0]
        market_action = actions[1] 
        coordination_action = actions[2]
        
        # Execute topology action
        if 1 <= topology_action < self.n_lines + 1:
            line_to_switch = topology_action - 1
            if 0 <= line_to_switch < self.n_lines:
                self.topology[line_to_switch] = 1 - self.topology[line_to_switch]
        
        # Recalculate power flow
        new_line_flows = self._calculate_power_flow()
        
        # Update grid state
        self.grid_state = GridState(
            bus_voltages=np.ones(self.n_buses),  # Simplified
            line_flows=new_line_flows,
            line_limits=self.line_limits,
            topology_matrix=self.topology.copy(),
            congestion_status=self._check_congestion(new_line_flows),
            timestamp=datetime.now()
        )
        
        # Calculate reward
        reward = self._calculate_reward(topology_action, market_action, coordination_action)
        
        # Check if episode is done
        done = self.current_step >= self.max_steps
        
        # Generate new market signals
        self.market_signals = self._generate_market_signals()
        
        return self._get_observation(), reward, done, {}
    
    def _calculate_reward(self, topology_action, market_action, coordination_action):
        """Calculate multi-objective reward"""
        # Congestion relief reward
        congestion_before = np.sum(self.grid_state.congestion_status)
        
        # Simulate next state for congestion comparison
        temp_topology = self.topology.copy()
        if 1 <= topology_action < self.n_lines:
            line_to_switch = topology_action - 1
            temp_topology[line_to_switch] = 1 - temp_topology[line_to_switch]
        
        # Temporarily apply topology to check congestion
        old_topology = self.topology.copy()
        self.topology = temp_topology
        future_flows = self._calculate_power_flow()
        future_congestion = self._check_congestion(future_flows)
        self.topology = old_topology
        
        congestion_after = np.sum(future_congestion)
        congestion_reward = 10 * (congestion_before - congestion_after)
        
        # Market efficiency reward
        surplus_deficit_match = 0
        for surplus_bus in self.market_signals.surplus_areas:
            for deficit_bus in self.market_signals.deficit_areas:
                # Check if topology action improves connectivity
                if self._improves_connectivity(surplus_bus, deficit_bus, topology_action):
                    surplus_deficit_match += 1
        
        market_reward = 5 * surplus_deficit_match
        
        # Coordination reward (penalize conflicting actions)
        coordination_reward = -2 * coordination_action if coordination_action > 2 else 0
        
        # Switching cost penalty
        switching_penalty = -0.5 if topology_action > 0 else 0
        
        total_reward = congestion_reward + market_reward + coordination_reward + switching_penalty
        
        return total_reward
    
    def _improves_connectivity(self, bus1, bus2, topology_action):
        """Check if topology action improves connectivity between two buses"""
        # Simplified connectivity check
        if topology_action == 0:
            return False
        
        line_to_switch = topology_action - 1
        
        # Boundary check
        if line_to_switch < 0 or line_to_switch >= self.n_lines:
            return False
        
        from_bus = int(self.line_data[line_to_switch, 0])
        to_bus = int(self.line_data[line_to_switch, 1])
        
        # Check if this line helps connect the buses
        return (from_bus == bus1 and to_bus == bus2) or (from_bus == bus2 and to_bus == bus1)

class MultiAgentNetwork(nn.Module):
    """Neural network for multi-agent decision making"""
    
    def __init__(self, input_dim, hidden_dim=256, n_agents=3):
        super().__init__()
        
        self.input_dim = input_dim
        self.hidden_dim = hidden_dim
        self.n_agents = n_agents
        
        # Shared feature extraction
        self.feature_extractor = nn.Sequential(
            nn.Linear(input_dim, hidden_dim),
            nn.ReLU(),
            nn.Linear(hidden_dim, hidden_dim),
            nn.ReLU(),
            nn.Linear(hidden_dim, hidden_dim // 2),
            nn.ReLU()
        )
        
        # Agent-specific heads
        self.agent_heads = nn.ModuleList([
            nn.Sequential(
                nn.Linear(hidden_dim // 2, hidden_dim // 4),
                nn.ReLU(),
                nn.Linear(hidden_dim // 4, 50)  # Topology actions
            ),
            nn.Sequential(
                nn.Linear(hidden_dim // 2, hidden_dim // 4),
                nn.ReLU(),
                nn.Linear(hidden_dim // 4, 3)    # Market actions
            ),
            nn.Sequential(
                nn.Linear(hidden_dim // 2, hidden_dim // 4),
                nn.ReLU(),
                nn.Linear(hidden_dim // 4, 5)    # Coordination actions
            )
        ])
        
        # Value head for critic
        self.value_head = nn.Sequential(
            nn.Linear(hidden_dim // 2, hidden_dim // 4),
            nn.ReLU(),
            nn.Linear(hidden_dim // 4, 1)
        )
    
    def forward(self, state):
        features = self.feature_extractor(state)
        
        # Agent actions
        actions = []
        for head in self.agent_heads:
            action_logits = head(features)
            actions.append(action_logits)
        
        # Value estimate
        value = self.value_head(features)
        
        return actions, value

class PPOAgent:
    """Proximal Policy Optimization agent"""
    
    def __init__(self, input_dim, lr=3e-4):
        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        
        self.network = MultiAgentNetwork(input_dim).to(self.device)
        self.optimizer = optim.Adam(self.network.parameters(), lr=lr)
        
        # PPO hyperparameters
        self.clip_ratio = 0.2
        self.entropy_coef = 0.01
        self.value_coef = 0.5
        self.gamma = 0.99
        self.gae_lambda = 0.95
        
        # Experience buffer
        self.buffer = []
        
    def select_action(self, state):
        """Select actions using current policy"""
        state_tensor = torch.FloatTensor(state).unsqueeze(0).to(self.device)
        print(f"State tensor shape: {state_tensor.shape}")  # Debug print
        print(f"Network input_dim: {self.network.input_dim}")  # Debug print
        
        with torch.no_grad():
            action_logits, value = self.network(state_tensor)
        
        actions = []
        action_log_probs = []
        
        for i, logits in enumerate(action_logits):
            if i == 0:  # Topology agent
                action_probs = torch.softmax(logits, dim=-1)
                dist = torch.distributions.Categorical(action_probs)
                action = dist.sample()
                actions.append(action.item())
                action_log_probs.append(dist.log_prob(action))
            elif i == 1:  # Market agent
                action_probs = torch.softmax(logits, dim=-1)
                dist = torch.distributions.Categorical(action_probs)
                action = dist.sample()
                actions.append(action.item())
                action_log_probs.append(dist.log_prob(action))
            else:  # Coordination agent
                action_probs = torch.softmax(logits, dim=-1)
                dist = torch.distributions.Categorical(action_probs)
                action = dist.sample()
                actions.append(action.item())
                action_log_probs.append(dist.log_prob(action))
        
        return actions, torch.stack(action_log_probs), value.squeeze()
    
    def store_experience(self, state, actions, log_probs, value, reward, next_state, done):
        """Store experience in buffer"""
        self.buffer.append({
            'state': state,
            'actions': actions,
            'log_probs': log_probs,
            'value': value,
            'reward': reward,
            'next_state': next_state,
            'done': done
        })
    
    def update(self, n_epochs=10, batch_size=64):
        """Update policy using PPO"""
        if len(self.buffer) < batch_size:
            return
        
        # Convert buffer to tensors
        states = torch.FloatTensor([exp['state'] for exp in self.buffer]).to(self.device)
        actions = torch.LongTensor([exp['actions'] for exp in self.buffer]).to(self.device)
        old_log_probs = torch.stack([exp['log_probs'] for exp in self.buffer]).to(self.device)
        old_values = torch.FloatTensor([exp['value'].item() for exp in self.buffer]).to(self.device)
        rewards = torch.FloatTensor([exp['reward'] for exp in self.buffer]).to(self.device)
        dones = torch.FloatTensor([exp['done'] for exp in self.buffer]).to(self.device)
        
        # Calculate advantages
        advantages = self._calculate_advantages(rewards, old_values, dones)
        returns = advantages + old_values
        
        # Normalize advantages
        advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)
        
        # PPO update
        for epoch in range(n_epochs):
            # Random shuffle
            indices = torch.randperm(len(self.buffer))
            
            for start in range(0, len(self.buffer), batch_size):
                end = start + batch_size
                batch_indices = indices[start:end]
                
                batch_states = states[batch_indices]
                batch_actions = actions[batch_indices]
                batch_old_log_probs = old_log_probs[batch_indices]
                batch_old_values = old_values[batch_indices]
                batch_advantages = advantages[batch_indices]
                batch_returns = returns[batch_indices]
                
                # Forward pass
                new_action_logits, new_values = self.network(batch_states)
                
                # Calculate loss for each agent
                total_policy_loss = 0
                total_entropy_loss = 0
                
                for i in range(3):  # 3 agents
                    if i == 0:  # Topology agent
                        new_action_probs = torch.softmax(new_action_logits[i], dim=-1)
                        batch_agent_actions = batch_actions[:, i]
                        new_log_probs = torch.distributions.Categorical(new_action_probs).log_prob(batch_agent_actions)
                        old_log_probs_agent = batch_old_log_probs[:, i]
                    elif i == 1:  # Market agent
                        new_action_probs = torch.softmax(new_action_logits[i], dim=-1)
                        batch_agent_actions = batch_actions[:, i]
                        new_log_probs = torch.distributions.Categorical(new_action_probs).log_prob(batch_agent_actions)
                        old_log_probs_agent = batch_old_log_probs[:, i]
                    else:  # Coordination agent
                        new_action_probs = torch.softmax(new_action_logits[i], dim=-1)
                        batch_agent_actions = batch_actions[:, i]
                        new_log_probs = torch.distributions.Categorical(new_action_probs).log_prob(batch_agent_actions)
                        old_log_probs_agent = batch_old_log_probs[:, i]
                    
                    # PPO ratio
                    ratio = torch.exp(new_log_probs - old_log_probs_agent)
                    
                    # Clipped surrogate objective
                    surr1 = ratio * batch_advantages
                    surr2 = torch.clamp(ratio, 1 - self.clip_ratio, 1 + self.clip_ratio) * batch_advantages
                    policy_loss = -torch.min(surr1, surr2).mean()
                    
                    total_policy_loss += policy_loss
                    
                    # Entropy bonus
                    entropy = -torch.sum(new_action_probs * torch.log(new_action_probs + 1e-8), dim=-1).mean()
                    total_entropy_loss += -entropy
                
                # Value loss
                value_loss = F.mse_loss(new_values.squeeze(), batch_returns)
                
                # Total loss
                loss = total_policy_loss + self.value_coef * value_loss + self.entropy_coef * total_entropy_loss
                
                # Backward pass
                self.optimizer.zero_grad()
                loss.backward()
                torch.nn.utils.clip_grad_norm_(self.network.parameters(), 0.5)
                self.optimizer.step()
        
        # Clear buffer
        self.buffer.clear()
    
    def _calculate_advantages(self, rewards, values, dones):
        """Calculate Generalized Advantage Estimation"""
        advantages = torch.zeros_like(rewards)
        last_advantage = 0
        
        for t in reversed(range(len(rewards))):
            if t == len(rewards) - 1:
                next_value = 0
            else:
                next_value = values[t + 1]
            
            delta = rewards[t] + self.gamma * next_value * (1 - dones[t]) - values[t]
            advantages[t] = delta + self.gamma * self.gae_lambda * (1 - dones[t]) * last_advantage
            last_advantage = advantages[t]
        
        return advantages

def train_simulation():
    """Main training loop"""
    # Initialize environment and agent
    env = GridEnvironment()
    input_dim = env.observation_space.shape[0]
    agent = PPOAgent(input_dim)
    
    # Training parameters
    n_episodes = 1000
    max_steps_per_episode = 500
    update_frequency = 10
    
    # Tracking metrics
    episode_rewards = []
    congestion_levels = []
    switching_actions = []
    
    print("Starting Multi-Agent RL Training for Market-Aware Grid Topology Optimization")
    print("=" * 80)
    
    for episode in range(n_episodes):
        state = env.reset()
        episode_reward = 0
        episode_congestion = []
        episode_switches = []
        
        for step in range(max_steps_per_episode):
            # Select actions
            actions, log_probs, value = agent.select_action(state)
            
            # Take step
            next_state, reward, done, _ = env.step(actions)
            
            # Store experience
            agent.store_experience(state, actions, log_probs, value, reward, next_state, done)
            
            # Track metrics
            episode_reward += reward
            episode_congestion.append(np.sum(env.grid_state.congestion_status))
            episode_switches.append(1 if actions[0] > 0 else 0)
            
            state = next_state
            
            if done:
                break
        
        # Update agent
        if episode % update_frequency == 0:
            agent.update()
        
        # Record episode metrics
        episode_rewards.append(episode_reward)
        congestion_levels.append(np.mean(episode_congestion))
        switching_actions.append(np.sum(episode_switches))
        
        # Print progress
        if episode % 50 == 0:
            avg_reward = np.mean(episode_rewards[-50:])
            avg_congestion = np.mean(congestion_levels[-50:])
            avg_switches = np.mean(switching_actions[-50:])
            
            print(f"Episode {episode:4d} | Avg Reward: {avg_reward:8.2f} | "
                  f"Avg Congestion: {avg_congestion:5.2f} | Avg Switches: {avg_switches:5.1f}")
    
    print("\nTraining completed!")
    
    # Plot results
    plot_training_results(episode_rewards, congestion_levels, switching_actions)
    
    return agent, env, episode_rewards, congestion_levels, switching_actions

def plot_training_results(rewards, congestion, switches):
    """Plot training metrics"""
    fig, axes = plt.subplots(3, 1, figsize=(12, 10))
    
    # Smooth the curves
    window = 50
    rewards_smooth = np.convolve(rewards, np.ones(window)/window, mode='valid')
    congestion_smooth = np.convolve(congestion, np.ones(window)/window, mode='valid')
    switches_smooth = np.convolve(switches, np.ones(window)/window, mode='valid')
    
    # Episode rewards
    axes[0].plot(rewards_smooth)
    axes[0].set_title('Episode Rewards (Smoothed)')
    axes[0].set_ylabel('Reward')
    axes[0].grid(True)
    
    # Congestion levels
    axes[1].plot(congestion_smooth)
    axes[1].set_title('Average Congestion Levels (Smoothed)')
    axes[1].set_ylabel('Congested Lines')
    axes[1].grid(True)
    
    # Switching actions
    axes[2].plot(switches_smooth)
    axes[2].set_title('Topology Switching Frequency (Smoothed)')
    axes[2].set_ylabel('Switches per Episode')
    axes[2].set_xlabel('Episode')
    axes[2].grid(True)
    
    plt.tight_layout()
    plt.savefig('training_results.png', dpi=300, bbox_inches='tight')
    plt.show()

def evaluate_agent(agent, env, n_episodes=100):
    """Evaluate trained agent"""
    print("\nEvaluating trained agent...")
    
    total_rewards = []
    total_congestion = []
    total_switches = []
    
    for episode in range(n_episodes):
        state = env.reset()
        episode_reward = 0
        episode_congestion = []
        episode_switches = []
        
        for step in range(500):  # Fixed evaluation steps
            actions, _, _ = agent.select_action(state)
            next_state, reward, done, _ = env.step(actions)
            
            episode_reward += reward
            episode_congestion.append(np.sum(env.grid_state.congestion_status))
            episode_switches.append(1 if actions[0] > 0 else 0)
            
            state = next_state
            
            if done:
                break
        
        total_rewards.append(episode_reward)
        total_congestion.append(np.mean(episode_congestion))
        total_switches.append(np.sum(episode_switches))
    
    print(f"Evaluation Results ({n_episodes} episodes):")
    print(f"  Average Reward: {np.mean(total_rewards):.2f} ± {np.std(total_rewards):.2f}")
    print(f"  Average Congestion: {np.mean(total_congestion):.2f} ± {np.std(total_congestion):.2f}")
    print(f"  Average Switches: {np.mean(total_switches):.1f} ± {np.std(total_switches):.1f}")
    
    return total_rewards, total_congestion, total_switches

def save_results(agent, rewards, congestion, switches):
    """Save training results and model"""
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    
    # Save model
    torch.save(agent.network.state_dict(), f'multi_agent_model_{timestamp}.pth')
    
    # Save training data
    results = {
        'episode_rewards': rewards,
        'congestion_levels': congestion,
        'switching_actions': switches,
        'training_timestamp': timestamp
    }
    
    with open(f'training_results_{timestamp}.json', 'w') as f:
        json.dump(results, f, indent=2)
    
    print(f"\nResults saved with timestamp: {timestamp}")

if __name__ == "__main__":
    # Import PyTorch functional interface
    import torch.nn.functional as F
    
    # Train the multi-agent system
    agent, env, rewards, congestion, switches = train_simulation()
    
    # Evaluate the trained agent
    eval_rewards, eval_congestion, eval_switches = evaluate_agent(agent, env)
    
    # Save results
    save_results(agent, rewards, congestion, switches)
    
    print("\n" + "=" * 80)
    print("Multi-Agent RL Simulation Completed Successfully!")
    print("=" * 80)
    print("\nKey Achievements:")
    print("✅ Implemented market-aware topology optimization")
    print("✅ Multi-agent coordination (topology + market + coordination)")
    print("✅ Congestion management through proactive switching")
    print("✅ Integration of P2P trading signals")
    print("✅ Reward function balancing reliability and economics")
    print("\nNext Steps:")
    print("📋 Integrate with blockchain audit framework")
    print("📋 Add zero-knowledge proofs for privacy")
    print("📋 Scale to larger grid models")
    print("📋 Implement real-time market data feeds")