Upload 6 files

Environment_Wrapper.py

@@ -0,0 +1,75 @@
+import gymnasium as gym
+import numpy as np
+import cv2
+from gymnasium.spaces import Box
+
+class CarRacingEnvWrapper(gym.Wrapper):
+    def __init__(self, env, num_stack_frames, grayscale, resize_dim):
+        super().__init__(env)
+        self.num_stack_frames = num_stack_frames
+        self.grayscale = grayscale
+        self.resize_dim = resize_dim
+        self.orig_observation_space = self.env.observation_space
+
+        if self.grayscale:
+            shape = (self.resize_dim[0], self.resize_dim[1], self.num_stack_frames)
+        else:
+            shape = (self.resize_dim[0], self.resize_dim[1], self.env.observation_space.shape[2] * self.num_stack_frames)
+            
+        self.observation_space = Box(low=0, high=255, shape=shape, dtype=np.uint8)
+        self.frames = np.zeros(shape, dtype=np.uint8)
+
+    def _process_obs(self, obs):
+       
+        obs = obs.astype(np.float32) / 255.0
+        
+        if self.grayscale:
+            
+            grayscale_obs = np.dot(obs[...,:3], [0.2989, 0.5870, 0.1140])
+            resized_obs = cv2.resize(grayscale_obs, self.resize_dim, interpolation=cv2.INTER_AREA)
+           
+            processed_obs = np.expand_dims(resized_obs, axis=-1)
+        else:
+            
+            processed_obs = cv2.resize(obs, self.resize_dim, interpolation=cv2.INTER_AREA)
+            
+        return (processed_obs * 255).astype(np.uint8)
+
+    def _stack_frames(self, processed_obs):
+        self.frames = np.roll(self.frames, shift=-1, axis=-1)
+        if self.grayscale:
+            self.frames[..., -1] = processed_obs[..., 0]
+        else:
+            self.frames[..., -3:] = processed_obs
+        return self.frames
+
+    def reset(self, **kwargs):
+        obs, info = self.env.reset(**kwargs)
+        processed_obs = self._process_obs(obs)
+        
+        if self.grayscale:
+            self.frames = np.stack([processed_obs[..., 0]] * self.num_stack_frames, axis=-1)
+        else:
+            self.frames = np.stack([processed_obs] * self.num_stack_frames, axis=-1)
+            
+        return self.frames, info
+
+    def step(self, action):
+        obs, reward, terminated, truncated, info = self.env.step(action)
+        
+        is_stuck = np.mean(obs[64:72, 32:64, 1]) < 10
+        if is_stuck and reward < 0:
+            truncated = True
+        
+        is_on_grass = np.mean(obs[64:72, 32:64, 1]) > 100
+        if is_on_grass:
+            reward -= 10
+        
+        if info.get('is_complete'):
+            reward += 100
+        
+        processed_obs = self._process_obs(obs)
+        stacked_frames = self._stack_frames(processed_obs)
+        info['original_reward'] = info.get('original_reward', reward)
+        
+        return stacked_frames, reward, terminated, truncated, info

GPU_test.py

@@ -0,0 +1,19 @@
+import tensorflow as tf
+print("Num GPUs Available: ", len(tf.config.list_physical_devices('GPU')))
+
+gpus = tf.config.list_physical_devices('GPU')
+if gpus:
+    try:
+    
+        for gpu in gpus:
+            tf.config.experimental.set_memory_growth(gpu, True)
+        logical_gpus = tf.config.experimental.list_logical_devices('GPU')
+        print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
+    except RuntimeError as e:
+    
+        print(e)
+
+import gymnasium as gym
+env = gym.make("CarRacing-v3", render_mode="rgb_array")
+print("CarRacing-v3 environment created successfully.")
+env.close()

PPO_Model.py

@@ -0,0 +1,273 @@
+import gymnasium as gym
+import numpy as np
+from collections import deque
+import tensorflow as tf
+from keras import optimizers
+from keras.optimizers import Adam
+import tensorflow_probability as tfp
+import os
+from datetime import datetime
+import json
+from RaceCar import ActorCritic
+from Environment_Wrapper import CarRacingEnvWrapper
+from gymnasium.wrappers import TimeLimit
+tfd = tfp.distributions
+
+class PPOAgent:
+    def __init__(self, env_id="CarRacing-v3", num_envs=21,
+                 gamma=0.99, lam=0.95, clip_epsilon=0.2,
+                 actor_lr=3e-4, critic_lr=3e-4,
+                 ppo_epochs=10, minibatches=4,
+                 steps_per_batch=1024,
+                 num_stack_frames=4, resize_dim=(84, 84), grayscale=True,
+                 seed=42, log_dir="./ppo_logs",
+                 entropy_coeff=0.01,
+                 save_interval_timesteps=537600,
+                 hidden_layer_sizes=[512, 512, 512]):
+        self.env_id = env_id
+        self.num_envs = num_envs
+        self.gamma = gamma
+        self.lam = lam
+        self.clip_epsilon = clip_epsilon
+        self.ppo_epochs = ppo_epochs
+        self.minibatches = minibatches
+        self.steps_per_batch = steps_per_batch
+        self.num_stack_frames = num_stack_frames
+        self.resize_dim = resize_dim
+        self.grayscale = grayscale
+        self.seed = seed
+        self.log_dir = log_dir
+        self.entropy_coeff = entropy_coeff
+        self.save_interval_timesteps = save_interval_timesteps
+        self.hidden_layer_sizes = hidden_layer_sizes
+        self.envs = self._make_vec_envs()
+        self.action_dim = self.envs.single_action_space.shape[0]
+        self.observation_shape = self.envs.single_observation_space.shape
+        self.model = ActorCritic(self.action_dim,
+                                 self.num_stack_frames,
+                                 self.resize_dim[1],
+                                 self.resize_dim[0],
+                                 hidden_layer_sizes=self.hidden_layer_sizes)
+        self.actor_optimizer = Adam(learning_rate=actor_lr)
+        self.critic_optimizer = Adam(learning_rate=critic_lr)
+        self.train_log_dir = None
+        self.summary_writer = None
+        tf.random.set_seed(self.seed)
+        np.random.seed(self.seed)
+        dummy_input = np.zeros((1, *self.observation_shape), dtype=np.uint8)
+        _ = self.model(dummy_input)
+
+    def _make_env(self):
+        env = gym.make(self.env_id, render_mode="rgb_array", continuous=True)
+        env = CarRacingEnvWrapper(env, num_stack_frames=self.num_stack_frames,
+                                  grayscale=self.grayscale, resize_dim=self.resize_dim)
+        env = TimeLimit(env, max_episode_steps=1000)
+        return env
+
+    def _make_vec_envs(self):
+        return gym.vector.AsyncVectorEnv([lambda: self._make_env() for _ in range(self.num_envs)])
+
+    def _compute_returns_and_advantages(self, rewards, values, dones):
+        advantages = np.zeros_like(rewards, dtype=np.float32)
+        returns = np.zeros_like(rewards, dtype=np.float32)
+        last_gae_lam = np.zeros(self.num_envs, dtype=np.float32)
+        for t in reversed(range(self.steps_per_batch)):
+            next_non_terminal = 1.0 - dones[t]
+            delta = rewards[t] + self.gamma * values[t+1] * next_non_terminal - values[t]
+            last_gae_lam = delta + self.gamma * self.lam * next_non_terminal * last_gae_lam
+            advantages[t] = last_gae_lam
+        returns = advantages + values[:-1]
+        return advantages, returns
+
+    @tf.function
+    def _train_step(self, observations, actions, old_log_probs, advantages, returns):
+        with tf.GradientTape() as tape:
+            action_distribution, value_pred = self.model(observations)
+            value_pred = tf.squeeze(value_pred, axis=-1)
+            critic_loss = tf.reduce_mean(tf.square(returns - value_pred))
+            log_prob = action_distribution.log_prob(actions)
+            ratio = tf.exp(log_prob - old_log_probs)
+            ratio = tf.where(tf.math.is_nan(ratio), 1.0, ratio)
+            ratio = tf.where(tf.math.is_inf(ratio), tf.sign(ratio) * 1e5, ratio)
+            pg_loss1 = ratio * advantages
+            pg_loss2 = tf.clip_by_value(ratio, 1 - self.clip_epsilon, 1 + self.clip_epsilon) * advantages
+            actor_loss = -tf.reduce_mean(tf.minimum(pg_loss1, pg_loss2))
+            entropy = tf.reduce_mean(action_distribution.entropy())
+            entropy_loss = -self.entropy_coeff * entropy
+            total_loss = actor_loss + critic_loss + entropy_loss
+        grads = tape.gradient(total_loss, self.model.trainable_variables)
+        self.actor_optimizer.apply_gradients(zip(grads, self.model.trainable_variables))
+        return actor_loss, critic_loss, entropy, total_loss
+
+    def train(self, total_timesteps, resume_from_timestep=0, resume_model_path=None, run_log_dir=None):
+        global_timestep = resume_from_timestep
+        ep_rewards = deque(maxlen=100)
+        ep_modified_rewards = deque(maxlen=100)
+        resume_json_path = "resume_config.json"
+        if run_log_dir is None:
+            if os.path.exists(resume_json_path):
+                with open(resume_json_path, "r") as f:
+                    resume_info = json.load(f)
+                    self.train_log_dir = resume_info.get("run_log_directory")
+            if self.train_log_dir is None:
+                current_time = datetime.now().strftime("%Y%m%d-%H%M%S")
+                self.train_log_dir = os.path.join(self.log_dir, current_time)
+        else:
+            self.train_log_dir = run_log_dir
+        os.makedirs(os.path.join(self.train_log_dir, "checkpoints"), exist_ok=True)
+        self.summary_writer = tf.summary.create_file_writer(self.train_log_dir)
+        if resume_model_path and os.path.exists(resume_model_path):
+            self.model.load_weights(resume_model_path)
+            print(f"Resuming training from timestep {global_timestep} and loaded model from: {resume_model_path}")
+        elif resume_from_timestep > 0:
+            print(f"WARNING: Attempting to resume from timestep {global_timestep} but no valid model path provided or found. Starting fresh.")
+        else:
+            print("Starting new training run.")
+        obs, _ = self.envs.reset(seed=self.seed)
+        print(f"Target total timesteps: {total_timesteps}")
+        print(f"Current global timestep: {global_timestep}")
+        print("Starting training loop...")
+        while global_timestep < total_timesteps:
+            batch_observations = np.zeros((self.steps_per_batch, self.num_envs, *self.observation_shape), dtype=np.uint8)
+            batch_actions = np.zeros((self.steps_per_batch, self.num_envs, self.action_dim), dtype=np.float32)
+            batch_rewards = np.zeros((self.steps_per_batch, self.num_envs), dtype=np.float32)
+            batch_dones = np.zeros((self.steps_per_batch, self.num_envs), dtype=bool)
+            batch_values = np.zeros((self.steps_per_batch, self.num_envs), dtype=np.float32)
+            batch_log_probs = np.zeros((self.steps_per_batch, self.num_envs), dtype=np.float32)
+            batch_original_rewards = np.zeros((self.steps_per_batch, self.num_envs), dtype=np.float32)
+            for i in range(self.steps_per_batch):
+                
+                #print("Step:", i)
+                tf_obs = tf.convert_to_tensor(obs, dtype=tf.uint8)
+                action_dist, value = self.model(tf_obs)
+                #print("Model forward pass complete.")
+
+                action = action_dist.sample()
+                log_prob = action_dist.log_prob(action)
+                #print("Action sampled and log_prob calculated.")
+
+                action_np = action.numpy()
+                value_np = tf.squeeze(value).numpy()
+                log_prob_np = log_prob.numpy()
+                next_obs, reward, terminated, truncated, info = self.envs.step(action_np)
+                #print("Environment step complete.")
+
+                done = terminated | truncated
+                batch_observations[i] = obs
+                batch_actions[i] = action_np
+                batch_rewards[i, :] = reward
+                batch_dones[i] = done
+                batch_values[i] = value_np
+                batch_log_probs[i] = log_prob_np
+                
+                for single_env_info in info:
+                    if isinstance(single_env_info, dict) and single_env_info.get('episode'):
+                        episode_reward = single_env_info['episode']['r']
+                        ep_rewards.append(episode_reward)
+
+                obs = next_obs
+                global_timestep += self.num_envs
+            
+            _, last_values_np = self.model(tf.convert_to_tensor(obs, dtype=tf.uint8))
+            last_values_np = tf.squeeze(last_values_np).numpy()
+            batch_values = np.concatenate((batch_values, last_values_np[np.newaxis, :]), axis=0)
+            advantages, returns = self._compute_returns_and_advantages(
+                batch_rewards, batch_values, batch_dones
+            )
+            flat_observations = batch_observations.reshape((-1, *self.observation_shape))
+            flat_actions = batch_actions.reshape((-1, self.action_dim))
+            flat_old_log_probs = batch_log_probs.reshape(-1)
+            flat_advantages = advantages.reshape(-1)
+            flat_returns = returns.reshape(-1)
+            flat_advantages = (flat_advantages - np.mean(flat_advantages)) / (np.std(flat_advantages) + 1e-8)
+            batch_original_total_reward = np.sum(batch_original_rewards)
+            batch_modified_total_reward = np.sum(batch_rewards)
+            batch_indices = np.arange(self.steps_per_batch * self.num_envs)
+            for _ in range(self.ppo_epochs):
+                np.random.shuffle(batch_indices)
+                for start_idx in range(0, len(batch_indices) // self.minibatches * self.minibatches, len(batch_indices) // self.minibatches):
+                    end_idx = start_idx + len(batch_indices) // self.minibatches
+                    minibatch_indices = batch_indices[start_idx:end_idx]
+                    mb_obs = tf.constant(flat_observations[minibatch_indices], dtype=tf.uint8)
+                    mb_actions = tf.constant(flat_actions[minibatch_indices], dtype=tf.float32)
+                    mb_old_log_probs = tf.constant(flat_old_log_probs[minibatch_indices], dtype=tf.float32)
+                    mb_advantages = tf.constant(flat_advantages[minibatch_indices], dtype=tf.float32)
+                    mb_returns = tf.constant(flat_returns[minibatch_indices], dtype=tf.float32)
+                    actor_loss, critic_loss, entropy, total_loss = self._train_step(
+                        mb_obs, mb_actions, mb_old_log_probs, mb_advantages, mb_returns
+                    )
+            with self.summary_writer.as_default():
+                tf.summary.scalar("charts/total_timesteps", global_timestep, step=global_timestep)
+                tf.summary.scalar("losses/actor_loss", actor_loss, step=global_timestep)
+                tf.summary.scalar("losses/critic_loss", critic_loss, step=global_timestep)
+                tf.summary.scalar("losses/entropy", entropy, step=global_timestep)
+                tf.summary.scalar("losses/total_loss", total_loss, step=global_timestep)
+                if ep_rewards:
+                    tf.summary.scalar("charts/avg_episode_reward_original_env", np.mean(ep_rewards), step=global_timestep)
+                    tf.summary.scalar("charts/min_episode_reward_original_env", np.min(ep_rewards), step=global_timestep)
+                    tf.summary.scalar("charts/max_episode_reward_original_env", np.max(ep_rewards), step=global_timestep)
+                tf.summary.scalar("charts/batch_total_reward_original", batch_original_total_reward, step=global_timestep)
+                tf.summary.scalar("charts/batch_total_reward_modified", batch_modified_total_reward, step=global_timestep)
+                with tf.name_scope('actor_std'):
+                    tf.summary.histogram('log_std', self.model.actor_log_std, step=global_timestep)
+                    tf.summary.scalar('std_mean_overall', tf.reduce_mean(tf.exp(self.model.actor_log_std)), step=global_timestep)
+            self.summary_writer.flush()
+            if global_timestep % self.save_interval_timesteps == 0:
+                if ep_rewards:
+                    avg_orig_reward_str = f"{np.mean(ep_rewards):.2f}"
+                else:
+                    avg_orig_reward_str = 'N/A'
+                print(f"Timestep: {global_timestep}, Number of episodes in Timestep: (525 eps)") #{avg_orig_reward_str}")
+                current_checkpoint_path = os.path.join(self.train_log_dir, "checkpoints", f"Actor-Critic_at_{global_timestep}.weights.h5")
+                self.model.save_weights(current_checkpoint_path)
+                resume_info = {
+                    "last_global_timestep": global_timestep,
+                    "last_checkpoint_path": current_checkpoint_path,
+                    "run_log_directory": self.train_log_dir
+                }
+                with open(resume_json_path, "w") as f:
+                    json.dump(resume_info, f, indent=4)
+                print(f"Resume info saved to {resume_json_path}")
+        print("Training finished.")
+        self.envs.close()
+        self.summary_writer.close()
+        final_model_path = os.path.join(self.train_log_dir, "Actor-Critic_final_model.weights.h5")
+        self.model.save_weights(final_model_path)
+        if os.path.exists(resume_json_path):
+            os.remove(resume_json_path)
+            print(f"Removed {resume_json_path} as training completed successfully.")
+    
+    def evaluate(self, num_episodes=5, render=True, model_path=None):
+        eval_env = gym.make(self.env_id, render_mode="human" if render else "rgb_array", continuous=True)
+        eval_env = CarRacingEnvWrapper(eval_env, num_stack_frames=self.num_stack_frames,
+                                       grayscale=self.grayscale, resize_dim=self.resize_dim)
+        if model_path:
+            dummy_input = np.zeros((1, *self.observation_shape), dtype=np.uint8)
+            _ = self.model(dummy_input)
+            self.model.load_weights(model_path)
+            print(f"Loaded model from {model_path}")
+        episode_rewards = []
+        episode_original_rewards = []
+        for ep in range(num_episodes):
+            obs, _ = eval_env.reset()
+            done = False
+            total_reward = 0
+            total_original_reward = 0
+            while not done:
+                tf_obs = tf.convert_to_tensor(obs[np.newaxis, :], dtype=tf.uint8)
+                action_dist, _ = self.model(tf_obs)
+                action = action_dist.mean().numpy().flatten()
+                action[0] = np.clip(action[0], -1.0, 1.0)
+                action[1] = np.clip(action[1], 0.0, 1.0)
+                action[2] = np.clip(action[2], 0.0, 1.0)
+                obs, reward, terminated, truncated, info = eval_env.step(action)
+                done = terminated or truncated
+                total_reward += reward
+                total_original_reward += info.get('original_reward', reward)
+            episode_rewards.append(total_reward)
+            episode_original_rewards.append(total_original_reward)
+            print(f"Episode {ep+1} finished. Modified Reward: {total_reward:.2f}, Original Env Reward: {total_original_reward:.2f}")
+        eval_env.close()
+        print(f"Average modified evaluation reward over {num_episodes} episodes: {np.mean(episode_rewards):.2f}")
+        print(f"Average original environment reward over {num_episodes} episodes: {np.mean(episode_original_rewards):.2f}")
+        return episode_rewards, episode_original_rewards

RaceCar.py

@@ -0,0 +1,49 @@
+import tensorflow as tf
+import keras
+from keras import layers, Model
+import tensorflow_probability as tfp
+
+tfd = tfp.distributions
+
+class ActorCritic(keras.Model):
+    def __init__(self, action_dim, num_stack_frames, img_height, img_width, hidden_layer_sizes):
+        super().__init__()
+        self.conv_layers = keras.Sequential([
+            layers.Conv2D(32, 8, strides=4, activation="relu", input_shape=(img_height, img_width, num_stack_frames)),
+            layers.Conv2D(64, 4, strides=2, activation="relu"),
+            layers.Conv2D(64, 3, strides=1, activation="relu"),
+            layers.Flatten(),
+        ])
+
+        actor_layers = []
+
+        for size in hidden_layer_sizes:
+            actor_layers.append(layers.Dense(size, activation="relu"))
+        self.common_actor_layer = keras.Sequential(actor_layers)
+
+        self.actor_mean = layers.Dense(action_dim, activation=None) 
+        self.actor_log_std = tf.Variable(tf.zeros(action_dim, dtype=tf.float32), trainable=True)
+
+        critic_layers = []
+        for size in hidden_layer_sizes:
+            critic_layers.append(layers.Dense(size, activation="relu"))
+        self.common_critic_layer = keras.Sequential(critic_layers)
+
+        self.critic_value = layers.Dense(1, activation=None) 
+
+    def call(self, inputs):
+        normalized_inputs = tf.cast(inputs, tf.float32) / 255.0
+        
+        features = self.conv_layers(normalized_inputs)
+        
+        actor_features = self.common_actor_layer(features)
+        mean = self.actor_mean(actor_features)
+        
+        std = tf.exp(self.actor_log_std) 
+        
+        action_distribution = tfd.MultivariateNormalDiag(loc=mean, scale_diag=std)
+
+        critic_features = self.common_critic_layer(features)
+        value = self.critic_value(critic_features)
+        
+        return action_distribution, value

Train.py

@@ -0,0 +1,100 @@
+import os
+import json
+from PPO_Model import PPOAgent
+from datetime import datetime
+import tensorflow as tf
+
+gpus = tf.config.list_physical_devices('GPU')
+if gpus:
+  try:
+    for gpu in gpus:
+      tf.config.experimental.set_memory_growth(gpu, True)
+    print("GPU memory growth is enabled.")
+  except RuntimeError as e:
+
+    print(e)
+
+if __name__ == '__main__':
+
+    ENV_ID = "CarRacing-v3"
+    NUM_ENVS = 21
+    TOTAL_TIMESTEPS = 30_000_000
+    SAVE_INTERVAL_TIMESTEPS = 537600
+    PPO_EPOCHS = 10
+    MINIBATCHES = 4
+    STEPS_PER_BATCH = 1024
+    GAMMA = 0.99
+    LAM = 0.95
+    CLIP_EPSILON = 0.2
+    ACTOR_LR = 3e-4
+    CRITIC_LR = 3e-4
+    ENTROPY_COEFF = 0.01
+
+    NUM_STACK_FRAMES = 4
+    RESIZE_DIM = (84, 84)
+    GRAYSCALE = True
+    HIDDEN_LAYER_SIZES = [512, 512, 512]
+
+    SEED = 42
+    LOG_DIR = "./ppo_car_racing_logs"
+    RESUME_CONFIG_PATH = "resume_config.json"
+    
+    resume_from_timestep = 0
+    resume_model_path = None
+    run_log_dir = None
+
+    if os.path.exists(RESUME_CONFIG_PATH):
+        try:
+            with open(RESUME_CONFIG_PATH, "r") as f:
+                resume_info = json.load(f)
+                run_log_dir = resume_info.get("run_log_directory")
+            
+            if run_log_dir:
+                checkpoints_dir = os.path.join(run_log_dir, "checkpoints")
+                if os.path.exists(checkpoints_dir):
+                    checkpoint_files = [f for f in os.listdir(checkpoints_dir) if f.endswith(".weights.h5")]
+                    if checkpoint_files:
+                    
+                        latest_checkpoint = max(checkpoint_files, key=lambda f: os.path.getmtime(os.path.join(checkpoints_dir, f)))
+                        resume_model_path = os.path.join(checkpoints_dir, latest_checkpoint)
+                        
+                        timestep_str = latest_checkpoint.split("_at_")[1].split(".")[0]
+                        resume_from_timestep = int(timestep_str)
+                        print(f"Resume file found. Will resume from timestep {resume_from_timestep} using model: {resume_model_path}")
+
+        except (IOError, json.JSONDecodeError) as e:
+            print(f"Error reading resume file, starting a new session: {e}")
+            run_log_dir = None
+            if os.path.exists(RESUME_CONFIG_PATH):
+                os.remove(RESUME_CONFIG_PATH)
+
+    if not run_log_dir:
+        run_log_dir = os.path.join(LOG_DIR, datetime.now().strftime("%Y%m%d-%H%M%S"))
+    
+    agent = PPOAgent(
+        env_id=ENV_ID,
+        num_envs=NUM_ENVS,
+        gamma=GAMMA,
+        lam=LAM,
+        clip_epsilon=CLIP_EPSILON,
+        actor_lr=ACTOR_LR,
+        critic_lr=CRITIC_LR,
+        ppo_epochs=PPO_EPOCHS,
+        minibatches=MINIBATCHES,
+        steps_per_batch=STEPS_PER_BATCH,
+        num_stack_frames=NUM_STACK_FRAMES,
+        resize_dim=RESIZE_DIM,
+        grayscale=GRAYSCALE,
+        seed=SEED,
+        log_dir=LOG_DIR,
+        entropy_coeff=ENTROPY_COEFF,
+        save_interval_timesteps=SAVE_INTERVAL_TIMESTEPS,
+        hidden_layer_sizes=HIDDEN_LAYER_SIZES
+    )
+    
+    agent.train(
+        total_timesteps=TOTAL_TIMESTEPS,
+        resume_from_timestep=resume_from_timestep,
+        resume_model_path=resume_model_path,
+        run_log_dir=run_log_dir
+    )

Trained_Agent.py

@@ -0,0 +1,82 @@
+import tensorflow as tf
+import os
+from PPO_Model import PPOAgent
+
+gpus = tf.config.list_physical_devices('GPU')
+if gpus:
+    try:
+        for gpu in gpus:
+            tf.config.experimental.set_memory_growth(gpu, True)
+        logical_gpus = tf.config.experimental.list_logical_devices('GPU')
+        print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
+    except RuntimeError as e:
+        print(e)
+
+agent_config = {
+    "env_id": "CarRacing-v3",
+    "num_envs": 21,
+    "gamma": 0.99,
+    "lam": 0.95,
+    "clip_epsilon": 0.2,
+    "actor_lr": 3e-4,
+    "critic_lr": 3e-4,
+    "ppo_epochs": 10,
+    "minibatches": 4,
+    "steps_per_batch": 1024,
+    "num_stack_frames": 4,
+    "resize_dim": (84, 84),
+    "grayscale": True,
+    "seed": 42,
+    "log_dir": "./ppo_car_racing_logs",
+    "entropy_coeff": 0.01,
+    'save_interval_timesteps': 537600,
+    'hidden_layer_sizes': [512, 512, 512]
+}
+
+if __name__ == "__main__":
+    print("Initializing PPO Agent for evaluation...")
+    
+    agent = PPOAgent(**agent_config)
+
+    root_log_dir = "./ppo_car_racing_logs"
+
+    latest_log_run_dir = None
+    if os.path.exists(root_log_dir):
+        all_runs = [os.path.join(root_log_dir, d) for d in os.listdir(root_log_dir) if os.path.isdir(os.path.join(root_log_dir, d))]
+        if all_runs:
+    
+            latest_log_run_dir = max(all_runs, key=os.path.getmtime)
+            print(f"Found latest training run directory: {latest_log_run_dir}")
+        else:
+            print(f"No training run directories found in {root_log_dir}.")
+    else:
+        print(f"Log directory {root_log_dir} does not exist. Cannot find trained model.")
+
+
+    model_to_load = None
+    if latest_log_run_dir:
+
+        final_model_path = os.path.join(latest_log_run_dir, "final_model.weights.h5")
+        if os.path.exists(final_model_path):
+            model_to_load = final_model_path
+        else:
+            print(f"Final model weights not found in {latest_log_run_dir}. Checking checkpoints...")
+            
+            checkpoint_dir = os.path.join(latest_log_run_dir, "checkpoints")
+            if os.path.exists(checkpoint_dir):
+                all_checkpoints = [os.path.join(checkpoint_dir, f) for f in os.listdir(checkpoint_dir) if f.endswith(".weights.h5")]
+                if all_checkpoints:
+                    model_to_load = max(all_checkpoints, key=os.path.getmtime)
+                    print(f"Loading latest checkpoint: {model_to_load}")
+                else:
+                    print("No checkpoints found.")
+            else:
+                print("Checkpoints directory does not exist.")
+
+
+    if model_to_load:
+        
+        print("\n--- Evaluation ---")
+        agent.evaluate(num_episodes=10, render=True, model_path=model_to_load)
+    else:
+        print("No trained model found to evaluate. Please train an agent first.")