pushing model

README.md

@@ -16,7 +16,7 @@ model-index:
       type: CartPole-v1
     metrics:
     - type: mean_reward
-      value: 495.67 +/- 0.00
+      value: 499.44 +/- 0.00
       name: mean_reward
       verified: false
 ---
@@ -67,7 +67,7 @@ python dqn.py --track --wandb-entity pfunk --wandb-project-name dqpn --capture-v
  'save_model': True,
  'seed': 4,
  'start_e': 1.0,
- 'target_network_frequency': 20,
+ 'target_network_frequency': 100,
  'target_tau': 1.0,
  'torch_deterministic': True,
  'total_timesteps': 500000,

dqn.py

@@ -54,7 +54,7 @@ def parse_args():
         help="the discount factor gamma")
     parser.add_argument("--target-tau", type=float, default=1.0,
         help="the target network update rate")
-    parser.add_argument("--target-network-frequency", type=int, default=20,
+    parser.add_argument("--target-network-frequency", type=int, default=100,
         help="the timesteps it takes to update the target network")
     parser.add_argument("--batch-size", type=int, default=256,
         help="the batch size of sample from the reply memory")
@@ -137,17 +137,16 @@ if __name__ == "__main__":
         wandb.log({name: x, "global_step": y})
 
     # TRY NOT TO MODIFY: seeding
-    random.seed(args.seed)
-    np.random.seed(args.seed)
     torch.manual_seed(args.seed)
-    torch.backends.cudnn.deterministic = args.torch_deterministic
+    # torch.backends.cudnn.deterministic = args.torch_deterministic
+    np.random.seed(args.seed)
+    random.seed(args.seed)
 
     device = torch.device("cuda" if torch.cuda.is_available() and args.cuda else "cpu")
 
     # env setup
     envs = gym.vector.SyncVectorEnv([make_env(args.env_id, args.seed, 0, args.capture_video, run_name)])
     assert isinstance(envs.single_action_space, gym.spaces.Discrete), "only discrete action space is supported"
-    envs.seed(args.seed)
 
     q_network = QNetwork(envs).to(device)
     optimizer = optim.RMSprop(q_network.parameters(), lr=args.learning_rate)
@@ -159,10 +158,11 @@ if __name__ == "__main__":
         envs.single_observation_space,
         envs.single_action_space,
         device,
-        optimize_memory_usage=True,
+        # optimize_memory_usage=True,
         handle_timeout_termination=True,
     )
     start_time = time.time()
+    target_update_counter = 0
     policy_update_counter = 0
     episode_returns = []
 
@@ -247,10 +247,10 @@ if __name__ == "__main__":
                     log_value("td/a_below", a_below, global_step)
                     log_value("td/above", above, global_step)
                     log_value("td/a_above", a_above, global_step)
-                    log_value("pu/pu_scalar", pu_scalar, global_step)
-                    log_value("pu/a_pu_scalar", a_pu_scalar, global_step)
-                    log_value("pu/policy_frequency_scalar_ratio", policy_frequency_scalar_ratio, global_step)
-                    log_value("pu/a_policy_frequency_scalar_ratio", a_policy_frequency_scalar_ratio, global_step)
+                    log_value("alg/pu_scalar", pu_scalar, global_step)
+                    log_value("alg/a_pu_scalar", a_pu_scalar, global_step)
+                    log_value("alg/policy_frequency_scalar_ratio", policy_frequency_scalar_ratio, global_step)
+                    log_value("alg/a_policy_frequency_scalar_ratio", a_policy_frequency_scalar_ratio, global_step)
                     log_value("debug/steps_per_second", int(global_step / (time.time() - start_time)), global_step)
 
                 # optimize the model
@@ -260,13 +260,16 @@ if __name__ == "__main__":
 
             # update target network
             if global_step % args.target_network_frequency == 0:
+                target_update_counter += 1
                 for target_network_param, q_network_param in zip(target_network.parameters(), q_network.parameters()):
                     target_network_param.data.copy_(
                         args.target_tau * q_network_param.data + (1.0 - args.target_tau) * target_network_param.data
                     )
             policy_update_counter += 1
+
             if global_step % 100 == 0:
-                log_value("pu/n_policy_update", policy_update_counter, global_step)
+                log_value("alg/n_target_update", target_update_counter, global_step)
+                log_value("alg/n_policy_update", policy_update_counter, global_step)
 
     if args.save_model:
         model_path = f"runs/{run_name}/{args.exp_name}.cleanrl_model"

events.out.tfevents.1679277727.redi.183238.0 → events.out.tfevents.1679332037.redi.1483546.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd727b11020bbb1ab84daa394ece4c3536ecb961988782fd59fa7b26901f5bec
-size 627
+oid sha256:ec2f3a5396fd9f4dd2f39620190713722ea55bd71a5e676744a4a7336e1edec8
+size 628