Upload folder using huggingface_hub

.summary/0/events.out.tfevents.1696938031.rhmmedcatt-proliant-ml350-gen10

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d84d50f86e90fdb71d066a8e5dad42a87b0771f2c818239489a0bd788bef47fd
+size 7527932

.summary/1/events.out.tfevents.1696938031.rhmmedcatt-proliant-ml350-gen10

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0d0336e4d3d124f318beca3ed9eaf9f7e946b2b5d7ad67e868ca25733244349
+size 4362998

README.md

@@ -15,7 +15,7 @@ model-index:
       type: atari_defender
     metrics:
     - type: mean_reward
-      value: 19800.00 +/- 12695.89
+      value: 59120.00 +/- 11022.48
       name: mean_reward
       verified: false
 ---
@@ -30,15 +30,128 @@ Documentation for how to use Sample-Factory can be found at https://www.samplefa
 
 After installing Sample-Factory, download the model with:
 ```
-python -m sample_factory.huggingface.load_from_hub -r MattStammers/appo-atari_defender
+python -m sample_factory.huggingface.load_from_hub -r MattStammers/APPO-atari_defender
 ```
 
     
+## About the Model
+
+This model as with all the others in the benchmarks was trained initially asynchronously un-seeded to 10 million steps for the purposes of setting a sample factory async baseline for this model on this environment but only 3/57 made it. 
+
+The aim is to reach state-of-the-art (SOTA) performance on each atari environment. I will flag the models with SOTA when they reach at or near these levels. 
+
+The hyperparameters used in the model are the ones I have pushed to my fork of sample-factory: https://github.com/MattStammers/sample-factory. Given that https://huggingface.co/edbeeching has kindly shared his.
+I saved time and energy by using many of his tuned hyperparameters to maximise performance. However, he used 2 billion training steps. I have started as explained above at 10 million then moved to 100m to see how performance goes:
+```
+hyperparameters =  {
+  "device": "gpu",
+  "seed": 1234,
+  "num_policies": 2,
+  "async_rl": true,
+  "serial_mode": false,
+  "batched_sampling": true,
+  "num_batches_to_accumulate": 2,
+  "worker_num_splits": 1,
+  "policy_workers_per_policy": 1,
+  "max_policy_lag": 1000,
+  "num_workers": 16,
+  "num_envs_per_worker": 2,
+  "batch_size": 1024,
+  "num_batches_per_epoch": 8,
+  "num_epochs": 4,
+  "rollout": 128,
+  "recurrence": 1,
+  "shuffle_minibatches": false,
+  "gamma": 0.99,
+  "reward_scale": 1.0,
+  "reward_clip": 1000.0,
+  "value_bootstrap": false,
+  "normalize_returns": true,
+  "exploration_loss_coeff": 0.0004677351413,
+  "value_loss_coeff": 0.5,
+  "kl_loss_coeff": 0.0,
+  "exploration_loss": "entropy",
+  "gae_lambda": 0.95,
+  "ppo_clip_ratio": 0.1,
+  "ppo_clip_value": 1.0,
+  "with_vtrace": false,
+  "vtrace_rho": 1.0,
+  "vtrace_c": 1.0,
+  "optimizer": "adam",
+  "adam_eps": 1e-05,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "max_grad_norm": 0.0,
+  "learning_rate": 0.0003033891184,
+  "lr_schedule": "linear_decay",
+  "lr_schedule_kl_threshold": 0.008,
+  "lr_adaptive_min": 1e-06,
+  "lr_adaptive_max": 0.01,
+  "obs_subtract_mean": 0.0,
+  "obs_scale": 255.0,
+  "normalize_input": true,
+  "normalize_input_keys": [
+    "obs"
+  ],
+  "decorrelate_experience_max_seconds": 0,
+  "decorrelate_envs_on_one_worker": true,
+  "actor_worker_gpus": [],
+  "set_workers_cpu_affinity": true,
+  "force_envs_single_thread": false,
+  "default_niceness": 0,
+  "log_to_file": true,
+  "experiment_summaries_interval": 3,
+  "flush_summaries_interval": 30,
+  "stats_avg": 100,
+  "summaries_use_frameskip": true,
+  "heartbeat_interval": 10,
+  "heartbeat_reporting_interval": 60,
+  "train_for_env_steps": 100000000,
+  "train_for_seconds": 10000000000,
+  "save_every_sec": 120,
+  "keep_checkpoints": 2,
+  "load_checkpoint_kind": "latest",
+  "save_milestones_sec": 1200,
+  "save_best_every_sec": 5,
+  "save_best_metric": "reward",
+  "save_best_after": 100000,
+  "benchmark": false,
+  "encoder_mlp_layers": [
+    512,
+    512
+  ],
+  "encoder_conv_architecture": "convnet_atari",
+  "encoder_conv_mlp_layers": [
+    512
+  ],
+  "use_rnn": false,
+  "rnn_size": 512,
+  "rnn_type": "gru",
+  "rnn_num_layers": 1,
+  "decoder_mlp_layers": [],
+  "nonlinearity": "relu",
+  "policy_initialization": "orthogonal",
+  "policy_init_gain": 1.0,
+  "actor_critic_share_weights": true,
+  "adaptive_stddev": false,
+  "continuous_tanh_scale": 0.0,
+  "initial_stddev": 1.0,
+  "use_env_info_cache": false,
+  "env_gpu_actions": false,
+  "env_gpu_observations": true,
+  "env_frameskip": 4,
+  "env_framestack": 4,
+  }
+
+  ```
+
+
+    
 ## Using the model
 
 To run the model after download, use the `enjoy` script corresponding to this environment:
 ```
-python -m sf_examples.atari.enjoy_atari --algo=APPO --env=atari_defender --train_dir=./train_dir --experiment=appo-atari_defender
+python -m sf_examples.atari.enjoy_atari --algo=APPO --env=atari_defender --train_dir=./train_dir --experiment=APPO-atari_defender
 ```
 
 
@@ -49,7 +162,7 @@ See https://www.samplefactory.dev/10-huggingface/huggingface/ for more details
 
 To continue training with this model, use the `train` script corresponding to this environment:
 ```
-python -m sf_examples.atari.train_atari --algo=APPO --env=atari_defender --train_dir=./train_dir --experiment=appo-atari_defender --restart_behavior=resume --train_for_env_steps=10000000000
+python -m sf_examples.atari.train_atari --algo=APPO --env=atari_defender --train_dir=./train_dir --experiment=APPO-atari_defender --restart_behavior=resume --train_for_env_steps=10000000000
 ```
 
 Note, you may have to adjust `--train_for_env_steps` to a suitably high number as the experiment will resume at the number of steps it concluded at.

checkpoint_p0/best_000083552_85557248_reward_47.340.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2522fa16cf6e593a559780b4244df8d484fee9e89a4f848921cf2e47b88297f
+size 20795763

checkpoint_p0/checkpoint_000097184_99516416.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88904d8fe0e1b97d4c6aa3eba99447f9b8e700efebdafcdcd40671524d006640
+size 20796099

checkpoint_p0/checkpoint_000097888_100237312.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbdd3e090e3cf0f1258218e2d22eb3e2bf0353db75570fe87d627320ffdec3d1
+size 20796099

checkpoint_p0/milestones/checkpoint_000008384_8585216.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0948eb3fc9ba7e87c61e2e8a8e99dbf7968cfca9865c4bbb4049b7b60e051716
+size 20796955

checkpoint_p0/milestones/checkpoint_000016928_17334272.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15963821cbd3bac492e0953d7fce70c2e77d797d2c5378e02d097c8ab8b2518b
+size 20797011

checkpoint_p0/milestones/checkpoint_000025440_26050560.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8eb0f438ce32de94a61c62f4b8ca8ce82c4bf0410cd55520ce836ca2154f3b5f
+size 20797011

checkpoint_p0/milestones/checkpoint_000033984_34799616.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:909960e4896330d15cf1f5661f10af2dea59646fcefaaef94b594a0cf6913fb8
+size 20797011

checkpoint_p0/milestones/checkpoint_000042496_43515904.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b60eaa394fa52dc04dca6a7fcdb915a8f9bce8892d9cddf5cd22197202d75d97
+size 20797011

checkpoint_p0/milestones/checkpoint_000051040_52264960.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:842f16fe94ce4d1acf24cf884b955ccea5399ca15ff66f4f98e05cfa8e8402e1
+size 20797011

checkpoint_p0/milestones/checkpoint_000059584_61014016.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d443edd155b87a85ebdf6930ba043c12cb89c93b333fd40412801b4b065aa58
+size 20797011

checkpoint_p0/milestones/checkpoint_000068128_69763072.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4ba081dea7f10ef11d4e1d1cfe5d523e556571b083bc400d3b107df0ee6c842
+size 20797011

checkpoint_p0/milestones/checkpoint_000076640_78479360.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c413ef88fdb4ffcdaab55cb33f323094d173d548859b60428bf8d2b36bb8d79
+size 20797011

checkpoint_p0/milestones/checkpoint_000085216_87261184.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbcadc0f751c33771dcc811b75c1be7308483151b8d2c2c9ab768c87ceec30f0
+size 20797011

checkpoint_p0/milestones/checkpoint_000093760_96010240.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6ed867edd2c359fe44ebd219ec086ef3c297946a1973384d8810422e54b01c8
+size 20797011

checkpoint_p1/best_000085280_87326720_reward_44.510.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08b7fac119a29fbb7a63f5469f73fd169869d4719bb057dafe28897dbda6ea0a
+size 20795763

checkpoint_p1/checkpoint_000096992_99319808.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:487659e798d7edc473607b56bba8bcd72373862eada692308317d1a14dd216f5
+size 20796099

checkpoint_p1/checkpoint_000097664_100007936.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:beeb07ced01040601d52d50e46e913cf11332f9092ac0808a875a3c978389cd3
+size 20796099

checkpoint_p1/milestones/checkpoint_000008384_8585216.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c236090c6fa81532a75e7d9fc0a8ca45de13e8564ce36fccf237060c9b629588
+size 20796955

checkpoint_p1/milestones/checkpoint_000016896_17301504.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5ad8febbe059e00fce51624af96ef488dda6f5c8356428a0d2810fb47e1a83f
+size 20797011

checkpoint_p1/milestones/checkpoint_000025440_26050560.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0747753b2770c5d106a019e4680b00f6ee078d4575a351b4b50de5f076084582
+size 20797011

checkpoint_p1/milestones/checkpoint_000033920_34734080.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de511de64ec9194b84fcf5a42b1d261f966faa5f0136ae94da98bbb84671fd70
+size 20797011

checkpoint_p1/milestones/checkpoint_000042464_43483136.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8492acfdd3f83ad6463ae3738d97fd02bfc249f6ae60647846c022948aa54da
+size 20797011

checkpoint_p1/milestones/checkpoint_000050976_52199424.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4d5ed344b9137e5239550edf24f675fd4ddc7a773d9ed4dae7d39fc55cfd3b1
+size 20797011

checkpoint_p1/milestones/checkpoint_000059488_60915712.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52ecc39342a8929c9077b80ecff7a49130edf58eb6ac909402374514e6a72cfd
+size 20797011

checkpoint_p1/milestones/checkpoint_000068000_69632000.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dc0f4e4ee881a7e77fb8027430175b4bedc8c4ae5d3042bca8f98faa0ac08b1
+size 20797011

checkpoint_p1/milestones/checkpoint_000076512_78348288.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32dd9d21b50657e744dea67553e67bd0a5981c5281ff533512356f694559607b
+size 20797011

checkpoint_p1/milestones/checkpoint_000085024_87064576.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac08c34b22e57570e594e447bef56db18e41d24c9760273b281cb0e5dbf23df8
+size 20797011

checkpoint_p1/milestones/checkpoint_000093568_95813632.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e2c66b051591a9883a32a35105b55dab5fccccfbf86d49847d95a2001ecd8e5
+size 20797011

config.json

@@ -2,23 +2,23 @@
   "help": false,
   "algo": "APPO",
   "env": "atari_defender",
-  "experiment": "atari_defender",
+  "experiment": "atari_defender_APPO",
   "train_dir": "./train_atari",
-  "restart_behavior": "resume",
+  "restart_behavior": "restart",
   "device": "gpu",
-  "seed": null,
+  "seed": 1234,
   "num_policies": 2,
-  "async_rl": false,
+  "async_rl": true,
   "serial_mode": false,
-  "batched_sampling": false,
+  "batched_sampling": true,
   "num_batches_to_accumulate": 2,
   "worker_num_splits": 1,
   "policy_workers_per_policy": 1,
   "max_policy_lag": 1000,
-  "num_workers": 8,
-  "num_envs_per_worker": 1,
-  "batch_size": 256,
-  "num_batches_per_epoch": 4,
+  "num_workers": 16,
+  "num_envs_per_worker": 2,
+  "batch_size": 1024,
+  "num_batches_per_epoch": 8,
   "num_epochs": 4,
   "rollout": 128,
   "recurrence": 1,
@@ -28,7 +28,7 @@
   "reward_clip": 1000.0,
   "value_bootstrap": false,
   "normalize_returns": true,
-  "exploration_loss_coeff": 0.01,
+  "exploration_loss_coeff": 0.0004677351413,
   "value_loss_coeff": 0.5,
   "kl_loss_coeff": 0.0,
   "exploration_loss": "entropy",
@@ -42,8 +42,8 @@
   "adam_eps": 1e-05,
   "adam_beta1": 0.9,
   "adam_beta2": 0.999,
-  "max_grad_norm": 0.5,
-  "learning_rate": 0.00025,
+  "max_grad_norm": 0.0,
+  "learning_rate": 0.0003033891184,
   "lr_schedule": "linear_decay",
   "lr_schedule_kl_threshold": 0.008,
   "lr_adaptive_min": 1e-06,
@@ -51,7 +51,9 @@
   "obs_subtract_mean": 0.0,
   "obs_scale": 255.0,
   "normalize_input": true,
-  "normalize_input_keys": null,
+  "normalize_input_keys": [
+    "obs"
+  ],
   "decorrelate_experience_max_seconds": 0,
   "decorrelate_envs_on_one_worker": true,
   "actor_worker_gpus": [],
@@ -63,14 +65,14 @@
   "flush_summaries_interval": 30,
   "stats_avg": 100,
   "summaries_use_frameskip": true,
-  "heartbeat_interval": 20,
-  "heartbeat_reporting_interval": 180,
-  "train_for_env_steps": 10000000,
+  "heartbeat_interval": 10,
+  "heartbeat_reporting_interval": 60,
+  "train_for_env_steps": 100000000,
   "train_for_seconds": 10000000000,
   "save_every_sec": 120,
   "keep_checkpoints": 2,
   "load_checkpoint_kind": "latest",
-  "save_milestones_sec": -1,
+  "save_milestones_sec": 1200,
   "save_best_every_sec": 5,
   "save_best_metric": "reward",
   "save_best_after": 100000,
@@ -104,7 +106,7 @@
   "use_record_episode_statistics": true,
   "with_wandb": true,
   "wandb_user": "matt-stammers",
-  "wandb_project": "atari",
+  "wandb_project": "atari_APPO",
   "wandb_group": "atari_defender",
   "wandb_job_type": "SF",
   "wandb_tags": [
@@ -122,16 +124,32 @@
   "pbt_target_objective": "true_objective",
   "pbt_perturb_min": 1.1,
   "pbt_perturb_max": 1.5,
-  "command_line": "--algo=APPO --env=atari_defender --experiment=atari_defender --num_policies=2 --train_dir=./train_atari --with_wandb=true --wandb_user=matt-stammers --wandb_project=atari --wandb_group=atari_defender --wandb_job_type=SF --wandb_tags=atari",
+  "command_line": "--algo=APPO --env=atari_defender --experiment=atari_defender_APPO --num_policies=2 --restart_behavior=restart --train_dir=./train_atari --train_for_env_steps=100000000 --seed=1234 --num_workers=16 --num_envs_per_worker=2 --num_batches_per_epoch=8 --async_rl=true --batched_sampling=true --batch_size=1024 --max_grad_norm=0 --learning_rate=0.0003033891184 --heartbeat_interval=10 --heartbeat_reporting_interval=60 --save_milestones_sec=1200 --num_epochs=4 --exploration_loss_coeff=0.0004677351413 --with_wandb=true --wandb_user=matt-stammers --wandb_project=atari_APPO --wandb_group=atari_defender --wandb_job_type=SF --wandb_tags=atari",
   "cli_args": {
     "algo": "APPO",
     "env": "atari_defender",
-    "experiment": "atari_defender",
+    "experiment": "atari_defender_APPO",
     "train_dir": "./train_atari",
+    "restart_behavior": "restart",
+    "seed": 1234,
     "num_policies": 2,
+    "async_rl": true,
+    "batched_sampling": true,
+    "num_workers": 16,
+    "num_envs_per_worker": 2,
+    "batch_size": 1024,
+    "num_batches_per_epoch": 8,
+    "num_epochs": 4,
+    "exploration_loss_coeff": 0.0004677351413,
+    "max_grad_norm": 0.0,
+    "learning_rate": 0.0003033891184,
+    "heartbeat_interval": 10,
+    "heartbeat_reporting_interval": 60,
+    "train_for_env_steps": 100000000,
+    "save_milestones_sec": 1200,
     "with_wandb": true,
     "wandb_user": "matt-stammers",
-    "wandb_project": "atari",
+    "wandb_project": "atari_APPO",
     "wandb_group": "atari_defender",
     "wandb_job_type": "SF",
     "wandb_tags": [
@@ -140,5 +158,5 @@
   },
   "git_hash": "5fff97c2f535da5987d358cdbe6927cccd43621e",
   "git_repo_name": "not a git repository",
-  "wandb_unique_id": "atari_defender_20230926_015055_282929"
+  "wandb_unique_id": "atari_defender_APPO_20231010_124028_375230"
 }

git.diff

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:28f3fe545f35e5db1e419bf1e198c0bc5ef89459d55f59e2d06d3c108907e732
-size 14403396
+oid sha256:3357904f421d3f4924836316b1741bf64d5dd0e807d5e80ac07059b4c52a7008
+size 14426734

replay.mp4

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:12eca1bf1f720989a1c5ea9cc4c46793c9167c71fda15642807da03651155251
-size 5259758
+oid sha256:6a41cce53440aafd1a1dba87af7ed9c53e5b7e688d1deaa3cf3f8378c70d912f
+size 10526907