Upload folder using huggingface_hub

.summary/0/events.out.tfevents.1697498132.rhmmedcatt-proliant-ml350-gen10

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:467de16e343c0c23885712ef947983f6b9e0eef3c8e435cd147144889e2404ce
+size 7770753

.summary/1/events.out.tfevents.1697498132.rhmmedcatt-proliant-ml350-gen10

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df40ba2eefe776a0ccda839756aeff0c512481c8871e90ff83c87dca6c10cc3e
+size 4497164

README.md

@@ -15,7 +15,7 @@ model-index:
       type: atari_wizardofwor
     metrics:
     - type: mean_reward
-      value: 2590.00 +/- 1905.49
+      value: 13670.00 +/- 8376.40
       name: mean_reward
       verified: false
 ---
@@ -30,15 +30,128 @@ Documentation for how to use Sample-Factory can be found at https://www.samplefa
 
 After installing Sample-Factory, download the model with:
 ```
-python -m sample_factory.huggingface.load_from_hub -r MattStammers/appo-atari_wizardofwor
+python -m sample_factory.huggingface.load_from_hub -r MattStammers/APPO-atari_wizardofwor
 ```
 
     
+## About the Model
+
+This model as with all the others in the benchmarks was trained initially asynchronously un-seeded to 10 million steps for the purposes of setting a sample factory async baseline for this model on this environment but only 3/57 made it. 
+
+The aim is to reach state-of-the-art (SOTA) performance on each atari environment. I will flag the models with SOTA when they reach at or near these levels. 
+
+The hyperparameters used in the model are the ones I have pushed to my fork of sample-factory: https://github.com/MattStammers/sample-factory. Given that https://huggingface.co/edbeeching has kindly shared his.
+I saved time and energy by using many of his tuned hyperparameters to maximise performance. However, he used 2 billion training steps. I have started as explained above at 10 million then moved to 100m to see how performance goes:
+```
+hyperparameters =  {
+  "device": "gpu",
+  "seed": 1234,
+  "num_policies": 2,
+  "async_rl": true,
+  "serial_mode": false,
+  "batched_sampling": true,
+  "num_batches_to_accumulate": 2,
+  "worker_num_splits": 1,
+  "policy_workers_per_policy": 1,
+  "max_policy_lag": 1000,
+  "num_workers": 16,
+  "num_envs_per_worker": 2,
+  "batch_size": 1024,
+  "num_batches_per_epoch": 8,
+  "num_epochs": 4,
+  "rollout": 128,
+  "recurrence": 1,
+  "shuffle_minibatches": false,
+  "gamma": 0.99,
+  "reward_scale": 1.0,
+  "reward_clip": 1000.0,
+  "value_bootstrap": false,
+  "normalize_returns": true,
+  "exploration_loss_coeff": 0.0004677351413,
+  "value_loss_coeff": 0.5,
+  "kl_loss_coeff": 0.0,
+  "exploration_loss": "entropy",
+  "gae_lambda": 0.95,
+  "ppo_clip_ratio": 0.1,
+  "ppo_clip_value": 1.0,
+  "with_vtrace": false,
+  "vtrace_rho": 1.0,
+  "vtrace_c": 1.0,
+  "optimizer": "adam",
+  "adam_eps": 1e-05,
+  "adam_beta1": 0.9,
+  "adam_beta2": 0.999,
+  "max_grad_norm": 0.0,
+  "learning_rate": 0.0003033891184,
+  "lr_schedule": "linear_decay",
+  "lr_schedule_kl_threshold": 0.008,
+  "lr_adaptive_min": 1e-06,
+  "lr_adaptive_max": 0.01,
+  "obs_subtract_mean": 0.0,
+  "obs_scale": 255.0,
+  "normalize_input": true,
+  "normalize_input_keys": [
+    "obs"
+  ],
+  "decorrelate_experience_max_seconds": 0,
+  "decorrelate_envs_on_one_worker": true,
+  "actor_worker_gpus": [],
+  "set_workers_cpu_affinity": true,
+  "force_envs_single_thread": false,
+  "default_niceness": 0,
+  "log_to_file": true,
+  "experiment_summaries_interval": 3,
+  "flush_summaries_interval": 30,
+  "stats_avg": 100,
+  "summaries_use_frameskip": true,
+  "heartbeat_interval": 10,
+  "heartbeat_reporting_interval": 60,
+  "train_for_env_steps": 100000000,
+  "train_for_seconds": 10000000000,
+  "save_every_sec": 120,
+  "keep_checkpoints": 2,
+  "load_checkpoint_kind": "latest",
+  "save_milestones_sec": 1200,
+  "save_best_every_sec": 5,
+  "save_best_metric": "reward",
+  "save_best_after": 100000,
+  "benchmark": false,
+  "encoder_mlp_layers": [
+    512,
+    512
+  ],
+  "encoder_conv_architecture": "convnet_atari",
+  "encoder_conv_mlp_layers": [
+    512
+  ],
+  "use_rnn": false,
+  "rnn_size": 512,
+  "rnn_type": "gru",
+  "rnn_num_layers": 1,
+  "decoder_mlp_layers": [],
+  "nonlinearity": "relu",
+  "policy_initialization": "orthogonal",
+  "policy_init_gain": 1.0,
+  "actor_critic_share_weights": true,
+  "adaptive_stddev": false,
+  "continuous_tanh_scale": 0.0,
+  "initial_stddev": 1.0,
+  "use_env_info_cache": false,
+  "env_gpu_actions": false,
+  "env_gpu_observations": true,
+  "env_frameskip": 4,
+  "env_framestack": 4,
+  }
+
+  ```
+
+
+    
 ## Using the model
 
 To run the model after download, use the `enjoy` script corresponding to this environment:
 ```
-python -m sf_examples.atari.enjoy_atari --algo=APPO --env=atari_wizardofwor --train_dir=./train_dir --experiment=appo-atari_wizardofwor
+python -m sf_examples.atari.enjoy_atari --algo=APPO --env=atari_wizardofwor --train_dir=./train_dir --experiment=APPO-atari_wizardofwor
 ```
 
 
@@ -49,7 +162,7 @@ See https://www.samplefactory.dev/10-huggingface/huggingface/ for more details
 
 To continue training with this model, use the `train` script corresponding to this environment:
 ```
-python -m sf_examples.atari.train_atari --algo=APPO --env=atari_wizardofwor --train_dir=./train_dir --experiment=appo-atari_wizardofwor --restart_behavior=resume --train_for_env_steps=10000000000
+python -m sf_examples.atari.train_atari --algo=APPO --env=atari_wizardofwor --train_dir=./train_dir --experiment=APPO-atari_wizardofwor --restart_behavior=resume --train_for_env_steps=10000000000
 ```
 
 Note, you may have to adjust `--train_for_env_steps` to a suitably high number as the experiment will resume at the number of steps it concluded at.

checkpoint_p0/best_000098304_100663296_reward_13.780.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f47cba3c4d534e3e7b1f3a20cf16bde285a286813b7bf0e790529cce76c097e9
+size 20746419

checkpoint_p0/checkpoint_000098144_100499456.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99c5aacb4933212b31a8226c15c24b36a1e315e021179d08ddde18b8f6469987
+size 20746755

checkpoint_p0/checkpoint_000098400_100761600.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0154568803f3795594d3ac9eeab8d47062e9492a1ce3c19a481f491287f1b69a
+size 20746755

checkpoint_p0/milestones/checkpoint_000008192_8388608.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af80002b428900a66d7777906c1b6c8d37ca942e87b5510c5b6780364a93d46b
+size 20747611

checkpoint_p0/milestones/checkpoint_000016512_16908288.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09867956d2d2885ba29a9dc30821b23e905b80afc44496987c750223b8a76a9b
+size 20747667

checkpoint_p0/milestones/checkpoint_000024832_25427968.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c74e646f375ee80c78a0223f70cc92496431c7f0f3e83147f4607c4bf30a95f
+size 20747667

checkpoint_p0/milestones/checkpoint_000033152_33947648.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75cd62958fbb1ef41bbac4a0855453b960ba223b3f22ed5971263cb2fc46103b
+size 20747667

checkpoint_p0/milestones/checkpoint_000041504_42500096.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1d8ec8e982ec0b9edcab98e5dc6a2137c9faf9878c169c824c2edc8a6f9e981
+size 20747667

checkpoint_p0/milestones/checkpoint_000049824_51019776.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45a1466633cdbd158bcc784a433164b39728f00df12248b54e4faf124d266592
+size 20747667

checkpoint_p0/milestones/checkpoint_000058144_59539456.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f07122338192b7f437ce17d03a2cc6f30ab45926ef249c1892b9443f7c2712c1
+size 20747667

checkpoint_p0/milestones/checkpoint_000066464_68059136.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b905e6db5a4ffea6c45925811e5893a362b8efb521fcc65b287c4ee608a705c2
+size 20747667

checkpoint_p0/milestones/checkpoint_000074816_76611584.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96815dc45add32a209e3b8072a0d864496ae0aa05d20cb0c9d2b97ac5e2a9ca1
+size 20747667

checkpoint_p0/milestones/checkpoint_000083136_85131264.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78137e3e4156d636c77ec9af6ba793ed4dac4526d4b5018adcff487b4f0e569e
+size 20747667

checkpoint_p0/milestones/checkpoint_000091488_93683712.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e130dd6f4ce45d32e4f79f603167493591197268ad52585f34d3b1ccfda8642
+size 20747667

checkpoint_p1/best_000092320_94535680_reward_13.220.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f729968942bca223f2a36bb94b9de45923ad872bbf366bd5b23365878d706b6b
+size 20746419

checkpoint_p1/checkpoint_000097440_99778560.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6745a671697f2f45217531e7db6bb70259885644182257f7a274c5c409cae614
+size 20746755

checkpoint_p1/checkpoint_000097664_100007936.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f21b9e3f9c44ef60fbc8082db4dc34ca8a26d790fba3ecb8c9b694662af40aa
+size 20746755

checkpoint_p1/milestones/checkpoint_000008128_8323072.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d55a8a073dbe11f85ceb74b76b4b3f0a2ba9346a3b72d376d8800950d0b4b5f0
+size 20747611

checkpoint_p1/milestones/checkpoint_000016384_16777216.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27ed4103cb4a349eabd5016a31537bf01eda54ac2414f21c9be7dc07e1657f6b
+size 20747667

checkpoint_p1/milestones/checkpoint_000024672_25264128.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8a7b9aca4493597500bbf0cd67903393dacb0d108d9ec0392a52d19c9992ee6
+size 20747667

checkpoint_p1/milestones/checkpoint_000032928_33718272.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11c6b5d52ebb37009e2965de7b97bac268f2effe78e154bdd6d54ad190b4f416
+size 20747667

checkpoint_p1/milestones/checkpoint_000041184_42172416.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5eedde6701c4b4b4a8439e2bfeffaf5f6662c02792060de98c0bba5145b40416
+size 20747667

checkpoint_p1/milestones/checkpoint_000049440_50626560.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6dd2240b3fb954a1a494954015916769dd049724ec3952432eb249bb8ea03c1
+size 20747667

checkpoint_p1/milestones/checkpoint_000057728_59113472.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ff542f3d441b5a703fa471eb3239ebcfb8ede4de669d365b489ea9b097aac93
+size 20747667

checkpoint_p1/milestones/checkpoint_000065984_67567616.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f31f0cc237dbda9c5d5b2833a573c45c7a8df843027f4309b811307050a72ad
+size 20747667

checkpoint_p1/milestones/checkpoint_000074272_76054528.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc5380c2be153627fa313a06b0f0376baead655a3d7bf4333465d3c00d0beeef
+size 20747667

checkpoint_p1/milestones/checkpoint_000082528_84508672.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a887e7bf96ee1c2a7986d1ed48244ef109fdbfce343f572a588a79511eb5b86a
+size 20747667

checkpoint_p1/milestones/checkpoint_000090816_92995584.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6daa7d3140b2c79cbd84ac3436b05706542aa7ad61230eab62717376ff6a3cd8
+size 20747667

config.json

@@ -2,23 +2,23 @@
   "help": false,
   "algo": "APPO",
   "env": "atari_wizardofwor",
-  "experiment": "atari_wizardofwor",
+  "experiment": "atari_wizardofwor_APPO",
   "train_dir": "./train_atari",
-  "restart_behavior": "resume",
+  "restart_behavior": "restart",
   "device": "gpu",
-  "seed": null,
+  "seed": 1234,
   "num_policies": 2,
-  "async_rl": false,
+  "async_rl": true,
   "serial_mode": false,
-  "batched_sampling": false,
+  "batched_sampling": true,
   "num_batches_to_accumulate": 2,
   "worker_num_splits": 1,
   "policy_workers_per_policy": 1,
   "max_policy_lag": 1000,
-  "num_workers": 8,
-  "num_envs_per_worker": 1,
-  "batch_size": 256,
-  "num_batches_per_epoch": 4,
+  "num_workers": 16,
+  "num_envs_per_worker": 2,
+  "batch_size": 1024,
+  "num_batches_per_epoch": 8,
   "num_epochs": 4,
   "rollout": 128,
   "recurrence": 1,
@@ -28,7 +28,7 @@
   "reward_clip": 1000.0,
   "value_bootstrap": false,
   "normalize_returns": true,
-  "exploration_loss_coeff": 0.01,
+  "exploration_loss_coeff": 0.0004677351413,
   "value_loss_coeff": 0.5,
   "kl_loss_coeff": 0.0,
   "exploration_loss": "entropy",
@@ -42,8 +42,8 @@
   "adam_eps": 1e-05,
   "adam_beta1": 0.9,
   "adam_beta2": 0.999,
-  "max_grad_norm": 0.5,
-  "learning_rate": 0.00025,
+  "max_grad_norm": 0.0,
+  "learning_rate": 0.0003033891184,
   "lr_schedule": "linear_decay",
   "lr_schedule_kl_threshold": 0.008,
   "lr_adaptive_min": 1e-06,
@@ -51,7 +51,9 @@
   "obs_subtract_mean": 0.0,
   "obs_scale": 255.0,
   "normalize_input": true,
-  "normalize_input_keys": null,
+  "normalize_input_keys": [
+    "obs"
+  ],
   "decorrelate_experience_max_seconds": 0,
   "decorrelate_envs_on_one_worker": true,
   "actor_worker_gpus": [],
@@ -63,14 +65,14 @@
   "flush_summaries_interval": 30,
   "stats_avg": 100,
   "summaries_use_frameskip": true,
-  "heartbeat_interval": 20,
-  "heartbeat_reporting_interval": 180,
-  "train_for_env_steps": 10000000,
+  "heartbeat_interval": 10,
+  "heartbeat_reporting_interval": 60,
+  "train_for_env_steps": 100000000,
   "train_for_seconds": 10000000000,
   "save_every_sec": 120,
   "keep_checkpoints": 2,
   "load_checkpoint_kind": "latest",
-  "save_milestones_sec": -1,
+  "save_milestones_sec": 1200,
   "save_best_every_sec": 5,
   "save_best_metric": "reward",
   "save_best_after": 100000,
@@ -104,7 +106,7 @@
   "use_record_episode_statistics": true,
   "with_wandb": true,
   "wandb_user": "matt-stammers",
-  "wandb_project": "atari",
+  "wandb_project": "atari_APPO",
   "wandb_group": "atari_wizardofwor",
   "wandb_job_type": "SF",
   "wandb_tags": [
@@ -122,16 +124,32 @@
   "pbt_target_objective": "true_objective",
   "pbt_perturb_min": 1.1,
   "pbt_perturb_max": 1.5,
-  "command_line": "--algo=APPO --env=atari_wizardofwor --experiment=atari_wizardofwor --num_policies=2 --train_dir=./train_atari --with_wandb=true --wandb_user=matt-stammers --wandb_project=atari --wandb_group=atari_wizardofwor --wandb_job_type=SF --wandb_tags=atari",
+  "command_line": "--algo=APPO --env=atari_wizardofwor --experiment=atari_wizardofwor_APPO --num_policies=2 --restart_behavior=restart --train_dir=./train_atari --train_for_env_steps=100000000 --seed=1234 --num_workers=16 --num_envs_per_worker=2 --num_batches_per_epoch=8 --async_rl=true --batched_sampling=true --batch_size=1024 --max_grad_norm=0 --learning_rate=0.0003033891184 --heartbeat_interval=10 --heartbeat_reporting_interval=60 --save_milestones_sec=1200 --num_epochs=4 --exploration_loss_coeff=0.0004677351413 --with_wandb=true --wandb_user=matt-stammers --wandb_project=atari_APPO --wandb_group=atari_wizardofwor --wandb_job_type=SF --wandb_tags=atari",
   "cli_args": {
     "algo": "APPO",
     "env": "atari_wizardofwor",
-    "experiment": "atari_wizardofwor",
+    "experiment": "atari_wizardofwor_APPO",
     "train_dir": "./train_atari",
+    "restart_behavior": "restart",
+    "seed": 1234,
     "num_policies": 2,
+    "async_rl": true,
+    "batched_sampling": true,
+    "num_workers": 16,
+    "num_envs_per_worker": 2,
+    "batch_size": 1024,
+    "num_batches_per_epoch": 8,
+    "num_epochs": 4,
+    "exploration_loss_coeff": 0.0004677351413,
+    "max_grad_norm": 0.0,
+    "learning_rate": 0.0003033891184,
+    "heartbeat_interval": 10,
+    "heartbeat_reporting_interval": 60,
+    "train_for_env_steps": 100000000,
+    "save_milestones_sec": 1200,
     "with_wandb": true,
     "wandb_user": "matt-stammers",
-    "wandb_project": "atari",
+    "wandb_project": "atari_APPO",
     "wandb_group": "atari_wizardofwor",
     "wandb_job_type": "SF",
     "wandb_tags": [
@@ -140,5 +158,5 @@
   },
   "git_hash": "5fff97c2f535da5987d358cdbe6927cccd43621e",
   "git_repo_name": "not a git repository",
-  "wandb_unique_id": "atari_wizardofwor_20230927_154551_007232"
+  "wandb_unique_id": "atari_wizardofwor_APPO_20231017_001529_368527"
 }

git.diff

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:28f3fe545f35e5db1e419bf1e198c0bc5ef89459d55f59e2d06d3c108907e732
-size 14403396
+oid sha256:3357904f421d3f4924836316b1741bf64d5dd0e807d5e80ac07059b4c52a7008
+size 14426734

replay.mp4

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:64e0030ab1c5a4a651eb95644f0b49c3b47e2c9b3a8ab9631d8519ca5e66e073
-size 2271554
+oid sha256:e1dd6a6075b509c5969f83a03b5642ad4a4e18d8ea8a880393fbbe36310a6810
+size 3366316