HF RL course

Pyramids.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:90126fab66e521f486d70ee8130d80a09a3768bcbcce67233a39d11d4f690195
+oid sha256:72661ba1c7ed0a078011f9768a5356948417588ce83d20010e191e39a58f0da6
 size 3520295

Pyramids/Pyramids-1499926.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72661ba1c7ed0a078011f9768a5356948417588ce83d20010e191e39a58f0da6
+size 3520295

Pyramids/Pyramids-1499926.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fc4f70f6791c2b5b7082685cbbd92083b55e444aa61ba568b0bdc6d1eba8ac9
+size 21266278

Pyramids/Pyramids-1500054.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72661ba1c7ed0a078011f9768a5356948417588ce83d20010e191e39a58f0da6
+size 3520295

Pyramids/Pyramids-1500054.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2e93d8fff3c37da49274430e118aed8c80cb25dd1b54e7dbb1dc830ac1bf2be
+size 21266278

Pyramids/checkpoint.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df1fa5d0d52628757809e9e617e8404dd2fe2ff446b7e374d4e1f8c0c060b7ae
+oid sha256:f2e93d8fff3c37da49274430e118aed8c80cb25dd1b54e7dbb1dc830ac1bf2be
 size 21266278

Pyramids/events.out.tfevents.1683876347.3f9412ea1207.14666.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e32b887eb869c3fc2f363fd335b268ad44dd6c82fa2bf28cc1416e060c84d9e7
+size 168055

config.json

@@ -1 +1 @@
-{"default_settings": null, "behaviors": {"Pyramids": {"trainer_type": "ppo", "hyperparameters": {"batch_size": 128, "buffer_size": 10240, "learning_rate": 0.0003, "beta": 0.01, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 4, "shared_critic": false, "learning_rate_schedule": "linear", "beta_schedule": "linear", "epsilon_schedule": "linear"}, "checkpoint_interval": 500000, "network_settings": {"normalize": false, "hidden_units": 512, "num_layers": 4, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.99, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}, "rnd": {"gamma": 0.99, "strength": 0.01, "network_settings": {"normalize": false, "hidden_units": 64, "num_layers": 3, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "learning_rate": 0.0001, "encoding_size": null}}, "init_path": null, "keep_checkpoints": 5, "even_checkpoints": false, "max_steps": 1000000, "time_horizon": 128, "summary_freq": 30000, "threaded": false, "self_play": null, "behavioral_cloning": null}}, "env_settings": {"env_path": "./training-envs-executables/linux/Pyramids/Pyramids", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true}, "environment_parameters": null, "checkpoint_settings": {"run_id": "Pyramids Training", "initialize_from": null, "load_model": false, "resume": false, "force": true, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}
+{"default_settings": null, "behaviors": {"Pyramids": {"trainer_type": "ppo", "hyperparameters": {"batch_size": 128, "buffer_size": 10240, "learning_rate": 0.0003, "beta": 0.01, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 4, "shared_critic": false, "learning_rate_schedule": "linear", "beta_schedule": "linear", "epsilon_schedule": "linear"}, "checkpoint_interval": 500000, "network_settings": {"normalize": false, "hidden_units": 512, "num_layers": 4, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.99, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}, "rnd": {"gamma": 0.99, "strength": 0.01, "network_settings": {"normalize": false, "hidden_units": 64, "num_layers": 3, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "learning_rate": 0.0001, "encoding_size": null}}, "init_path": null, "keep_checkpoints": 5, "even_checkpoints": false, "max_steps": 1500000, "time_horizon": 128, "summary_freq": 30000, "threaded": false, "self_play": null, "behavioral_cloning": null}}, "env_settings": {"env_path": "./training-envs-executables/linux/Pyramids/Pyramids", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true}, "environment_parameters": null, "checkpoint_settings": {"run_id": "Pyramids Training", "initialize_from": null, "load_model": false, "resume": true, "force": false, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}

configuration.yaml

@@ -51,7 +51,7 @@ behaviors:
     init_path: null
     keep_checkpoints: 5
     even_checkpoints: false
-    max_steps: 1000000
+    max_steps: 1500000
     time_horizon: 128
     summary_freq: 30000
     threaded: false
@@ -80,8 +80,8 @@ checkpoint_settings:
   run_id: Pyramids Training
   initialize_from: null
   load_model: false
-  resume: false
-  force: true
+  resume: true
+  force: false
   train_model: false
   inference: false
   results_dir: results

run_logs/Player-0.log

@@ -1,7 +1,7 @@
 Mono path[0] = '/content/ml-agents/training-envs-executables/linux/Pyramids/Pyramids_Data/Managed'
 Mono config path = '/content/ml-agents/training-envs-executables/linux/Pyramids/Pyramids_Data/MonoBleedingEdge/etc'
 Found 1 interfaces on host : 0) 172.28.0.12
-Multi-casting "[IP] 172.28.0.12 [Port] 55032 [Flags] 2 [Guid] 1356917595 [EditorId] 764847374 [Version] 1048832 [Id] LinuxPlayer(13,172.28.0.12) [Debug] 0 [PackageName] LinuxPlayer [ProjectName] UnityEnvironment" to [225.0.0.222:54997]...
+Multi-casting "[IP] 172.28.0.12 [Port] 55342 [Flags] 2 [Guid] 1498458477 [EditorId] 764847374 [Version] 1048832 [Id] LinuxPlayer(13,172.28.0.12) [Debug] 0 [PackageName] LinuxPlayer [ProjectName] UnityEnvironment" to [225.0.0.222:54997]...
 Preloaded 'lib_burst_generated.so'
 Preloaded 'libgrpc_csharp_ext.x64.so'
 Initialize engine version: 2021.3.5f1 (40eb3a945986)
@@ -33,7 +33,7 @@ ALSA lib pcm.c:2642:(snd_pcm_open_noupdate) Unknown PCM default
 FMOD failed to initialize the output device.: "Error initializing output device. " (60)
 FMOD initialized on nosound output
 Begin MonoManager ReloadAssembly
-- Completed reload, in  0.091 seconds
+- Completed reload, in  0.088 seconds
 ERROR: Shader Sprites/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Sprites/Mask shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Legacy Shaders/VertexLit shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
@@ -65,7 +65,7 @@ ERROR: Shader Standard shader is not supported on this GPU (none of subshaders/f
 WARNING: Shader Unsupported: 'Standard' - All subshaders removed
 WARNING: Shader Did you use #pragma only_renderers and omit this platform?
 WARNING: Shader If subshaders removal was intentional, you may have forgotten turning Fallback off?
-UnloadTime: 0.805319 ms
+UnloadTime: 0.810411 ms
 ERROR: Shader UI/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 requesting resize 84 x 84
 Setting up 1 worker threads for Enlighten.
@@ -73,7 +73,7 @@ PlayerConnection::CleanupMemory Statistics:
 [ALLOC_TEMP_TLS] TLS Allocator
   StackAllocators : 
     [ALLOC_TEMP_MAIN]
-      Peak usage frame count: [16.0 KB-32.0 KB]: 1171 frames, [32.0 KB-64.0 KB]: 14898 frames, [64.0 KB-128.0 KB]: 2702 frames, [2.0 MB-4.0 MB]: 1 frames
+      Peak usage frame count: [16.0 KB-32.0 KB]: 20 frames, [32.0 KB-64.0 KB]: 6532 frames, [64.0 KB-128.0 KB]: 2841 frames, [2.0 MB-4.0 MB]: 1 frames
       Initial Block Size 4.0 MB
       Current Block Size 4.0 MB
       Peak Allocated Bytes 2.0 MB
@@ -123,7 +123,7 @@ PlayerConnection::CleanupMemory Statistics:
       Current Block Size 64.0 KB
       Peak Allocated Bytes 0 B
       Overflow Count 0
-    [ALLOC_TEMP_Background Job.Worker 2]
+    [ALLOC_TEMP_Background Job.Worker 15]
       Initial Block Size 32.0 KB
       Current Block Size 32.0 KB
       Peak Allocated Bytes 0 B
@@ -133,7 +133,7 @@ PlayerConnection::CleanupMemory Statistics:
       Current Block Size 32.0 KB
       Peak Allocated Bytes 0 B
       Overflow Count 0
-    [ALLOC_TEMP_Background Job.Worker 15]
+    [ALLOC_TEMP_Background Job.Worker 2]
       Initial Block Size 32.0 KB
       Current Block Size 32.0 KB
       Peak Allocated Bytes 0 B
@@ -204,28 +204,28 @@ PlayerConnection::CleanupMemory Statistics:
       Peak Allocated Bytes 128 B
       Overflow Count 0
 [ALLOC_MEMORYPROFILER]
-  Peak usage frame count: [0.5 MB-1.0 MB]: 299 frames, [1.0 MB-2.0 MB]: 18473 frames
+  Peak usage frame count: [0.5 MB-1.0 MB]: 37 frames, [1.0 MB-2.0 MB]: 9357 frames
   Requested Block Size 1.0 MB
   Peak Block count 2
-  Peak Allocated memory 1.6 MB
+  Peak Allocated memory 1.1 MB
   Peak Large allocation bytes 0 B
 [ALLOC_DEFAULT] Dual Thread Allocator
-  Peak main deferred allocation count 11813
+  Peak main deferred allocation count 9780
     [ALLOC_BUCKET]
       Large Block size 4.0 MB
       Used Block count 1
-      Peak Allocated bytes 1.7 MB
+      Peak Allocated bytes 1.3 MB
     [ALLOC_DEFAULT_MAIN]
-      Peak usage frame count: [8.0 MB-16.0 MB]: 1 frames, [16.0 MB-32.0 MB]: 18771 frames
+      Peak usage frame count: [8.0 MB-16.0 MB]: 1 frames, [16.0 MB-32.0 MB]: 9393 frames
       Requested Block Size 16.0 MB
-      Peak Block count 3
-      Peak Allocated memory 31.9 MB
+      Peak Block count 2
+      Peak Allocated memory 24.2 MB
       Peak Large allocation bytes 0 B
     [ALLOC_DEFAULT_THREAD]
-      Peak usage frame count: [16.0 MB-32.0 MB]: 18772 frames
+      Peak usage frame count: [16.0 MB-32.0 MB]: 9394 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
-      Peak Allocated memory 22.1 MB
+      Peak Allocated memory 22.9 MB
       Peak Large allocation bytes 16.0 MB
 [ALLOC_TEMP_JOB_1_FRAME]
   Initial Block Size 2.0 MB
@@ -252,15 +252,15 @@ PlayerConnection::CleanupMemory Statistics:
     [ALLOC_BUCKET]
       Large Block size 4.0 MB
       Used Block count 1
-      Peak Allocated bytes 1.7 MB
+      Peak Allocated bytes 1.3 MB
     [ALLOC_GFX_MAIN]
-      Peak usage frame count: [32.0 KB-64.0 KB]: 18771 frames, [64.0 KB-128.0 KB]: 1 frames
+      Peak usage frame count: [32.0 KB-64.0 KB]: 9393 frames, [64.0 KB-128.0 KB]: 1 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 67.0 KB
       Peak Large allocation bytes 0 B
     [ALLOC_GFX_THREAD]
-      Peak usage frame count: [64.0 KB-128.0 KB]: 18772 frames
+      Peak usage frame count: [64.0 KB-128.0 KB]: 9394 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 67.7 KB
@@ -270,15 +270,15 @@ PlayerConnection::CleanupMemory Statistics:
     [ALLOC_BUCKET]
       Large Block size 4.0 MB
       Used Block count 1
-      Peak Allocated bytes 1.7 MB
+      Peak Allocated bytes 1.3 MB
     [ALLOC_CACHEOBJECTS_MAIN]
-      Peak usage frame count: [0.5 MB-1.0 MB]: 299 frames, [1.0 MB-2.0 MB]: 18473 frames
+      Peak usage frame count: [0.5 MB-1.0 MB]: 51 frames, [1.0 MB-2.0 MB]: 9343 frames
       Requested Block Size 4.0 MB
       Peak Block count 1
       Peak Allocated memory 1.4 MB
       Peak Large allocation bytes 0 B
     [ALLOC_CACHEOBJECTS_THREAD]
-      Peak usage frame count: [1.0 MB-2.0 MB]: 18771 frames, [4.0 MB-8.0 MB]: 1 frames
+      Peak usage frame count: [1.0 MB-2.0 MB]: 9393 frames, [4.0 MB-8.0 MB]: 1 frames
       Requested Block Size 4.0 MB
       Peak Block count 2
       Peak Allocated memory 4.7 MB
@@ -288,21 +288,21 @@ PlayerConnection::CleanupMemory Statistics:
     [ALLOC_BUCKET]
       Large Block size 4.0 MB
       Used Block count 1
-      Peak Allocated bytes 1.7 MB
+      Peak Allocated bytes 1.3 MB
     [ALLOC_TYPETREE_MAIN]
-      Peak usage frame count: [0-1.0 KB]: 18772 frames
+      Peak usage frame count: [0-1.0 KB]: 9394 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 1.0 KB
       Peak Large allocation bytes 0 B
     [ALLOC_TYPETREE_THREAD]
-      Peak usage frame count: [1.0 KB-2.0 KB]: 18772 frames
+      Peak usage frame count: [1.0 KB-2.0 KB]: 9394 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 1.9 KB
       Peak Large allocation bytes 0 B
 [ALLOC_PROFILER]
-  Peak usage frame count: [16.0 KB-32.0 KB]: 18772 frames
+  Peak usage frame count: [16.0 KB-32.0 KB]: 9394 frames
   Requested Block Size 16.0 MB
   Peak Block count 1
   Peak Allocated memory 29.2 KB
@@ -311,4 +311,4 @@ PlayerConnection::CleanupMemory Statistics:
       Large Block size 4.0 MB
       Used Block count 1
       Peak Allocated bytes 396 B
-##utp:{"type":"MemoryLeaks","version":2,"phase":"Immediate","time":1683876255787,"processId":4645,"allocatedMemory":1970133,"memoryLabels":[{"Default":9033},{"Permanent":1264},{"NewDelete":12801},{"Thread":34460},{"Manager":10603},{"VertexData":12},{"Geometry":280},{"Texture":16},{"Shader":69173},{"Material":24},{"GfxDevice":35248},{"Animation":304},{"Audio":3976},{"Physics":288},{"Serialization":216},{"Input":9176},{"JobScheduler":200},{"Mono":40},{"ScriptingNativeRuntime":216},{"BaseObject":1609212},{"Resource":592},{"Renderer":1936},{"Transform":48},{"File":800},{"WebCam":24},{"Culling":40},{"Terrain":953},{"Wind":24},{"String":3447},{"DynamicArray":30868},{"HashMap":7680},{"Utility":1360},{"PoolAlloc":1160},{"TypeTree":1792},{"ScriptManager":80},{"RuntimeInitializeOnLoadManager":72},{"SpriteAtlas":112},{"GI":3272},{"Unet":16},{"Director":7760},{"WebRequest":720},{"VR":45473},{"SceneManager":424},{"Video":32},{"LazyScriptCache":32},{"NativeArray":384},{"Camera":25},{"Secure":1},{"SerializationCache":624},{"APIUpdating":5872},{"Subsystems":384},{"VirtualTexturing":57552},{"AssetReference":32}]}
+##utp:{"type":"MemoryLeaks","version":2,"phase":"Immediate","time":1683877696656,"processId":14684,"allocatedMemory":1970133,"memoryLabels":[{"Default":9033},{"Permanent":1264},{"NewDelete":12801},{"Thread":34460},{"Manager":10603},{"VertexData":12},{"Geometry":280},{"Texture":16},{"Shader":69173},{"Material":24},{"GfxDevice":35248},{"Animation":304},{"Audio":3976},{"Physics":288},{"Serialization":216},{"Input":9176},{"JobScheduler":200},{"Mono":40},{"ScriptingNativeRuntime":216},{"BaseObject":1609212},{"Resource":592},{"Renderer":1936},{"Transform":48},{"File":800},{"WebCam":24},{"Culling":40},{"Terrain":953},{"Wind":24},{"String":3447},{"DynamicArray":30868},{"HashMap":7680},{"Utility":1360},{"PoolAlloc":1160},{"TypeTree":1792},{"ScriptManager":80},{"RuntimeInitializeOnLoadManager":72},{"SpriteAtlas":112},{"GI":3272},{"Unet":16},{"Director":7760},{"WebRequest":720},{"VR":45473},{"SceneManager":424},{"Video":32},{"LazyScriptCache":32},{"NativeArray":384},{"Camera":25},{"Secure":1},{"SerializationCache":624},{"APIUpdating":5872},{"Subsystems":384},{"VirtualTexturing":57552},{"AssetReference":32}]}

run_logs/timers.json

@@ -2,249 +2,249 @@
     "name": "root",
     "gauges": {
         "Pyramids.Policy.Entropy.mean": {
-            "value": 0.8385428190231323,
-            "min": 0.8385428190231323,
-            "max": 1.5890600681304932,
-            "count": 33
+            "value": 0.4622502326965332,
+            "min": 0.4622502326965332,
+            "max": 0.8226544260978699,
+            "count": 17
         },
         "Pyramids.Policy.Entropy.sum": {
-            "value": 25075.78515625,
-            "min": 25075.78515625,
-            "max": 48205.7265625,
-            "count": 33
+            "value": 13971.05078125,
+            "min": 13971.05078125,
+            "max": 24366.34375,
+            "count": 17
+        },
+        "Pyramids.Environment.EpisodeLength.mean": {
+            "value": 337.07865168539325,
+            "min": 285.04672897196264,
+            "max": 398.05333333333334,
+            "count": 17
+        },
+        "Pyramids.Environment.EpisodeLength.sum": {
+            "value": 30000.0,
+            "min": 16732.0,
+            "max": 31570.0,
+            "count": 17
         },
         "Pyramids.Step.mean": {
-            "value": 989925.0,
-            "min": 29952.0,
-            "max": 989925.0,
-            "count": 33
+            "value": 1499926.0,
+            "min": 1019984.0,
+            "max": 1499926.0,
+            "count": 17
         },
         "Pyramids.Step.sum": {
-            "value": 989925.0,
-            "min": 29952.0,
-            "max": 989925.0,
-            "count": 33
+            "value": 1499926.0,
+            "min": 1019984.0,
+            "max": 1499926.0,
+            "count": 17
         },
         "Pyramids.Policy.ExtrinsicValueEstimate.mean": {
-            "value": 0.4319862127304077,
-            "min": -0.08273633569478989,
-            "max": 0.45908117294311523,
-            "count": 33
+            "value": 0.6091444492340088,
+            "min": 0.44797760248184204,
+            "max": 0.6157050728797913,
+            "count": 17
         },
         "Pyramids.Policy.ExtrinsicValueEstimate.sum": {
-            "value": 116.63627624511719,
-            "min": -19.939456939697266,
-            "max": 122.11559295654297,
-            "count": 33
+            "value": 169.95130920410156,
+            "min": 80.18798828125,
+            "max": 178.9472198486328,
+            "count": 17
         },
         "Pyramids.Policy.RndValueEstimate.mean": {
-            "value": 0.0059456657618284225,
-            "min": -0.0035603377036750317,
-            "max": 0.22215938568115234,
-            "count": 33
+            "value": -0.0009388831676915288,
+            "min": -0.002596844919025898,
+            "max": 0.02771277353167534,
+            "count": 17
         },
         "Pyramids.Policy.RndValueEstimate.sum": {
-            "value": 1.6053297519683838,
-            "min": -0.9506101608276367,
-            "max": 53.31825256347656,
-            "count": 33
+            "value": -0.2619484066963196,
+            "min": -0.7582787275314331,
+            "max": 7.759576797485352,
+            "count": 17
+        },
+        "Pyramids.Environment.CumulativeReward.mean": {
+            "value": 1.57300896875644,
+            "min": 1.4129749780520797,
+            "max": 1.7149532583550873,
+            "count": 17
+        },
+        "Pyramids.Environment.CumulativeReward.sum": {
+            "value": 139.99779821932316,
+            "min": 73.26699831336737,
+            "max": 183.49999864399433,
+            "count": 17
+        },
+        "Pyramids.Policy.ExtrinsicReward.mean": {
+            "value": 1.57300896875644,
+            "min": 1.4129749780520797,
+            "max": 1.7149532583550873,
+            "count": 17
+        },
+        "Pyramids.Policy.ExtrinsicReward.sum": {
+            "value": 139.99779821932316,
+            "min": 73.26699831336737,
+            "max": 183.49999864399433,
+            "count": 17
+        },
+        "Pyramids.Policy.RndReward.mean": {
+            "value": 0.016932798826509794,
+            "min": 0.014436905065586215,
+            "max": 0.02103800734153386,
+            "count": 17
+        },
+        "Pyramids.Policy.RndReward.sum": {
+            "value": 1.5070190955593716,
+            "min": 0.8800037323162542,
+            "max": 1.6817980849882588,
+            "count": 17
         },
         "Pyramids.Losses.PolicyLoss.mean": {
-            "value": 0.07681248507433339,
-            "min": 0.06329345493474701,
-            "max": 0.07681248507433339,
-            "count": 33
+            "value": 0.07209071715866837,
+            "min": 0.06803177403198787,
+            "max": 0.07367595214115379,
+            "count": 17
         },
         "Pyramids.Losses.PolicyLoss.sum": {
-            "value": 0.15362497014866677,
-            "min": 0.13458952701919424,
-            "max": 0.22450187834474492,
-            "count": 33
+            "value": 0.2162721514760051,
+            "min": 0.07306043186981696,
+            "max": 0.22102785642346134,
+            "count": 17
         },
         "Pyramids.Losses.ValueLoss.mean": {
-            "value": 0.013083591018585139,
-            "min": 3.952648239402317e-05,
-            "max": 0.014212872976107368,
-            "count": 33
+            "value": 0.014723541859711984,
+            "min": 0.01300292170223353,
+            "max": 0.015810804319335146,
+            "count": 17
         },
         "Pyramids.Losses.ValueLoss.sum": {
-            "value": 0.026167182037170278,
-            "min": 0.00011857944718206952,
-            "max": 0.0426386189283221,
-            "count": 33
+            "value": 0.04417062557913595,
+            "min": 0.01432669066125527,
+            "max": 0.04718441395743867,
+            "count": 17
         },
         "Pyramids.Policy.LearningRate.mean": {
-            "value": 7.500847499749994e-06,
-            "min": 7.500847499749994e-06,
-            "max": 0.000294528001824,
-            "count": 33
+            "value": 3.116165627977783e-06,
+            "min": 3.116165627977783e-06,
+            "max": 9.792826735726667e-05,
+            "count": 17
         },
         "Pyramids.Policy.LearningRate.sum": {
-            "value": 1.5001694999499988e-05,
-            "min": 1.5001694999499988e-05,
-            "max": 0.0008350272216576,
-            "count": 33
+            "value": 9.34849688393335e-06,
+            "min": 9.34849688393335e-06,
+            "max": 0.0002813876062042,
+            "count": 17
         },
         "Pyramids.Policy.Epsilon.mean": {
-            "value": 0.10250025,
-            "min": 0.10250025,
-            "max": 0.19817599999999996,
-            "count": 33
+            "value": 0.1010386888888889,
+            "min": 0.1010386888888889,
+            "max": 0.13264273333333335,
+            "count": 17
         },
         "Pyramids.Policy.Epsilon.sum": {
-            "value": 0.2050005,
-            "min": 0.2050005,
-            "max": 0.5783423999999999,
-            "count": 33
+            "value": 0.3031160666666667,
+            "min": 0.13264273333333335,
+            "max": 0.39379580000000003,
+            "count": 17
         },
         "Pyramids.Policy.Beta.mean": {
-            "value": 0.0002597749749999999,
-            "min": 0.0002597749749999999,
-            "max": 0.0098177824,
-            "count": 33
+            "value": 0.0001137650200000002,
+            "min": 0.0001137650200000002,
+            "max": 0.00327100906,
+            "count": 17
         },
         "Pyramids.Policy.Beta.sum": {
-            "value": 0.0005195499499999998,
-            "min": 0.0005195499499999998,
-            "max": 0.02783640576,
-            "count": 33
+            "value": 0.0003412950600000006,
+            "min": 0.0003412950600000006,
+            "max": 0.009400200420000001,
+            "count": 17
         },
         "Pyramids.Losses.RNDLoss.mean": {
-            "value": 0.00516517274081707,
-            "min": 0.00516517274081707,
-            "max": 0.25410041213035583,
-            "count": 33
+            "value": 0.004762061405926943,
+            "min": 0.004667783156037331,
+            "max": 0.005338639952242374,
+            "count": 17
         },
         "Pyramids.Losses.RNDLoss.sum": {
-            "value": 0.01033034548163414,
-            "min": 0.01033034548163414,
-            "max": 0.5082008242607117,
-            "count": 33
-        },
-        "Pyramids.Environment.EpisodeLength.mean": {
-            "value": 393.3013698630137,
-            "min": 393.3013698630137,
-            "max": 999.0,
-            "count": 33
-        },
-        "Pyramids.Environment.EpisodeLength.sum": {
-            "value": 28711.0,
-            "min": 15984.0,
-            "max": 32949.0,
-            "count": 33
-        },
-        "Pyramids.Environment.CumulativeReward.mean": {
-            "value": 1.5473333098408248,
-            "min": -1.0000000521540642,
-            "max": 1.5473333098408248,
-            "count": 33
-        },
-        "Pyramids.Environment.CumulativeReward.sum": {
-            "value": 111.40799830853939,
-            "min": -32.000001668930054,
-            "max": 111.40799830853939,
-            "count": 33
-        },
-        "Pyramids.Policy.ExtrinsicReward.mean": {
-            "value": 1.5473333098408248,
-            "min": -1.0000000521540642,
-            "max": 1.5473333098408248,
-            "count": 33
-        },
-        "Pyramids.Policy.ExtrinsicReward.sum": {
-            "value": 111.40799830853939,
-            "min": -32.000001668930054,
-            "max": 111.40799830853939,
-            "count": 33
-        },
-        "Pyramids.Policy.RndReward.mean": {
-            "value": 0.02124990586404844,
-            "min": 0.02124990586404844,
-            "max": 10.889289078768343,
-            "count": 33
-        },
-        "Pyramids.Policy.RndReward.sum": {
-            "value": 1.5299932222114876,
-            "min": 1.5299932222114876,
-            "max": 174.22862526029348,
-            "count": 33
+            "value": 0.014286183752119541,
+            "min": 0.005338639952242374,
+            "max": 0.01533021405339241,
+            "count": 17
         },
         "Pyramids.IsTraining.mean": {
             "value": 1.0,
             "min": 1.0,
             "max": 1.0,
-            "count": 33
+            "count": 17
         },
         "Pyramids.IsTraining.sum": {
             "value": 1.0,
             "min": 1.0,
             "max": 1.0,
-            "count": 33
+            "count": 17
         }
     },
     "metadata": {
         "timer_format_version": "0.1.0",
-        "start_time_seconds": "1683873966",
+        "start_time_seconds": "1683876347",
         "python_version": "3.10.11 (main, Apr  5 2023, 14:15:10) [GCC 9.4.0]",
-        "command_line_arguments": "/usr/local/bin/mlagents-learn ./config/ppo/PyramidsRND.yaml --env=./training-envs-executables/linux/Pyramids/Pyramids --run-id=Pyramids Training --no-graphics --force",
+        "command_line_arguments": "/usr/local/bin/mlagents-learn ./config/ppo/PyramidsRND.yaml --env=./training-envs-executables/linux/Pyramids/Pyramids --run-id=Pyramids Training --no-graphics --resume",
         "mlagents_version": "0.31.0.dev0",
         "mlagents_envs_version": "0.31.0.dev0",
         "communication_protocol_version": "1.5.0",
         "pytorch_version": "1.11.0+cu102",
         "numpy_version": "1.21.2",
-        "end_time_seconds": "1683876255"
+        "end_time_seconds": "1683877696"
     },
-    "total": 2289.26686924,
+    "total": 1349.5704583749994,
     "count": 1,
-    "self": 0.5465206359999684,
+    "self": 0.9971535599993331,
     "children": {
         "run_training.setup": {
-            "total": 0.04116973500003951,
+            "total": 0.04258645500021885,
             "count": 1,
-            "self": 0.04116973500003951
+            "self": 0.04258645500021885
         },
         "TrainerController.start_learning": {
-            "total": 2288.6791788689998,
+            "total": 1348.5307183599998,
             "count": 1,
-            "self": 1.332685328066873,
+            "self": 0.6913183310116437,
             "children": {
                 "TrainerController._reset_env": {
-                    "total": 4.140816312999959,
+                    "total": 3.8799759660000745,
                     "count": 1,
-                    "self": 4.140816312999959
+                    "self": 3.8799759660000745
                 },
                 "TrainerController.advance": {
-                    "total": 2282.9927085829336,
-                    "count": 63567,
-                    "self": 1.3919436829814913,
+                    "total": 1343.650322576988,
+                    "count": 32444,
+                    "self": 0.6973726529513442,
                     "children": {
                         "env_step": {
-                            "total": 1419.5153943539535,
-                            "count": 63567,
-                            "self": 1305.1721908540112,
+                            "total": 911.2838344790152,
+                            "count": 32444,
+                            "self": 853.1210107630559,
                             "children": {
                                 "SubprocessEnvManager._take_step": {
-                                    "total": 113.55545853497597,
-                                    "count": 63567,
-                                    "self": 4.6116142029576395,
+                                    "total": 57.74587193495745,
+                                    "count": 32444,
+                                    "self": 2.386119256996153,
                                     "children": {
                                         "TorchPolicy.evaluate": {
-                                            "total": 108.94384433201833,
-                                            "count": 62571,
-                                            "self": 108.94384433201833
+                                            "total": 55.359752677961296,
+                                            "count": 31311,
+                                            "self": 55.359752677961296
                                         }
                                     }
                                 },
                                 "workers": {
-                                    "total": 0.7877449649662367,
-                                    "count": 63567,
+                                    "total": 0.4169517810018988,
+                                    "count": 32444,
                                     "self": 0.0,
                                     "children": {
                                         "worker_root": {
-                                            "total": 2283.7583395279426,
-                                            "count": 63567,
+                                            "total": 1345.3879893880235,
+                                            "count": 32444,
                                             "is_parallel": true,
-                                            "self": 1088.6410247929534,
+                                            "self": 549.5327546290437,
                                             "children": {
                                                 "run_training.setup": {
                                                     "total": 0.0,
@@ -253,48 +253,48 @@
                                                     "self": 0.0,
                                                     "children": {
                                                         "steps_from_proto": {
-                                                            "total": 0.001960275999977057,
+                                                            "total": 0.0017900959996950405,
                                                             "count": 1,
                                                             "is_parallel": true,
-                                                            "self": 0.0006320699998241253,
+                                                            "self": 0.0005459640001390653,
                                                             "children": {
                                                                 "_process_rank_one_or_two_observation": {
-                                                                    "total": 0.0013282060001529317,
+                                                                    "total": 0.0012441319995559752,
                                                                     "count": 8,
                                                                     "is_parallel": true,
-                                                                    "self": 0.0013282060001529317
+                                                                    "self": 0.0012441319995559752
                                                                 }
                                                             }
                                                         },
                                                         "UnityEnvironment.step": {
-                                                            "total": 0.10325929900000119,
+                                                            "total": 0.04645710200020403,
                                                             "count": 1,
                                                             "is_parallel": true,
-                                                            "self": 0.0006078629999137775,
+                                                            "self": 0.0005328070001269225,
                                                             "children": {
                                                                 "UnityEnvironment._generate_step_input": {
-                                                                    "total": 0.0006261280000217084,
+                                                                    "total": 0.0005093260001558519,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.0006261280000217084
+                                                                    "self": 0.0005093260001558519
                                                                 },
                                                                 "communicator.exchange": {
-                                                                    "total": 0.1001386600000842,
+                                                                    "total": 0.043684293999831425,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.1001386600000842
+                                                                    "self": 0.043684293999831425
                                                                 },
                                                                 "steps_from_proto": {
-                                                                    "total": 0.0018866479999815056,
+                                                                    "total": 0.0017306750000898319,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.0003893330002711082,
+                                                                    "self": 0.0003570369995031797,
                                                                     "children": {
                                                                         "_process_rank_one_or_two_observation": {
-                                                                            "total": 0.0014973149997103974,
+                                                                            "total": 0.0013736380005866522,
                                                                             "count": 8,
                                                                             "is_parallel": true,
-                                                                            "self": 0.0014973149997103974
+                                                                            "self": 0.0013736380005866522
                                                                         }
                                                                     }
                                                                 }
@@ -303,34 +303,34 @@
                                                     }
                                                 },
                                                 "UnityEnvironment.step": {
-                                                    "total": 1195.1173147349891,
-                                                    "count": 63566,
+                                                    "total": 795.8552347589798,
+                                                    "count": 32443,
                                                     "is_parallel": true,
-                                                    "self": 31.09438601096508,
+                                                    "self": 15.910338530988156,
                                                     "children": {
                                                         "UnityEnvironment._generate_step_input": {
-                                                            "total": 23.01712769598953,
-                                                            "count": 63566,
+                                                            "total": 11.221930144051385,
+                                                            "count": 32443,
                                                             "is_parallel": true,
-                                                            "self": 23.01712769598953
+                                                            "self": 11.221930144051385
                                                         },
                                                         "communicator.exchange": {
-                                                            "total": 1043.5965429670314,
-                                                            "count": 63566,
+                                                            "total": 719.7232266690116,
+                                                            "count": 32443,
                                                             "is_parallel": true,
-                                                            "self": 1043.5965429670314
+                                                            "self": 719.7232266690116
                                                         },
                                                         "steps_from_proto": {
-                                                            "total": 97.40925806100324,
-                                                            "count": 63566,
+                                                            "total": 48.99973941492863,
+                                                            "count": 32443,
                                                             "is_parallel": true,
-                                                            "self": 19.86569543411315,
+                                                            "self": 10.047815213208196,
                                                             "children": {
                                                                 "_process_rank_one_or_two_observation": {
-                                                                    "total": 77.5435626268901,
-                                                                    "count": 508528,
+                                                                    "total": 38.95192420172043,
+                                                                    "count": 259544,
                                                                     "is_parallel": true,
-                                                                    "self": 77.5435626268901
+                                                                    "self": 38.95192420172043
                                                                 }
                                                             }
                                                         }
@@ -343,31 +343,31 @@
                             }
                         },
                         "trainer_advance": {
-                            "total": 862.0853705459988,
-                            "count": 63567,
-                            "self": 2.3431310039768505,
+                            "total": 431.66911544502136,
+                            "count": 32444,
+                            "self": 1.4210923290652318,
                             "children": {
                                 "process_trajectory": {
-                                    "total": 109.71520268302129,
-                                    "count": 63567,
-                                    "self": 109.33717446602077,
+                                    "total": 57.78840222495546,
+                                    "count": 32444,
+                                    "self": 57.5113453509548,
                                     "children": {
                                         "RLTrainer._checkpoint": {
-                                            "total": 0.37802821700051936,
-                                            "count": 2,
-                                            "self": 0.37802821700051936
+                                            "total": 0.2770568740006638,
+                                            "count": 1,
+                                            "self": 0.2770568740006638
                                         }
                                     }
                                 },
                                 "_update_policy": {
-                                    "total": 750.0270368590006,
-                                    "count": 94,
-                                    "self": 456.4830460809884,
+                                    "total": 372.45962089100067,
+                                    "count": 48,
+                                    "self": 224.31880705297317,
                                     "children": {
                                         "TorchPPOOptimizer.update": {
-                                            "total": 293.54399077801224,
-                                            "count": 30784,
-                                            "self": 293.54399077801224
+                                            "total": 148.1408138380275,
+                                            "count": 15364,
+                                            "self": 148.1408138380275
                                         }
                                     }
                                 }
@@ -376,19 +376,19 @@
                     }
                 },
                 "trainer_threads": {
-                    "total": 1.0249996194033884e-06,
+                    "total": 1.1530000847415067e-06,
                     "count": 1,
-                    "self": 1.0249996194033884e-06
+                    "self": 1.1530000847415067e-06
                 },
                 "TrainerController._save_models": {
-                    "total": 0.21296761999974478,
+                    "total": 0.30910033300006035,
                     "count": 1,
-                    "self": 0.04351796499986449,
+                    "self": 0.05594445699989592,
                     "children": {
                         "RLTrainer._checkpoint": {
-                            "total": 0.1694496549998803,
+                            "total": 0.2531558760001644,
                             "count": 1,
-                            "self": 0.1694496549998803
+                            "self": 0.2531558760001644
                         }
                     }
                 }

run_logs/training_status.json

@@ -27,15 +27,33 @@
                 "auxillary_file_paths": [
                     "results/Pyramids Training/Pyramids/Pyramids-1000006.pt"
                 ]
+            },
+            {
+                "steps": 1499926,
+                "file_path": "results/Pyramids Training/Pyramids/Pyramids-1499926.onnx",
+                "reward": 1.5960666418075562,
+                "creation_time": 1683877695.3980637,
+                "auxillary_file_paths": [
+                    "results/Pyramids Training/Pyramids/Pyramids-1499926.pt"
+                ]
+            },
+            {
+                "steps": 1500054,
+                "file_path": "results/Pyramids Training/Pyramids/Pyramids-1500054.onnx",
+                "reward": 1.5960666418075562,
+                "creation_time": 1683877695.6693466,
+                "auxillary_file_paths": [
+                    "results/Pyramids Training/Pyramids/Pyramids-1500054.pt"
+                ]
             }
         ],
         "final_checkpoint": {
-            "steps": 1000006,
+            "steps": 1500054,
             "file_path": "results/Pyramids Training/Pyramids.onnx",
-            "reward": 1.460466641045752,
-            "creation_time": 1683876255.2496288,
+            "reward": 1.5960666418075562,
+            "creation_time": 1683877695.6693466,
             "auxillary_file_paths": [
-                "results/Pyramids Training/Pyramids/Pyramids-1000006.pt"
+                "results/Pyramids Training/Pyramids/Pyramids-1500054.pt"
             ]
         }
     },