9 million training steps

Pyramids.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2dd9cb774f76b889fc11c90f247a5e500de6644b138bd4f14628e9dc4067a7b2
+oid sha256:3b88748d5d7a22674e6194b1baf3d89ed5142a3e8f49950f2ac746362bb2478a
 size 1418184

Pyramids/Pyramids-7499962.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1de85d693794c2bca85153fffe1031f1bc99b9c7a39485a011a78be487779894
+size 1418184

Pyramids/Pyramids-7499962.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b553bd075c7a207aa980d2e947112a87b0b081000986586b08a61566e284c9f
+size 8650990

Pyramids/Pyramids-7999980.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf9e5e19c8114d114313604e6bcada06bc7632c9f196bd837cfb059a6742b23b
+size 1418184

Pyramids/Pyramids-7999980.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8a2cb980b3f8d963e7cfbad566bfccd8a88545ff7cfd6f220117a5902555051
+size 8650990

Pyramids/Pyramids-8499960.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7399db6739cf29a97e44f24bd3c2fdd261a7d6b80596edfd346e381341e35e46
+size 1418184

Pyramids/Pyramids-8499960.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75091a7f8e89aba3c534d66e67f7ff53f4a5b8f8079684aebad1773b9a3a1d63
+size 8650990

Pyramids/Pyramids-8999992.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b88748d5d7a22674e6194b1baf3d89ed5142a3e8f49950f2ac746362bb2478a
+size 1418184

Pyramids/Pyramids-8999992.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c8a949c09b5a580f87d6596b288b55f101e7d008a01ec19e2265787520d4de5
+size 8650990

Pyramids/Pyramids-9000120.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b88748d5d7a22674e6194b1baf3d89ed5142a3e8f49950f2ac746362bb2478a
+size 1418184

Pyramids/Pyramids-9000120.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:782b449dccf5e96f87a1adf13b8354cf6a0a6ed20ef35d3d10b3a43c697975c5
+size 8650990

Pyramids/checkpoint.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:816e7f538205288ee7fba318720249a2824e6bef6327ca9d8a1274623aa41df7
+oid sha256:782b449dccf5e96f87a1adf13b8354cf6a0a6ed20ef35d3d10b3a43c697975c5
 size 8650990

Pyramids/events.out.tfevents.1688313675.sekhmet.27649.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a5f70ec5d2ffa165ee31d694a0e89aaafeec6df2b9ba4dec71d6ce9eb797e04
+size 823453

config.json

@@ -1 +1 @@
-{"default_settings": null, "behaviors": {"Pyramids": {"trainer_type": "ppo", "hyperparameters": {"batch_size": 128, "buffer_size": 2048, "learning_rate": 0.0003, "beta": 0.01, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 3, "shared_critic": false, "learning_rate_schedule": "linear", "beta_schedule": "linear", "epsilon_schedule": "linear"}, "checkpoint_interval": 500000, "network_settings": {"normalize": false, "hidden_units": 512, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.99, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}, "rnd": {"gamma": 0.99, "strength": 0.01, "network_settings": {"normalize": false, "hidden_units": 64, "num_layers": 3, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "learning_rate": 0.0001, "encoding_size": null}}, "init_path": null, "keep_checkpoints": 5, "even_checkpoints": false, "max_steps": 6000000, "time_horizon": 128, "summary_freq": 30000, "threaded": false, "self_play": null, "behavioral_cloning": null}}, "env_settings": {"env_path": "training-envs-executables/linux/Pyramids/Pyramids", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true}, "environment_parameters": null, "checkpoint_settings": {"run_id": "Pyramids1", "initialize_from": null, "load_model": false, "resume": true, "force": false, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}
+{"default_settings": null, "behaviors": {"Pyramids": {"trainer_type": "ppo", "hyperparameters": {"batch_size": 128, "buffer_size": 2048, "learning_rate": 0.0003, "beta": 0.01, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 3, "shared_critic": false, "learning_rate_schedule": "linear", "beta_schedule": "linear", "epsilon_schedule": "linear"}, "checkpoint_interval": 500000, "network_settings": {"normalize": false, "hidden_units": 512, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.99, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}, "rnd": {"gamma": 0.99, "strength": 0.01, "network_settings": {"normalize": false, "hidden_units": 64, "num_layers": 3, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "learning_rate": 0.0001, "encoding_size": null}}, "init_path": null, "keep_checkpoints": 5, "even_checkpoints": false, "max_steps": 9000000, "time_horizon": 128, "summary_freq": 30000, "threaded": false, "self_play": null, "behavioral_cloning": null}}, "env_settings": {"env_path": "training-envs-executables/linux/Pyramids/Pyramids", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true}, "environment_parameters": null, "checkpoint_settings": {"run_id": "Pyramids1", "initialize_from": null, "load_model": false, "resume": true, "force": false, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}

configuration.yaml

@@ -51,7 +51,7 @@ behaviors:
     init_path: null
     keep_checkpoints: 5
     even_checkpoints: false
-    max_steps: 6000000
+    max_steps: 9000000
     time_horizon: 128
     summary_freq: 30000
     threaded: false

run_logs/Player-0.log

@@ -1,7 +1,7 @@
 Mono path[0] = '/home/magnus/src/huggingface-reinforcement-learning-course/ml-agents/training-envs-executables/linux/Pyramids/Pyramids_Data/Managed'
 Mono config path = '/home/magnus/src/huggingface-reinforcement-learning-course/ml-agents/training-envs-executables/linux/Pyramids/Pyramids_Data/MonoBleedingEdge/etc'
 Found 1 interfaces on host : 0) 192.168.1.111
-Multi-casting "[IP] 192.168.1.111 [Port] 55384 [Flags] 2 [Guid] 1505190532 [EditorId] 764847374 [Version] 1048832 [Id] LinuxPlayer(13,192.168.1.111) [Debug] 0 [PackageName] LinuxPlayer [ProjectName] UnityEnvironment" to [225.0.0.222:54997]...
+Multi-casting "[IP] 192.168.1.111 [Port] 55466 [Flags] 2 [Guid] 187809420 [EditorId] 764847374 [Version] 1048832 [Id] LinuxPlayer(13,192.168.1.111) [Debug] 0 [PackageName] LinuxPlayer [ProjectName] UnityEnvironment" to [225.0.0.222:54997]...
 Preloaded 'lib_burst_generated.so'
 Preloaded 'libgrpc_csharp_ext.x64.so'
 Initialize engine version: 2021.3.5f1 (40eb3a945986)
@@ -17,7 +17,7 @@ Forced to initialize FMOD to to the device driver's system output rate 48000, th
 FMOD failed to initialize the output device.: "Error initializing output device. " (60)
 FMOD initialized on nosound output
 Begin MonoManager ReloadAssembly
-- Completed reload, in  0.062 seconds
+- Completed reload, in  0.065 seconds
 ERROR: Shader Sprites/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Sprites/Mask shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Legacy Shaders/VertexLit shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
@@ -49,7 +49,7 @@ ERROR: Shader Standard shader is not supported on this GPU (none of subshaders/f
 WARNING: Shader Unsupported: 'Standard' - All subshaders removed
 WARNING: Shader Did you use #pragma only_renderers and omit this platform?
 WARNING: Shader If subshaders removal was intentional, you may have forgotten turning Fallback off?
-UnloadTime: 0.564000 ms
+UnloadTime: 0.543146 ms
 ERROR: Shader UI/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 requesting resize 84 x 84
 Setting up 6 worker threads for Enlighten.
@@ -57,7 +57,7 @@ PlayerConnection::CleanupMemory Statistics:
 [ALLOC_TEMP_TLS] TLS Allocator
   StackAllocators : 
     [ALLOC_TEMP_MAIN]
-      Peak usage frame count: [16.0 KB-32.0 KB]: 8492 frames, [32.0 KB-64.0 KB]: 27079 frames, [64.0 KB-128.0 KB]: 20698 frames, [2.0 MB-4.0 MB]: 1 frames
+      Peak usage frame count: [16.0 KB-32.0 KB]: 7532 frames, [32.0 KB-64.0 KB]: 26284 frames, [64.0 KB-128.0 KB]: 22442 frames, [2.0 MB-4.0 MB]: 1 frames
       Initial Block Size 4.0 MB
       Current Block Size 4.0 MB
       Peak Allocated Bytes 2.0 MB
@@ -117,11 +117,6 @@ PlayerConnection::CleanupMemory Statistics:
       Current Block Size 32.0 KB
       Peak Allocated Bytes 0 B
       Overflow Count 0
-    [ALLOC_TEMP_Background Job.Worker 15]
-      Initial Block Size 32.0 KB
-      Current Block Size 32.0 KB
-      Peak Allocated Bytes 0 B
-      Overflow Count 0
     [ALLOC_TEMP_Background Job.Worker 7]
       Initial Block Size 32.0 KB
       Current Block Size 32.0 KB
@@ -147,6 +142,11 @@ PlayerConnection::CleanupMemory Statistics:
       Current Block Size 320.0 KB
       Peak Allocated Bytes 251.6 KB
       Overflow Count 4
+    [ALLOC_TEMP_Background Job.Worker 15]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
     [ALLOC_TEMP_Profiler.Dispatcher]
       Initial Block Size 64.0 KB
       Current Block Size 64.0 KB
@@ -192,20 +192,20 @@ PlayerConnection::CleanupMemory Statistics:
       Current Block Size 32.0 KB
       Peak Allocated Bytes 0 B
       Overflow Count 0
-    [ALLOC_TEMP_Job.Worker 6]
+    [ALLOC_TEMP_Job.Worker 0]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
       Peak Allocated Bytes 0 B
       Overflow Count 0
-    [ALLOC_TEMP_Job.Worker 0]
+    [ALLOC_TEMP_Job.Worker 6]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
-      Peak Allocated Bytes 0.8 KB
+      Peak Allocated Bytes 0 B
       Overflow Count 0
     [ALLOC_TEMP_Job.Worker 5]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
-      Peak Allocated Bytes 0 B
+      Peak Allocated Bytes 0.8 KB
       Overflow Count 0
     [ALLOC_TEMP_Background Job.Worker 9]
       Initial Block Size 32.0 KB
@@ -238,28 +238,28 @@ PlayerConnection::CleanupMemory Statistics:
       Peak Allocated Bytes 150 B
       Overflow Count 0
 [ALLOC_MEMORYPROFILER]
-  Peak usage frame count: [0.5 MB-1.0 MB]: 48 frames, [1.0 MB-2.0 MB]: 56222 frames
+  Peak usage frame count: [0.5 MB-1.0 MB]: 15 frames, [1.0 MB-2.0 MB]: 56244 frames
   Requested Block Size 1.0 MB
   Peak Block count 2
   Peak Allocated memory 1.1 MB
   Peak Large allocation bytes 0 B
 [ALLOC_DEFAULT] Dual Thread Allocator
-  Peak main deferred allocation count 1806
+  Peak main deferred allocation count 1930
     [ALLOC_BUCKET]
       Large Block size 4.0 MB
       Used Block count 1
       Peak Allocated bytes 1.4 MB
     [ALLOC_DEFAULT_MAIN]
-      Peak usage frame count: [8.0 MB-16.0 MB]: 1 frames, [16.0 MB-32.0 MB]: 56269 frames
+      Peak usage frame count: [8.0 MB-16.0 MB]: 1 frames, [16.0 MB-32.0 MB]: 56258 frames
       Requested Block Size 16.0 MB
       Peak Block count 2
-      Peak Allocated memory 25.7 MB
+      Peak Allocated memory 24.7 MB
       Peak Large allocation bytes 0 B
     [ALLOC_DEFAULT_THREAD]
-      Peak usage frame count: [16.0 MB-32.0 MB]: 56270 frames
+      Peak usage frame count: [16.0 MB-32.0 MB]: 56259 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
-      Peak Allocated memory 25.3 MB
+      Peak Allocated memory 26.0 MB
       Peak Large allocation bytes 16.0 MB
 [ALLOC_TEMP_JOB_1_FRAME]
   Initial Block Size 2.0 MB
@@ -288,13 +288,13 @@ PlayerConnection::CleanupMemory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.4 MB
     [ALLOC_GFX_MAIN]
-      Peak usage frame count: [32.0 KB-64.0 KB]: 56269 frames, [64.0 KB-128.0 KB]: 1 frames
+      Peak usage frame count: [32.0 KB-64.0 KB]: 56258 frames, [64.0 KB-128.0 KB]: 1 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 67.0 KB
       Peak Large allocation bytes 0 B
     [ALLOC_GFX_THREAD]
-      Peak usage frame count: [64.0 KB-128.0 KB]: 56270 frames
+      Peak usage frame count: [64.0 KB-128.0 KB]: 56259 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 67.7 KB
@@ -306,13 +306,13 @@ PlayerConnection::CleanupMemory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.4 MB
     [ALLOC_CACHEOBJECTS_MAIN]
-      Peak usage frame count: [0.5 MB-1.0 MB]: 52 frames, [1.0 MB-2.0 MB]: 56218 frames
+      Peak usage frame count: [0.5 MB-1.0 MB]: 27 frames, [1.0 MB-2.0 MB]: 56232 frames
       Requested Block Size 4.0 MB
       Peak Block count 1
       Peak Allocated memory 1.4 MB
       Peak Large allocation bytes 0 B
     [ALLOC_CACHEOBJECTS_THREAD]
-      Peak usage frame count: [1.0 MB-2.0 MB]: 56269 frames, [4.0 MB-8.0 MB]: 1 frames
+      Peak usage frame count: [1.0 MB-2.0 MB]: 56258 frames, [4.0 MB-8.0 MB]: 1 frames
       Requested Block Size 4.0 MB
       Peak Block count 2
       Peak Allocated memory 4.7 MB
@@ -324,19 +324,19 @@ PlayerConnection::CleanupMemory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.4 MB
     [ALLOC_TYPETREE_MAIN]
-      Peak usage frame count: [0-1.0 KB]: 56270 frames
+      Peak usage frame count: [0-1.0 KB]: 56259 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 1.0 KB
       Peak Large allocation bytes 0 B
     [ALLOC_TYPETREE_THREAD]
-      Peak usage frame count: [1.0 KB-2.0 KB]: 56270 frames
+      Peak usage frame count: [1.0 KB-2.0 KB]: 56259 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 1.9 KB
       Peak Large allocation bytes 0 B
 [ALLOC_PROFILER]
-  Peak usage frame count: [32.0 KB-64.0 KB]: 56270 frames
+  Peak usage frame count: [32.0 KB-64.0 KB]: 56259 frames
   Requested Block Size 16.0 MB
   Peak Block count 1
   Peak Allocated memory 34.7 KB
@@ -345,4 +345,4 @@ PlayerConnection::CleanupMemory Statistics:
       Large Block size 4.0 MB
       Used Block count 1
       Peak Allocated bytes 396 B
-##utp:{"type":"MemoryLeaks","version":2,"phase":"Immediate","time":1688313506324,"processId":27133,"allocatedMemory":2004877,"memoryLabels":[{"Default":9033},{"Permanent":1264},{"NewDelete":37399},{"Thread":34740},{"Manager":16203},{"VertexData":12},{"Geometry":280},{"Texture":16},{"Shader":69173},{"Material":24},{"GfxDevice":35248},{"Animation":304},{"Audio":3976},{"Physics":288},{"Serialization":216},{"Input":9176},{"JobScheduler":200},{"Mono":40},{"ScriptingNativeRuntime":216},{"BaseObject":1609212},{"Resource":592},{"Renderer":1936},{"Transform":48},{"File":800},{"WebCam":24},{"Culling":40},{"Terrain":953},{"Wind":24},{"String":3633},{"DynamicArray":30868},{"HashMap":7680},{"Utility":5440},{"PoolAlloc":1160},{"TypeTree":1792},{"ScriptManager":80},{"RuntimeInitializeOnLoadManager":72},{"SpriteAtlas":112},{"GI":3272},{"Unet":16},{"Director":7760},{"WebRequest":720},{"VR":45473},{"SceneManager":424},{"Video":32},{"LazyScriptCache":32},{"NativeArray":384},{"Camera":25},{"Secure":1},{"SerializationCache":624},{"APIUpdating":5872},{"Subsystems":384},{"VirtualTexturing":57552},{"AssetReference":32}]}
+##utp:{"type":"MemoryLeaks","version":2,"phase":"Immediate","time":1688317030985,"processId":27702,"allocatedMemory":2004877,"memoryLabels":[{"Default":9033},{"Permanent":1264},{"NewDelete":37399},{"Thread":34740},{"Manager":16203},{"VertexData":12},{"Geometry":280},{"Texture":16},{"Shader":69173},{"Material":24},{"GfxDevice":35248},{"Animation":304},{"Audio":3976},{"Physics":288},{"Serialization":216},{"Input":9176},{"JobScheduler":200},{"Mono":40},{"ScriptingNativeRuntime":216},{"BaseObject":1609212},{"Resource":592},{"Renderer":1936},{"Transform":48},{"File":800},{"WebCam":24},{"Culling":40},{"Terrain":953},{"Wind":24},{"String":3633},{"DynamicArray":30868},{"HashMap":7680},{"Utility":5440},{"PoolAlloc":1160},{"TypeTree":1792},{"ScriptManager":80},{"RuntimeInitializeOnLoadManager":72},{"SpriteAtlas":112},{"GI":3272},{"Unet":16},{"Director":7760},{"WebRequest":720},{"VR":45473},{"SceneManager":424},{"Video":32},{"LazyScriptCache":32},{"NativeArray":384},{"Camera":25},{"Secure":1},{"SerializationCache":624},{"APIUpdating":5872},{"Subsystems":384},{"VirtualTexturing":57552},{"AssetReference":32}]}

run_logs/timers.json

@@ -2,171 +2,171 @@
     "name": "root",
     "gauges": {
         "Pyramids.Policy.Entropy.mean": {
-            "value": 0.1430526077747345,
-            "min": 0.1262112557888031,
-            "max": 0.25186875462532043,
+            "value": 0.12155546993017197,
+            "min": 0.10337793081998825,
+            "max": 0.166367307305336,
             "count": 100
         },
         "Pyramids.Policy.Entropy.sum": {
-            "value": 4303.0224609375,
-            "min": 3774.22119140625,
-            "max": 7556.0625,
+            "value": 3617.49072265625,
+            "min": 3098.02978515625,
+            "max": 5028.28564453125,
             "count": 100
         },
         "Pyramids.Environment.EpisodeLength.mean": {
-            "value": 194.8108108108108,
-            "min": 189.01986754966887,
-            "max": 330.94897959183675,
+            "value": 203.24137931034483,
+            "min": 178.5988023952096,
+            "max": 276.77064220183485,
             "count": 100
         },
         "Pyramids.Environment.EpisodeLength.sum": {
-            "value": 28832.0,
-            "min": 27065.0,
-            "max": 32596.0,
+            "value": 29470.0,
+            "min": 26913.0,
+            "max": 32611.0,
             "count": 100
         },
         "Pyramids.Step.mean": {
-            "value": 5999973.0,
-            "min": 3029884.0,
-            "max": 5999973.0,
+            "value": 8999992.0,
+            "min": 6029924.0,
+            "max": 8999992.0,
             "count": 100
         },
         "Pyramids.Step.sum": {
-            "value": 5999973.0,
-            "min": 3029884.0,
-            "max": 5999973.0,
+            "value": 8999992.0,
+            "min": 6029924.0,
+            "max": 8999992.0,
             "count": 100
         },
         "Pyramids.Policy.ExtrinsicValueEstimate.mean": {
-            "value": 0.8933548331260681,
-            "min": 0.5950189828872681,
-            "max": 0.895928144454956,
+            "value": 0.8813865184783936,
+            "min": 0.7026673555374146,
+            "max": 0.9559917449951172,
             "count": 100
         },
         "Pyramids.Policy.ExtrinsicValueEstimate.sum": {
-            "value": 273.3665771484375,
-            "min": 166.60531616210938,
-            "max": 278.22900390625,
+            "value": 274.1112060546875,
+            "min": 203.07086181640625,
+            "max": 306.87335205078125,
             "count": 100
         },
         "Pyramids.Policy.RndValueEstimate.mean": {
-            "value": -0.035036664456129074,
-            "min": -0.035036664456129074,
-            "max": 0.11137405037879944,
+            "value": 0.00841159000992775,
+            "min": -0.0707196518778801,
+            "max": 0.032943692058324814,
             "count": 100
         },
         "Pyramids.Policy.RndValueEstimate.sum": {
-            "value": -10.721219062805176,
-            "min": -10.721219062805176,
-            "max": 33.85771179199219,
+            "value": 2.616004467010498,
+            "min": -21.07445526123047,
+            "max": 9.718389511108398,
             "count": 100
         },
         "Pyramids.Environment.CumulativeReward.mean": {
-            "value": 1.7916729624408323,
-            "min": 1.5669795772706976,
-            "max": 1.805647044594771,
+            "value": 1.7829641251728452,
+            "min": 1.57638714865807,
+            "max": 1.8214011828521055,
             "count": 100
         },
         "Pyramids.Environment.CumulativeReward.sum": {
-            "value": 265.1675984412432,
-            "min": 153.56399857252836,
-            "max": 277.3915982693434,
+            "value": 258.52979815006256,
+            "min": 171.82619920372963,
+            "max": 304.1739975363016,
             "count": 100
         },
         "Pyramids.Policy.ExtrinsicReward.mean": {
-            "value": 1.7916729624408323,
-            "min": 1.5669795772706976,
-            "max": 1.805647044594771,
+            "value": 1.7829641251728452,
+            "min": 1.57638714865807,
+            "max": 1.8214011828521055,
             "count": 100
         },
         "Pyramids.Policy.ExtrinsicReward.sum": {
-            "value": 265.1675984412432,
-            "min": 153.56399857252836,
-            "max": 277.3915982693434,
+            "value": 258.52979815006256,
+            "min": 171.82619920372963,
+            "max": 304.1739975363016,
             "count": 100
         },
         "Pyramids.Policy.RndReward.mean": {
-            "value": 0.011331361728603166,
-            "min": 0.011228123687611297,
-            "max": 0.01837677085388671,
+            "value": 0.009233273781538973,
+            "min": 0.008271286948001944,
+            "max": 0.014159980635803331,
             "count": 100
         },
         "Pyramids.Policy.RndReward.sum": {
-            "value": 1.6770415358332684,
-            "min": 1.411525305113173,
-            "max": 2.202837912082032,
+            "value": 1.338824698323151,
+            "min": 1.228675196536642,
+            "max": 1.861013156008994,
             "count": 100
         },
         "Pyramids.Losses.PolicyLoss.mean": {
-            "value": 0.06742418237716064,
-            "min": 0.0642036380724878,
-            "max": 0.07328082145180641,
+            "value": 0.06888185016023705,
+            "min": 0.06377542618380375,
+            "max": 0.07488835589278356,
             "count": 100
         },
         "Pyramids.Losses.PolicyLoss.sum": {
-            "value": 1.0113627356574095,
-            "min": 0.9034849535867882,
-            "max": 1.0850595171938826,
+            "value": 0.9643459022433187,
+            "min": 0.8928559665732525,
+            "max": 1.094879996866974,
             "count": 100
         },
         "Pyramids.Losses.ValueLoss.mean": {
-            "value": 0.015541635502245446,
-            "min": 0.011974255370400247,
-            "max": 0.01834407493231443,
+            "value": 0.01753181091533238,
+            "min": 0.01119533650635276,
+            "max": 0.01753181091533238,
             "count": 100
         },
         "Pyramids.Losses.ValueLoss.sum": {
-            "value": 0.2331245325336817,
-            "min": 0.16763957518560346,
-            "max": 0.256817049052402,
+            "value": 0.24544535281465335,
+            "min": 0.16624239571655042,
+            "max": 0.24544535281465335,
             "count": 100
         },
         "Pyramids.Policy.LearningRate.mean": {
-            "value": 7.487964171011104e-07,
-            "min": 7.487964171011104e-07,
-            "max": 0.00014920776812123691,
+            "value": 4.860760284841276e-07,
+            "min": 4.860760284841276e-07,
+            "max": 9.947281446146507e-05,
             "count": 100
         },
         "Pyramids.Policy.LearningRate.sum": {
-            "value": 1.1231946256516656e-05,
-            "min": 1.1231946256516656e-05,
-            "max": 0.0021263646912120333,
+            "value": 6.805064398777786e-06,
+            "min": 6.805064398777786e-06,
+            "max": 0.0014474719841763442,
             "count": 100
         },
         "Pyramids.Policy.Epsilon.mean": {
-            "value": 0.10024956555555554,
-            "min": 0.10024956555555554,
-            "max": 0.14973590595238098,
+            "value": 0.10016199206349205,
+            "min": 0.10016199206349205,
+            "max": 0.13315758253968255,
             "count": 100
         },
         "Pyramids.Policy.Epsilon.sum": {
-            "value": 1.5037434833333332,
-            "min": 1.41063965,
-            "max": 2.2087879666666668,
+            "value": 1.4022678888888886,
+            "min": 1.4022678888888886,
+            "max": 1.9824903222222225,
             "count": 100
         },
         "Pyramids.Policy.Beta.mean": {
-            "value": 3.493159899999997e-05,
-            "min": 3.493159899999997e-05,
-            "max": 0.004978617004642856,
+            "value": 2.6183007142857167e-05,
+            "min": 2.6183007142857167e-05,
+            "max": 0.0033224424957142866,
             "count": 100
         },
         "Pyramids.Policy.Beta.sum": {
-            "value": 0.0005239739849999996,
-            "min": 0.0005239739849999996,
-            "max": 0.07095791787,
+            "value": 0.0003665621000000003,
+            "min": 0.0003665621000000003,
+            "max": 0.04835078319000001,
             "count": 100
         },
         "Pyramids.Losses.RNDLoss.mean": {
-            "value": 0.005648714955896139,
-            "min": 0.004770483355969191,
-            "max": 0.006634328979998827,
+            "value": 0.00432272395119071,
+            "min": 0.003942703828215599,
+            "max": 0.005807704292237759,
             "count": 100
         },
         "Pyramids.Losses.RNDLoss.sum": {
-            "value": 0.08473072201013565,
-            "min": 0.06968885660171509,
-            "max": 0.09891163557767868,
+            "value": 0.060518134385347366,
+            "min": 0.057913638651371,
+            "max": 0.08529843389987946,
             "count": 100
         },
         "Pyramids.IsTraining.mean": {
@@ -184,7 +184,7 @@
     },
     "metadata": {
         "timer_format_version": "0.1.0",
-        "start_time_seconds": "1688310181",
+        "start_time_seconds": "1688313673",
         "python_version": "3.10.12 (main, Jun  9 2023, 20:10:04) [GCC 13.1.1 20230522 [revision dd36656ada05731c069ecd5b1878380294fb1f3e]]",
         "command_line_arguments": "/home/magnus/.local/venv/ml-agents/bin/mlagents-learn --resume config/ppo/PyramidsRND.yaml --env training-envs-executables/linux/Pyramids/Pyramids --run-id=Pyramids1 --no-graphics",
         "mlagents_version": "0.31.0.dev0",
@@ -192,59 +192,59 @@
         "communication_protocol_version": "1.5.0",
         "pytorch_version": "1.11.0+cu102",
         "numpy_version": "1.21.2",
-        "end_time_seconds": "1688313506"
+        "end_time_seconds": "1688317031"
     },
-    "total": 3324.868793236001,
+    "total": 3357.0718518270005,
     "count": 1,
-    "self": 0.31922269300230255,
+    "self": 0.3190375850026612,
     "children": {
         "run_training.setup": {
-            "total": 0.007570371999463532,
+            "total": 0.0074317609978606924,
             "count": 1,
-            "self": 0.007570371999463532
+            "self": 0.0074317609978606924
         },
         "TrainerController.start_learning": {
-            "total": 3324.542000170999,
+            "total": 3356.745382481,
             "count": 1,
-            "self": 3.0024852928454493,
+            "self": 2.9512564832948556,
             "children": {
                 "TrainerController._reset_env": {
-                    "total": 2.696618435000346,
+                    "total": 2.612207900998328,
                     "count": 1,
-                    "self": 2.696618435000346
+                    "self": 2.612207900998328
                 },
                 "TrainerController.advance": {
-                    "total": 3318.7878656481535,
-                    "count": 197822,
-                    "self": 2.735950404443429,
+                    "total": 3351.123781086706,
+                    "count": 198806,
+                    "self": 2.758197276325518,
                     "children": {
                         "env_step": {
-                            "total": 2248.670260114952,
-                            "count": 197822,
-                            "self": 2037.7029809287633,
+                            "total": 2284.4431627157064,
+                            "count": 198806,
+                            "self": 2075.268968008055,
                             "children": {
                                 "SubprocessEnvManager._take_step": {
-                                    "total": 209.00800150362193,
-                                    "count": 197822,
-                                    "self": 7.87247752492658,
+                                    "total": 207.21744262683205,
+                                    "count": 198806,
+                                    "self": 8.163825870218716,
                                     "children": {
                                         "TorchPolicy.evaluate": {
-                                            "total": 201.13552397869535,
-                                            "count": 187566,
-                                            "self": 201.13552397869535
+                                            "total": 199.05361675661334,
+                                            "count": 187530,
+                                            "self": 199.05361675661334
                                         }
                                     }
                                 },
                                 "workers": {
-                                    "total": 1.9592776825666078,
-                                    "count": 197822,
+                                    "total": 1.9567520808195695,
+                                    "count": 198806,
                                     "self": 0.0,
                                     "children": {
                                         "worker_root": {
-                                            "total": 3320.8473949115905,
-                                            "count": 197822,
+                                            "total": 3353.061701289891,
+                                            "count": 198806,
                                             "is_parallel": true,
-                                            "self": 1477.8273678632904,
+                                            "self": 1475.1090512383926,
                                             "children": {
                                                 "run_training.setup": {
                                                     "total": 0.0,
@@ -253,48 +253,48 @@
                                                     "self": 0.0,
                                                     "children": {
                                                         "steps_from_proto": {
-                                                            "total": 0.0009898610005620867,
+                                                            "total": 0.0010249199985992163,
                                                             "count": 1,
                                                             "is_parallel": true,
-                                                            "self": 0.00026216399601253215,
+                                                            "self": 0.0002954640040115919,
                                                             "children": {
                                                                 "_process_rank_one_or_two_observation": {
-                                                                    "total": 0.0007276970045495545,
+                                                                    "total": 0.0007294559945876244,
                                                                     "count": 8,
                                                                     "is_parallel": true,
-                                                                    "self": 0.0007276970045495545
+                                                                    "self": 0.0007294559945876244
                                                                 }
                                                             }
                                                         },
                                                         "UnityEnvironment.step": {
-                                                            "total": 0.02377290300137247,
+                                                            "total": 0.022472407999885036,
                                                             "count": 1,
                                                             "is_parallel": true,
-                                                            "self": 0.0002492580024409108,
+                                                            "self": 0.0005902970005990937,
                                                             "children": {
                                                                 "UnityEnvironment._generate_step_input": {
-                                                                    "total": 0.0002108610005961964,
+                                                                    "total": 0.00018204400112153962,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.0002108610005961964
+                                                                    "self": 0.00018204400112153962
                                                                 },
                                                                 "communicator.exchange": {
-                                                                    "total": 0.022614385999986553,
+                                                                    "total": 0.020123067999520572,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.022614385999986553
+                                                                    "self": 0.020123067999520572
                                                                 },
                                                                 "steps_from_proto": {
-                                                                    "total": 0.0006983979983488098,
+                                                                    "total": 0.0015769989986438304,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.00014349500088428613,
+                                                                    "self": 0.00024784100605756976,
                                                                     "children": {
                                                                         "_process_rank_one_or_two_observation": {
-                                                                            "total": 0.0005549029974645237,
+                                                                            "total": 0.0013291579925862607,
                                                                             "count": 8,
                                                                             "is_parallel": true,
-                                                                            "self": 0.0005549029974645237
+                                                                            "self": 0.0013291579925862607
                                                                         }
                                                                     }
                                                                 }
@@ -303,34 +303,34 @@
                                                     }
                                                 },
                                                 "UnityEnvironment.step": {
-                                                    "total": 1843.0200270483,
-                                                    "count": 197821,
+                                                    "total": 1877.9526500514985,
+                                                    "count": 198805,
                                                     "is_parallel": true,
-                                                    "self": 58.0508139172689,
+                                                    "self": 60.930162151478726,
                                                     "children": {
                                                         "UnityEnvironment._generate_step_input": {
-                                                            "total": 29.1573587008279,
-                                                            "count": 197821,
+                                                            "total": 31.088286421967496,
+                                                            "count": 198805,
                                                             "is_parallel": true,
-                                                            "self": 29.1573587008279
+                                                            "self": 31.088286421967496
                                                         },
                                                         "communicator.exchange": {
-                                                            "total": 1597.5823520508966,
-                                                            "count": 197821,
+                                                            "total": 1619.3520383784635,
+                                                            "count": 198805,
                                                             "is_parallel": true,
-                                                            "self": 1597.5823520508966
+                                                            "self": 1619.3520383784635
                                                         },
                                                         "steps_from_proto": {
-                                                            "total": 158.22950237930672,
-                                                            "count": 197821,
+                                                            "total": 166.5821630995888,
+                                                            "count": 198805,
                                                             "is_parallel": true,
-                                                            "self": 29.99255649505176,
+                                                            "self": 31.637742599959893,
                                                             "children": {
                                                                 "_process_rank_one_or_two_observation": {
-                                                                    "total": 128.23694588425496,
-                                                                    "count": 1582568,
+                                                                    "total": 134.9444204996289,
+                                                                    "count": 1590440,
                                                                     "is_parallel": true,
-                                                                    "self": 128.23694588425496
+                                                                    "self": 134.9444204996289
                                                                 }
                                                             }
                                                         }
@@ -343,31 +343,31 @@
                             }
                         },
                         "trainer_advance": {
-                            "total": 1067.3816551287582,
-                            "count": 197822,
-                            "self": 4.800970576938198,
+                            "total": 1063.922421094674,
+                            "count": 198806,
+                            "self": 5.016832006564073,
                             "children": {
                                 "process_trajectory": {
-                                    "total": 195.43404567573816,
-                                    "count": 197822,
-                                    "self": 194.985590716733,
+                                    "total": 198.75968678008576,
+                                    "count": 198806,
+                                    "self": 198.33774820808685,
                                     "children": {
                                         "RLTrainer._checkpoint": {
-                                            "total": 0.4484549590051756,
+                                            "total": 0.4219385719989077,
                                             "count": 6,
-                                            "self": 0.4484549590051756
+                                            "self": 0.4219385719989077
                                         }
                                     }
                                 },
                                 "_update_policy": {
-                                    "total": 867.1466388760819,
+                                    "total": 860.1459023080242,
                                     "count": 1421,
-                                    "self": 556.6953621563243,
+                                    "self": 555.9807385543063,
                                     "children": {
                                         "TorchPPOOptimizer.update": {
-                                            "total": 310.4512767197575,
-                                            "count": 68394,
-                                            "self": 310.4512767197575
+                                            "total": 304.16516375371793,
+                                            "count": 68340,
+                                            "self": 304.16516375371793
                                         }
                                     }
                                 }
@@ -376,19 +376,19 @@
                     }
                 },
                 "trainer_threads": {
-                    "total": 6.230002327356488e-07,
+                    "total": 6.219997885636985e-07,
                     "count": 1,
-                    "self": 6.230002327356488e-07
+                    "self": 6.219997885636985e-07
                 },
                 "TrainerController._save_models": {
-                    "total": 0.055030171999533195,
+                    "total": 0.05813638800100307,
                     "count": 1,
-                    "self": 0.0008026110008358955,
+                    "self": 0.0008321550012624357,
                     "children": {
                         "RLTrainer._checkpoint": {
-                            "total": 0.0542275609986973,
+                            "total": 0.057304232999740634,
                             "count": 1,
-                            "self": 0.0542275609986973
+                            "self": 0.057304232999740634
                         }
                     }
                 }

run_logs/training_status.json

@@ -2,58 +2,58 @@
     "Pyramids": {
         "checkpoints": [
             {
-                "steps": 4499924,
-                "file_path": "results/Pyramids1/Pyramids/Pyramids-4499924.onnx",
-                "reward": 1.8919999599456787,
-                "creation_time": 1688311836.9549806,
+                "steps": 7499962,
+                "file_path": "results/Pyramids1/Pyramids/Pyramids-7499962.onnx",
+                "reward": 1.8141666613519192,
+                "creation_time": 1688315345.6504664,
                 "auxillary_file_paths": [
-                    "results/Pyramids1/Pyramids/Pyramids-4499924.pt"
+                    "results/Pyramids1/Pyramids/Pyramids-7499962.pt"
                 ]
             },
             {
-                "steps": 4999897,
-                "file_path": "results/Pyramids1/Pyramids/Pyramids-4999897.onnx",
-                "reward": 1.7857499606907368,
-                "creation_time": 1688312392.617981,
+                "steps": 7999980,
+                "file_path": "results/Pyramids1/Pyramids/Pyramids-7999980.onnx",
+                "reward": 1.7659999951720238,
+                "creation_time": 1688315908.9268777,
                 "auxillary_file_paths": [
-                    "results/Pyramids1/Pyramids/Pyramids-4999897.pt"
+                    "results/Pyramids1/Pyramids/Pyramids-7999980.pt"
                 ]
             },
             {
-                "steps": 5499985,
-                "file_path": "results/Pyramids1/Pyramids/Pyramids-5499985.onnx",
-                "reward": 1.8474999740719795,
-                "creation_time": 1688312947.9638438,
+                "steps": 8499960,
+                "file_path": "results/Pyramids1/Pyramids/Pyramids-8499960.onnx",
+                "reward": 1.8003333144717746,
+                "creation_time": 1688316468.5606103,
                 "auxillary_file_paths": [
-                    "results/Pyramids1/Pyramids/Pyramids-5499985.pt"
+                    "results/Pyramids1/Pyramids/Pyramids-8499960.pt"
                 ]
             },
             {
-                "steps": 5999973,
-                "file_path": "results/Pyramids1/Pyramids/Pyramids-5999973.onnx",
-                "reward": 1.819000020623207,
-                "creation_time": 1688313505.9541245,
+                "steps": 8999992,
+                "file_path": "results/Pyramids1/Pyramids/Pyramids-8999992.onnx",
+                "reward": 1.7987499684095383,
+                "creation_time": 1688317030.6443872,
                 "auxillary_file_paths": [
-                    "results/Pyramids1/Pyramids/Pyramids-5999973.pt"
+                    "results/Pyramids1/Pyramids/Pyramids-8999992.pt"
                 ]
             },
             {
-                "steps": 6000101,
-                "file_path": "results/Pyramids1/Pyramids/Pyramids-6000101.onnx",
-                "reward": 1.819000020623207,
-                "creation_time": 1688313506.0156755,
+                "steps": 9000120,
+                "file_path": "results/Pyramids1/Pyramids/Pyramids-9000120.onnx",
+                "reward": 1.7987499684095383,
+                "creation_time": 1688317030.7087064,
                 "auxillary_file_paths": [
-                    "results/Pyramids1/Pyramids/Pyramids-6000101.pt"
+                    "results/Pyramids1/Pyramids/Pyramids-9000120.pt"
                 ]
             }
         ],
         "final_checkpoint": {
-            "steps": 6000101,
+            "steps": 9000120,
             "file_path": "results/Pyramids1/Pyramids.onnx",
-            "reward": 1.819000020623207,
-            "creation_time": 1688313506.0156755,
+            "reward": 1.7987499684095383,
+            "creation_time": 1688317030.7087064,
             "auxillary_file_paths": [
-                "results/Pyramids1/Pyramids/Pyramids-6000101.pt"
+                "results/Pyramids1/Pyramids/Pyramids-9000120.pt"
             ]
         }
     },