6 million training steps

Pyramids.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b926eaf2caaeb26fd4befcbd84ddba118e2330853d9b8e7121692f49c84661d1
+oid sha256:2dd9cb774f76b889fc11c90f247a5e500de6644b138bd4f14628e9dc4067a7b2
 size 1418184

Pyramids/Pyramids-4499924.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:378128ed0467e1970be70367370db36ff770a01e40d4dfdf5ffda2594ce87bce
+size 1418184

Pyramids/Pyramids-4499924.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb44aec8b2d488bf270445911bd26b74f6ec4e10164bbc9ef409704bcbcb9b40
+size 8650990

Pyramids/Pyramids-4999897.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef9b346b450ada02f5bcd2b4e156a99f412df48ae0c519b4a7b55e16e0790e5e
+size 1418184

Pyramids/Pyramids-4999897.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d161ebf1d92b6965fcbb81038cc02aa816d7f65ab66d88f40fadd33cc0f3e30f
+size 8650990

Pyramids/Pyramids-5499985.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b1cab87a917bea2ba013bc28bca08ba36953f934e0dd07a30271c3a4aeadeb3
+size 1418184

Pyramids/Pyramids-5499985.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64ed6b81f3c5f14ea24f8d66c17ad212d2ae8e1e795b0a41ecff2744234bd1b9
+size 8650990

Pyramids/Pyramids-5999973.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2dd9cb774f76b889fc11c90f247a5e500de6644b138bd4f14628e9dc4067a7b2
+size 1418184

Pyramids/Pyramids-5999973.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93c927564cf572b0f39e34fb84f8109f9fadc6ffeada35d56c8866929a1f7c39
+size 8650990

Pyramids/Pyramids-6000101.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2dd9cb774f76b889fc11c90f247a5e500de6644b138bd4f14628e9dc4067a7b2
+size 1418184

Pyramids/Pyramids-6000101.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:816e7f538205288ee7fba318720249a2824e6bef6327ca9d8a1274623aa41df7
+size 8650990

Pyramids/checkpoint.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9095e2296582aa72cd8cd8c8ac473f92c7259a6230ef7ea8421afa587884367c
+oid sha256:816e7f538205288ee7fba318720249a2824e6bef6327ca9d8a1274623aa41df7
 size 8650990

Pyramids/events.out.tfevents.1688310182.sekhmet.27080.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e829b97792611a2ab771660aeffcb1b911ffc4fd335a4b296011f91762aa748
+size 851539

config.json

@@ -1 +1 @@
-{"default_settings": null, "behaviors": {"Pyramids": {"trainer_type": "ppo", "hyperparameters": {"batch_size": 128, "buffer_size": 2048, "learning_rate": 0.0003, "beta": 0.01, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 3, "shared_critic": false, "learning_rate_schedule": "linear", "beta_schedule": "linear", "epsilon_schedule": "linear"}, "checkpoint_interval": 500000, "network_settings": {"normalize": false, "hidden_units": 512, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.99, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}, "rnd": {"gamma": 0.99, "strength": 0.01, "network_settings": {"normalize": false, "hidden_units": 64, "num_layers": 3, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "learning_rate": 0.0001, "encoding_size": null}}, "init_path": null, "keep_checkpoints": 5, "even_checkpoints": false, "max_steps": 3000000, "time_horizon": 128, "summary_freq": 30000, "threaded": false, "self_play": null, "behavioral_cloning": null}}, "env_settings": {"env_path": "training-envs-executables/linux/Pyramids/Pyramids", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true}, "environment_parameters": null, "checkpoint_settings": {"run_id": "Pyramids1", "initialize_from": null, "load_model": false, "resume": true, "force": false, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}
+{"default_settings": null, "behaviors": {"Pyramids": {"trainer_type": "ppo", "hyperparameters": {"batch_size": 128, "buffer_size": 2048, "learning_rate": 0.0003, "beta": 0.01, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 3, "shared_critic": false, "learning_rate_schedule": "linear", "beta_schedule": "linear", "epsilon_schedule": "linear"}, "checkpoint_interval": 500000, "network_settings": {"normalize": false, "hidden_units": 512, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.99, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}, "rnd": {"gamma": 0.99, "strength": 0.01, "network_settings": {"normalize": false, "hidden_units": 64, "num_layers": 3, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "learning_rate": 0.0001, "encoding_size": null}}, "init_path": null, "keep_checkpoints": 5, "even_checkpoints": false, "max_steps": 6000000, "time_horizon": 128, "summary_freq": 30000, "threaded": false, "self_play": null, "behavioral_cloning": null}}, "env_settings": {"env_path": "training-envs-executables/linux/Pyramids/Pyramids", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true}, "environment_parameters": null, "checkpoint_settings": {"run_id": "Pyramids1", "initialize_from": null, "load_model": false, "resume": true, "force": false, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}

configuration.yaml

@@ -51,7 +51,7 @@ behaviors:
     init_path: null
     keep_checkpoints: 5
     even_checkpoints: false
-    max_steps: 3000000
+    max_steps: 6000000
     time_horizon: 128
     summary_freq: 30000
     threaded: false

run_logs/Player-0.log

@@ -1,7 +1,7 @@
 Mono path[0] = '/home/magnus/src/huggingface-reinforcement-learning-course/ml-agents/training-envs-executables/linux/Pyramids/Pyramids_Data/Managed'
 Mono config path = '/home/magnus/src/huggingface-reinforcement-learning-course/ml-agents/training-envs-executables/linux/Pyramids/Pyramids_Data/MonoBleedingEdge/etc'
 Found 1 interfaces on host : 0) 192.168.1.111
-Multi-casting "[IP] 192.168.1.111 [Port] 55309 [Flags] 2 [Guid] 4058568002 [EditorId] 764847374 [Version] 1048832 [Id] LinuxPlayer(13,192.168.1.111) [Debug] 0 [PackageName] LinuxPlayer [ProjectName] UnityEnvironment" to [225.0.0.222:54997]...
+Multi-casting "[IP] 192.168.1.111 [Port] 55384 [Flags] 2 [Guid] 1505190532 [EditorId] 764847374 [Version] 1048832 [Id] LinuxPlayer(13,192.168.1.111) [Debug] 0 [PackageName] LinuxPlayer [ProjectName] UnityEnvironment" to [225.0.0.222:54997]...
 Preloaded 'lib_burst_generated.so'
 Preloaded 'libgrpc_csharp_ext.x64.so'
 Initialize engine version: 2021.3.5f1 (40eb3a945986)
@@ -17,7 +17,7 @@ Forced to initialize FMOD to to the device driver's system output rate 48000, th
 FMOD failed to initialize the output device.: "Error initializing output device. " (60)
 FMOD initialized on nosound output
 Begin MonoManager ReloadAssembly
-- Completed reload, in  0.086 seconds
+- Completed reload, in  0.062 seconds
 ERROR: Shader Sprites/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Sprites/Mask shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Legacy Shaders/VertexLit shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
@@ -49,7 +49,7 @@ ERROR: Shader Standard shader is not supported on this GPU (none of subshaders/f
 WARNING: Shader Unsupported: 'Standard' - All subshaders removed
 WARNING: Shader Did you use #pragma only_renderers and omit this platform?
 WARNING: Shader If subshaders removal was intentional, you may have forgotten turning Fallback off?
-UnloadTime: 0.507405 ms
+UnloadTime: 0.564000 ms
 ERROR: Shader UI/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 requesting resize 84 x 84
 Setting up 6 worker threads for Enlighten.
@@ -57,7 +57,7 @@ PlayerConnection::CleanupMemory Statistics:
 [ALLOC_TEMP_TLS] TLS Allocator
   StackAllocators : 
     [ALLOC_TEMP_MAIN]
-      Peak usage frame count: [16.0 KB-32.0 KB]: 9905 frames, [32.0 KB-64.0 KB]: 18256 frames, [64.0 KB-128.0 KB]: 9357 frames, [2.0 MB-4.0 MB]: 1 frames
+      Peak usage frame count: [16.0 KB-32.0 KB]: 8492 frames, [32.0 KB-64.0 KB]: 27079 frames, [64.0 KB-128.0 KB]: 20698 frames, [2.0 MB-4.0 MB]: 1 frames
       Initial Block Size 4.0 MB
       Current Block Size 4.0 MB
       Peak Allocated Bytes 2.0 MB
@@ -180,7 +180,7 @@ PlayerConnection::CleanupMemory Statistics:
     [ALLOC_TEMP_Job.Worker 2]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
-      Peak Allocated Bytes 0.8 KB
+      Peak Allocated Bytes 0 B
       Overflow Count 0
     [ALLOC_TEMP_Background Job.Worker 0]
       Initial Block Size 32.0 KB
@@ -192,24 +192,29 @@ PlayerConnection::CleanupMemory Statistics:
       Current Block Size 32.0 KB
       Peak Allocated Bytes 0 B
       Overflow Count 0
-    [ALLOC_TEMP_Job.Worker 0]
+    [ALLOC_TEMP_Job.Worker 6]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
       Peak Allocated Bytes 0 B
       Overflow Count 0
+    [ALLOC_TEMP_Job.Worker 0]
+      Initial Block Size 256.0 KB
+      Current Block Size 256.0 KB
+      Peak Allocated Bytes 0.8 KB
+      Overflow Count 0
     [ALLOC_TEMP_Job.Worker 5]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
       Peak Allocated Bytes 0 B
       Overflow Count 0
-    [ALLOC_TEMP_Background Job.Worker 14]
+    [ALLOC_TEMP_Background Job.Worker 9]
       Initial Block Size 32.0 KB
       Current Block Size 32.0 KB
       Peak Allocated Bytes 0 B
       Overflow Count 0
-    [ALLOC_TEMP_Job.Worker 6]
-      Initial Block Size 256.0 KB
-      Current Block Size 256.0 KB
+    [ALLOC_TEMP_Background Job.Worker 14]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
       Peak Allocated Bytes 0 B
       Overflow Count 0
     [ALLOC_TEMP_EnlightenWorker] x 6
@@ -222,11 +227,6 @@ PlayerConnection::CleanupMemory Statistics:
       Current Block Size 256.0 KB
       Peak Allocated Bytes 0 B
       Overflow Count 0
-    [ALLOC_TEMP_Background Job.Worker 9]
-      Initial Block Size 32.0 KB
-      Current Block Size 32.0 KB
-      Peak Allocated Bytes 0 B
-      Overflow Count 0
     [ALLOC_TEMP_Job.Worker 7]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
@@ -238,7 +238,7 @@ PlayerConnection::CleanupMemory Statistics:
       Peak Allocated Bytes 150 B
       Overflow Count 0
 [ALLOC_MEMORYPROFILER]
-  Peak usage frame count: [0.5 MB-1.0 MB]: 80 frames, [1.0 MB-2.0 MB]: 37439 frames
+  Peak usage frame count: [0.5 MB-1.0 MB]: 48 frames, [1.0 MB-2.0 MB]: 56222 frames
   Requested Block Size 1.0 MB
   Peak Block count 2
   Peak Allocated memory 1.1 MB
@@ -250,16 +250,16 @@ PlayerConnection::CleanupMemory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.4 MB
     [ALLOC_DEFAULT_MAIN]
-      Peak usage frame count: [8.0 MB-16.0 MB]: 1 frames, [16.0 MB-32.0 MB]: 37518 frames
+      Peak usage frame count: [8.0 MB-16.0 MB]: 1 frames, [16.0 MB-32.0 MB]: 56269 frames
       Requested Block Size 16.0 MB
       Peak Block count 2
-      Peak Allocated memory 24.0 MB
+      Peak Allocated memory 25.7 MB
       Peak Large allocation bytes 0 B
     [ALLOC_DEFAULT_THREAD]
-      Peak usage frame count: [16.0 MB-32.0 MB]: 37519 frames
+      Peak usage frame count: [16.0 MB-32.0 MB]: 56270 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
-      Peak Allocated memory 26.3 MB
+      Peak Allocated memory 25.3 MB
       Peak Large allocation bytes 16.0 MB
 [ALLOC_TEMP_JOB_1_FRAME]
   Initial Block Size 2.0 MB
@@ -288,13 +288,13 @@ PlayerConnection::CleanupMemory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.4 MB
     [ALLOC_GFX_MAIN]
-      Peak usage frame count: [32.0 KB-64.0 KB]: 37518 frames, [64.0 KB-128.0 KB]: 1 frames
+      Peak usage frame count: [32.0 KB-64.0 KB]: 56269 frames, [64.0 KB-128.0 KB]: 1 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 67.0 KB
       Peak Large allocation bytes 0 B
     [ALLOC_GFX_THREAD]
-      Peak usage frame count: [64.0 KB-128.0 KB]: 37519 frames
+      Peak usage frame count: [64.0 KB-128.0 KB]: 56270 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 67.7 KB
@@ -306,13 +306,13 @@ PlayerConnection::CleanupMemory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.4 MB
     [ALLOC_CACHEOBJECTS_MAIN]
-      Peak usage frame count: [0.5 MB-1.0 MB]: 79 frames, [1.0 MB-2.0 MB]: 37440 frames
+      Peak usage frame count: [0.5 MB-1.0 MB]: 52 frames, [1.0 MB-2.0 MB]: 56218 frames
       Requested Block Size 4.0 MB
       Peak Block count 1
       Peak Allocated memory 1.4 MB
       Peak Large allocation bytes 0 B
     [ALLOC_CACHEOBJECTS_THREAD]
-      Peak usage frame count: [1.0 MB-2.0 MB]: 37518 frames, [4.0 MB-8.0 MB]: 1 frames
+      Peak usage frame count: [1.0 MB-2.0 MB]: 56269 frames, [4.0 MB-8.0 MB]: 1 frames
       Requested Block Size 4.0 MB
       Peak Block count 2
       Peak Allocated memory 4.7 MB
@@ -324,19 +324,19 @@ PlayerConnection::CleanupMemory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.4 MB
     [ALLOC_TYPETREE_MAIN]
-      Peak usage frame count: [0-1.0 KB]: 37519 frames
+      Peak usage frame count: [0-1.0 KB]: 56270 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 1.0 KB
       Peak Large allocation bytes 0 B
     [ALLOC_TYPETREE_THREAD]
-      Peak usage frame count: [1.0 KB-2.0 KB]: 37519 frames
+      Peak usage frame count: [1.0 KB-2.0 KB]: 56270 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 1.9 KB
       Peak Large allocation bytes 0 B
 [ALLOC_PROFILER]
-  Peak usage frame count: [32.0 KB-64.0 KB]: 37519 frames
+  Peak usage frame count: [32.0 KB-64.0 KB]: 56270 frames
   Requested Block Size 16.0 MB
   Peak Block count 1
   Peak Allocated memory 34.7 KB
@@ -345,4 +345,4 @@ PlayerConnection::CleanupMemory Statistics:
       Large Block size 4.0 MB
       Used Block count 1
       Peak Allocated bytes 396 B
-##utp:{"type":"MemoryLeaks","version":2,"phase":"Immediate","time":1688309985451,"processId":26689,"allocatedMemory":2004877,"memoryLabels":[{"Default":9033},{"Permanent":1264},{"NewDelete":37399},{"Thread":34740},{"Manager":16203},{"VertexData":12},{"Geometry":280},{"Texture":16},{"Shader":69173},{"Material":24},{"GfxDevice":35248},{"Animation":304},{"Audio":3976},{"Physics":288},{"Serialization":216},{"Input":9176},{"JobScheduler":200},{"Mono":40},{"ScriptingNativeRuntime":216},{"BaseObject":1609212},{"Resource":592},{"Renderer":1936},{"Transform":48},{"File":800},{"WebCam":24},{"Culling":40},{"Terrain":953},{"Wind":24},{"String":3633},{"DynamicArray":30868},{"HashMap":7680},{"Utility":5440},{"PoolAlloc":1160},{"TypeTree":1792},{"ScriptManager":80},{"RuntimeInitializeOnLoadManager":72},{"SpriteAtlas":112},{"GI":3272},{"Unet":16},{"Director":7760},{"WebRequest":720},{"VR":45473},{"SceneManager":424},{"Video":32},{"LazyScriptCache":32},{"NativeArray":384},{"Camera":25},{"Secure":1},{"SerializationCache":624},{"APIUpdating":5872},{"Subsystems":384},{"VirtualTexturing":57552},{"AssetReference":32}]}
+##utp:{"type":"MemoryLeaks","version":2,"phase":"Immediate","time":1688313506324,"processId":27133,"allocatedMemory":2004877,"memoryLabels":[{"Default":9033},{"Permanent":1264},{"NewDelete":37399},{"Thread":34740},{"Manager":16203},{"VertexData":12},{"Geometry":280},{"Texture":16},{"Shader":69173},{"Material":24},{"GfxDevice":35248},{"Animation":304},{"Audio":3976},{"Physics":288},{"Serialization":216},{"Input":9176},{"JobScheduler":200},{"Mono":40},{"ScriptingNativeRuntime":216},{"BaseObject":1609212},{"Resource":592},{"Renderer":1936},{"Transform":48},{"File":800},{"WebCam":24},{"Culling":40},{"Terrain":953},{"Wind":24},{"String":3633},{"DynamicArray":30868},{"HashMap":7680},{"Utility":5440},{"PoolAlloc":1160},{"TypeTree":1792},{"ScriptManager":80},{"RuntimeInitializeOnLoadManager":72},{"SpriteAtlas":112},{"GI":3272},{"Unet":16},{"Director":7760},{"WebRequest":720},{"VR":45473},{"SceneManager":424},{"Video":32},{"LazyScriptCache":32},{"NativeArray":384},{"Camera":25},{"Secure":1},{"SerializationCache":624},{"APIUpdating":5872},{"Subsystems":384},{"VirtualTexturing":57552},{"AssetReference":32}]}

run_logs/timers.json

@@ -2,189 +2,189 @@
     "name": "root",
     "gauges": {
         "Pyramids.Policy.Entropy.mean": {
-            "value": 0.25150182843208313,
-            "min": 0.23986271023750305,
-            "max": 1.2359827756881714,
-            "count": 67
+            "value": 0.1430526077747345,
+            "min": 0.1262112557888031,
+            "max": 0.25186875462532043,
+            "count": 100
         },
         "Pyramids.Policy.Entropy.sum": {
-            "value": 7589.3193359375,
-            "min": 7176.6923828125,
-            "max": 37297.015625,
-            "count": 67
+            "value": 4303.0224609375,
+            "min": 3774.22119140625,
+            "max": 7556.0625,
+            "count": 100
+        },
+        "Pyramids.Environment.EpisodeLength.mean": {
+            "value": 194.8108108108108,
+            "min": 189.01986754966887,
+            "max": 330.94897959183675,
+            "count": 100
+        },
+        "Pyramids.Environment.EpisodeLength.sum": {
+            "value": 28832.0,
+            "min": 27065.0,
+            "max": 32596.0,
+            "count": 100
         },
         "Pyramids.Step.mean": {
-            "value": 2999948.0,
-            "min": 1019935.0,
-            "max": 2999948.0,
-            "count": 67
+            "value": 5999973.0,
+            "min": 3029884.0,
+            "max": 5999973.0,
+            "count": 100
         },
         "Pyramids.Step.sum": {
-            "value": 2999948.0,
-            "min": 1019935.0,
-            "max": 2999948.0,
-            "count": 67
+            "value": 5999973.0,
+            "min": 3029884.0,
+            "max": 5999973.0,
+            "count": 100
         },
         "Pyramids.Policy.ExtrinsicValueEstimate.mean": {
-            "value": 0.7547377347946167,
-            "min": -0.07056643068790436,
-            "max": 0.7929686307907104,
-            "count": 67
+            "value": 0.8933548331260681,
+            "min": 0.5950189828872681,
+            "max": 0.895928144454956,
+            "count": 100
         },
         "Pyramids.Policy.ExtrinsicValueEstimate.sum": {
-            "value": 223.40237426757812,
-            "min": -17.077075958251953,
-            "max": 235.17056274414062,
-            "count": 67
+            "value": 273.3665771484375,
+            "min": 166.60531616210938,
+            "max": 278.22900390625,
+            "count": 100
         },
         "Pyramids.Policy.RndValueEstimate.mean": {
-            "value": 0.010818109847605228,
-            "min": -2.4031267166137695,
-            "max": 14.60046100616455,
-            "count": 67
+            "value": -0.035036664456129074,
+            "min": -0.035036664456129074,
+            "max": 0.11137405037879944,
+            "count": 100
         },
         "Pyramids.Policy.RndValueEstimate.sum": {
-            "value": 3.202160596847534,
-            "min": -636.8285522460938,
-            "max": 4000.5263671875,
-            "count": 67
-        },
-        "Pyramids.Environment.EpisodeLength.mean": {
-            "value": 239.45238095238096,
-            "min": 228.2983870967742,
-            "max": 963.4545454545455,
-            "count": 67
-        },
-        "Pyramids.Environment.EpisodeLength.sum": {
-            "value": 30171.0,
-            "min": 15966.0,
-            "max": 33959.0,
-            "count": 67
+            "value": -10.721219062805176,
+            "min": -10.721219062805176,
+            "max": 33.85771179199219,
+            "count": 100
         },
         "Pyramids.Environment.CumulativeReward.mean": {
-            "value": 1.728792051237727,
-            "min": -0.7822242909760186,
-            "max": 1.7543903112171157,
-            "count": 67
+            "value": 1.7916729624408323,
+            "min": 1.5669795772706976,
+            "max": 1.805647044594771,
+            "count": 100
         },
         "Pyramids.Environment.CumulativeReward.sum": {
-            "value": 217.8277984559536,
-            "min": -25.890001580119133,
-            "max": 217.8277984559536,
-            "count": 67
+            "value": 265.1675984412432,
+            "min": 153.56399857252836,
+            "max": 277.3915982693434,
+            "count": 100
         },
         "Pyramids.Policy.ExtrinsicReward.mean": {
-            "value": 1.728792051237727,
-            "min": -0.7822242909760186,
-            "max": 1.7543903112171157,
-            "count": 67
+            "value": 1.7916729624408323,
+            "min": 1.5669795772706976,
+            "max": 1.805647044594771,
+            "count": 100
         },
         "Pyramids.Policy.ExtrinsicReward.sum": {
-            "value": 217.8277984559536,
-            "min": -25.890001580119133,
-            "max": 217.8277984559536,
-            "count": 67
+            "value": 265.1675984412432,
+            "min": 153.56399857252836,
+            "max": 277.3915982693434,
+            "count": 100
         },
         "Pyramids.Policy.RndReward.mean": {
-            "value": 0.012760552965725461,
-            "min": 0.012408850353758974,
-            "max": 0.05711968737580862,
-            "count": 67
+            "value": 0.011331361728603166,
+            "min": 0.011228123687611297,
+            "max": 0.01837677085388671,
+            "count": 100
         },
         "Pyramids.Policy.RndReward.sum": {
-            "value": 1.6078296736814082,
-            "min": 0.7649297210737132,
-            "max": 2.7060816152661573,
-            "count": 67
+            "value": 1.6770415358332684,
+            "min": 1.411525305113173,
+            "max": 2.202837912082032,
+            "count": 100
         },
         "Pyramids.Losses.PolicyLoss.mean": {
-            "value": 0.06908763900606169,
-            "min": 0.06241923725938962,
-            "max": 0.07235957070601731,
-            "count": 67
+            "value": 0.06742418237716064,
+            "min": 0.0642036380724878,
+            "max": 0.07328082145180641,
+            "count": 100
         },
         "Pyramids.Losses.PolicyLoss.sum": {
-            "value": 1.0363145850909252,
-            "min": 0.621586629538797,
-            "max": 1.0627095143136103,
-            "count": 67
+            "value": 1.0113627356574095,
+            "min": 0.9034849535867882,
+            "max": 1.0850595171938826,
+            "count": 100
         },
         "Pyramids.Losses.ValueLoss.mean": {
-            "value": 0.015010668305654286,
-            "min": 0.004781421934297236,
-            "max": 13.36772389532555,
-            "count": 67
+            "value": 0.015541635502245446,
+            "min": 0.011974255370400247,
+            "max": 0.01834407493231443,
+            "count": 100
         },
         "Pyramids.Losses.ValueLoss.sum": {
-            "value": 0.2251600245848143,
-            "min": 0.0669399070801613,
-            "max": 187.1481345345577,
-            "count": 67
+            "value": 0.2331245325336817,
+            "min": 0.16763957518560346,
+            "max": 0.256817049052402,
+            "count": 100
         },
         "Pyramids.Policy.LearningRate.mean": {
-            "value": 1.4910795030066681e-06,
-            "min": 1.4910795030066681e-06,
-            "max": 0.00019893701146545185,
-            "count": 67
+            "value": 7.487964171011104e-07,
+            "min": 7.487964171011104e-07,
+            "max": 0.00014920776812123691,
+            "count": 100
         },
         "Pyramids.Policy.LearningRate.sum": {
-            "value": 2.236619254510002e-05,
-            "min": 2.236619254510002e-05,
-            "max": 0.0028576116474629664,
-            "count": 67
+            "value": 1.1231946256516656e-05,
+            "min": 1.1231946256516656e-05,
+            "max": 0.0021263646912120333,
+            "count": 100
         },
         "Pyramids.Policy.Epsilon.mean": {
-            "value": 0.10049699333333333,
-            "min": 0.10049699333333333,
-            "max": 0.16631232592592593,
-            "count": 67
+            "value": 0.10024956555555554,
+            "min": 0.10024956555555554,
+            "max": 0.14973590595238098,
+            "count": 100
         },
         "Pyramids.Policy.Epsilon.sum": {
-            "value": 1.5074549,
-            "min": 1.4212187,
-            "max": 2.4525370333333334,
-            "count": 67
+            "value": 1.5037434833333332,
+            "min": 1.41063965,
+            "max": 2.2087879666666668,
+            "count": 100
         },
         "Pyramids.Policy.Beta.mean": {
-            "value": 5.964963400000005e-05,
-            "min": 5.964963400000005e-05,
-            "max": 0.006634601360000001,
-            "count": 67
+            "value": 3.493159899999997e-05,
+            "min": 3.493159899999997e-05,
+            "max": 0.004978617004642856,
+            "count": 100
         },
         "Pyramids.Policy.Beta.sum": {
-            "value": 0.0008947445100000008,
-            "min": 0.0008947445100000008,
-            "max": 0.09530844963,
-            "count": 67
+            "value": 0.0005239739849999996,
+            "min": 0.0005239739849999996,
+            "max": 0.07095791787,
+            "count": 100
         },
         "Pyramids.Losses.RNDLoss.mean": {
-            "value": 0.0050939214415848255,
-            "min": 0.002439801814034581,
-            "max": 0.008640534244477749,
-            "count": 67
+            "value": 0.005648714955896139,
+            "min": 0.004770483355969191,
+            "max": 0.006634328979998827,
+            "count": 100
         },
         "Pyramids.Losses.RNDLoss.sum": {
-            "value": 0.07640881836414337,
-            "min": 0.034157223999500275,
-            "max": 0.12960802018642426,
-            "count": 67
+            "value": 0.08473072201013565,
+            "min": 0.06968885660171509,
+            "max": 0.09891163557767868,
+            "count": 100
         },
         "Pyramids.IsTraining.mean": {
             "value": 1.0,
             "min": 1.0,
             "max": 1.0,
-            "count": 67
+            "count": 100
         },
         "Pyramids.IsTraining.sum": {
             "value": 1.0,
             "min": 1.0,
             "max": 1.0,
-            "count": 67
+            "count": 100
         }
     },
     "metadata": {
         "timer_format_version": "0.1.0",
-        "start_time_seconds": "1688307813",
+        "start_time_seconds": "1688310181",
         "python_version": "3.10.12 (main, Jun  9 2023, 20:10:04) [GCC 13.1.1 20230522 [revision dd36656ada05731c069ecd5b1878380294fb1f3e]]",
         "command_line_arguments": "/home/magnus/.local/venv/ml-agents/bin/mlagents-learn --resume config/ppo/PyramidsRND.yaml --env training-envs-executables/linux/Pyramids/Pyramids --run-id=Pyramids1 --no-graphics",
         "mlagents_version": "0.31.0.dev0",
@@ -192,59 +192,59 @@
         "communication_protocol_version": "1.5.0",
         "pytorch_version": "1.11.0+cu102",
         "numpy_version": "1.21.2",
-        "end_time_seconds": "1688309985"
+        "end_time_seconds": "1688313506"
     },
-    "total": 2172.1748791690006,
+    "total": 3324.868793236001,
     "count": 1,
-    "self": 0.36911092800073675,
+    "self": 0.31922269300230255,
     "children": {
         "run_training.setup": {
-            "total": 0.00738377999914519,
+            "total": 0.007570371999463532,
             "count": 1,
-            "self": 0.00738377999914519
+            "self": 0.007570371999463532
         },
         "TrainerController.start_learning": {
-            "total": 2171.7983844610008,
+            "total": 3324.542000170999,
             "count": 1,
-            "self": 1.9695416254762677,
+            "self": 3.0024852928454493,
             "children": {
                 "TrainerController._reset_env": {
-                    "total": 6.073836388000927,
+                    "total": 2.696618435000346,
                     "count": 1,
-                    "self": 6.073836388000927
+                    "self": 2.696618435000346
                 },
                 "TrainerController.advance": {
-                    "total": 2163.6984990875244,
-                    "count": 129457,
-                    "self": 1.9047759958630195,
+                    "total": 3318.7878656481535,
+                    "count": 197822,
+                    "self": 2.735950404443429,
                     "children": {
                         "env_step": {
-                            "total": 1454.348816122112,
-                            "count": 129457,
-                            "self": 1316.3867875059732,
+                            "total": 2248.670260114952,
+                            "count": 197822,
+                            "self": 2037.7029809287633,
                             "children": {
                                 "SubprocessEnvManager._take_step": {
-                                    "total": 136.72278414898938,
-                                    "count": 129457,
-                                    "self": 5.218426685016311,
+                                    "total": 209.00800150362193,
+                                    "count": 197822,
+                                    "self": 7.87247752492658,
                                     "children": {
                                         "TorchPolicy.evaluate": {
-                                            "total": 131.50435746397307,
-                                            "count": 125062,
-                                            "self": 131.50435746397307
+                                            "total": 201.13552397869535,
+                                            "count": 187566,
+                                            "self": 201.13552397869535
                                         }
                                     }
                                 },
                                 "workers": {
-                                    "total": 1.2392444671495468,
-                                    "count": 129457,
+                                    "total": 1.9592776825666078,
+                                    "count": 197822,
                                     "self": 0.0,
                                     "children": {
                                         "worker_root": {
-                                            "total": 2169.3375711670924,
-                                            "count": 129457,
+                                            "total": 3320.8473949115905,
+                                            "count": 197822,
                                             "is_parallel": true,
-                                            "self": 981.2120239229353,
+                                            "self": 1477.8273678632904,
                                             "children": {
                                                 "run_training.setup": {
                                                     "total": 0.0,
@@ -253,48 +253,48 @@
                                                     "self": 0.0,
                                                     "children": {
                                                         "steps_from_proto": {
-                                                            "total": 0.001912709998578066,
+                                                            "total": 0.0009898610005620867,
                                                             "count": 1,
                                                             "is_parallel": true,
-                                                            "self": 0.00035811499947158154,
+                                                            "self": 0.00026216399601253215,
                                                             "children": {
                                                                 "_process_rank_one_or_two_observation": {
-                                                                    "total": 0.0015545949991064845,
+                                                                    "total": 0.0007276970045495545,
                                                                     "count": 8,
                                                                     "is_parallel": true,
-                                                                    "self": 0.0015545949991064845
+                                                                    "self": 0.0007276970045495545
                                                                 }
                                                             }
                                                         },
                                                         "UnityEnvironment.step": {
-                                                            "total": 0.021976440000798902,
+                                                            "total": 0.02377290300137247,
                                                             "count": 1,
                                                             "is_parallel": true,
-                                                            "self": 0.00020695800048997626,
+                                                            "self": 0.0002492580024409108,
                                                             "children": {
                                                                 "UnityEnvironment._generate_step_input": {
-                                                                    "total": 0.0001730149997456465,
+                                                                    "total": 0.0002108610005961964,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.0001730149997456465
+                                                                    "self": 0.0002108610005961964
                                                                 },
                                                                 "communicator.exchange": {
-                                                                    "total": 0.02092029800041928,
+                                                                    "total": 0.022614385999986553,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.02092029800041928
+                                                                    "self": 0.022614385999986553
                                                                 },
                                                                 "steps_from_proto": {
-                                                                    "total": 0.0006761690001440002,
+                                                                    "total": 0.0006983979983488098,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.00014743599967914633,
+                                                                    "self": 0.00014349500088428613,
                                                                     "children": {
                                                                         "_process_rank_one_or_two_observation": {
-                                                                            "total": 0.0005287330004648538,
+                                                                            "total": 0.0005549029974645237,
                                                                             "count": 8,
                                                                             "is_parallel": true,
-                                                                            "self": 0.0005287330004648538
+                                                                            "self": 0.0005549029974645237
                                                                         }
                                                                     }
                                                                 }
@@ -303,34 +303,34 @@
                                                     }
                                                 },
                                                 "UnityEnvironment.step": {
-                                                    "total": 1188.1255472441571,
-                                                    "count": 129456,
+                                                    "total": 1843.0200270483,
+                                                    "count": 197821,
                                                     "is_parallel": true,
-                                                    "self": 38.17118912143269,
+                                                    "self": 58.0508139172689,
                                                     "children": {
                                                         "UnityEnvironment._generate_step_input": {
-                                                            "total": 19.108109347025675,
-                                                            "count": 129456,
+                                                            "total": 29.1573587008279,
+                                                            "count": 197821,
                                                             "is_parallel": true,
-                                                            "self": 19.108109347025675
+                                                            "self": 29.1573587008279
                                                         },
                                                         "communicator.exchange": {
-                                                            "total": 1025.0177220240184,
-                                                            "count": 129456,
+                                                            "total": 1597.5823520508966,
+                                                            "count": 197821,
                                                             "is_parallel": true,
-                                                            "self": 1025.0177220240184
+                                                            "self": 1597.5823520508966
                                                         },
                                                         "steps_from_proto": {
-                                                            "total": 105.82852675168033,
-                                                            "count": 129456,
+                                                            "total": 158.22950237930672,
+                                                            "count": 197821,
                                                             "is_parallel": true,
-                                                            "self": 20.39225625698782,
+                                                            "self": 29.99255649505176,
                                                             "children": {
                                                                 "_process_rank_one_or_two_observation": {
-                                                                    "total": 85.43627049469251,
-                                                                    "count": 1035648,
+                                                                    "total": 128.23694588425496,
+                                                                    "count": 1582568,
                                                                     "is_parallel": true,
-                                                                    "self": 85.43627049469251
+                                                                    "self": 128.23694588425496
                                                                 }
                                                             }
                                                         }
@@ -343,31 +343,31 @@
                             }
                         },
                         "trainer_advance": {
-                            "total": 707.4449069695493,
-                            "count": 129457,
-                            "self": 3.2045918964086013,
+                            "total": 1067.3816551287582,
+                            "count": 197822,
+                            "self": 4.800970576938198,
                             "children": {
                                 "process_trajectory": {
-                                    "total": 128.41997512312264,
-                                    "count": 129457,
-                                    "self": 128.17507259811828,
+                                    "total": 195.43404567573816,
+                                    "count": 197822,
+                                    "self": 194.985590716733,
                                     "children": {
                                         "RLTrainer._checkpoint": {
-                                            "total": 0.24490252500436327,
-                                            "count": 4,
-                                            "self": 0.24490252500436327
+                                            "total": 0.4484549590051756,
+                                            "count": 6,
+                                            "self": 0.4484549590051756
                                         }
                                     }
                                 },
                                 "_update_policy": {
-                                    "total": 575.820339950018,
-                                    "count": 945,
-                                    "self": 372.8879248711728,
+                                    "total": 867.1466388760819,
+                                    "count": 1421,
+                                    "self": 556.6953621563243,
                                     "children": {
                                         "TorchPPOOptimizer.update": {
-                                            "total": 202.93241507884522,
-                                            "count": 45537,
-                                            "self": 202.93241507884522
+                                            "total": 310.4512767197575,
+                                            "count": 68394,
+                                            "self": 310.4512767197575
                                         }
                                     }
                                 }
@@ -376,19 +376,19 @@
                     }
                 },
                 "trainer_threads": {
-                    "total": 6.009995558997616e-07,
+                    "total": 6.230002327356488e-07,
                     "count": 1,
-                    "self": 6.009995558997616e-07
+                    "self": 6.230002327356488e-07
                 },
                 "TrainerController._save_models": {
-                    "total": 0.056506758999603335,
+                    "total": 0.055030171999533195,
                     "count": 1,
-                    "self": 0.0007781290005368646,
+                    "self": 0.0008026110008358955,
                     "children": {
                         "RLTrainer._checkpoint": {
-                            "total": 0.05572862999906647,
+                            "total": 0.0542275609986973,
                             "count": 1,
-                            "self": 0.05572862999906647
+                            "self": 0.0542275609986973
                         }
                     }
                 }

run_logs/training_status.json

@@ -2,58 +2,58 @@
     "Pyramids": {
         "checkpoints": [
             {
-                "steps": 1499964,
-                "file_path": "results/Pyramids1/Pyramids/Pyramids-1499964.onnx",
-                "reward": 0.16919998079538345,
-                "creation_time": 1688308375.1250827,
+                "steps": 4499924,
+                "file_path": "results/Pyramids1/Pyramids/Pyramids-4499924.onnx",
+                "reward": 1.8919999599456787,
+                "creation_time": 1688311836.9549806,
                 "auxillary_file_paths": [
-                    "results/Pyramids1/Pyramids/Pyramids-1499964.pt"
+                    "results/Pyramids1/Pyramids/Pyramids-4499924.pt"
                 ]
             },
             {
-                "steps": 1999973,
-                "file_path": "results/Pyramids1/Pyramids/Pyramids-1999973.onnx",
-                "reward": null,
-                "creation_time": 1688308898.380308,
+                "steps": 4999897,
+                "file_path": "results/Pyramids1/Pyramids/Pyramids-4999897.onnx",
+                "reward": 1.7857499606907368,
+                "creation_time": 1688312392.617981,
                 "auxillary_file_paths": [
-                    "results/Pyramids1/Pyramids/Pyramids-1999973.pt"
+                    "results/Pyramids1/Pyramids/Pyramids-4999897.pt"
                 ]
             },
             {
-                "steps": 2499922,
-                "file_path": "results/Pyramids1/Pyramids/Pyramids-2499922.onnx",
-                "reward": 1.8329999521374702,
-                "creation_time": 1688309441.457185,
+                "steps": 5499985,
+                "file_path": "results/Pyramids1/Pyramids/Pyramids-5499985.onnx",
+                "reward": 1.8474999740719795,
+                "creation_time": 1688312947.9638438,
                 "auxillary_file_paths": [
-                    "results/Pyramids1/Pyramids/Pyramids-2499922.pt"
+                    "results/Pyramids1/Pyramids/Pyramids-5499985.pt"
                 ]
             },
             {
-                "steps": 2999948,
-                "file_path": "results/Pyramids1/Pyramids/Pyramids-2999948.onnx",
-                "reward": 1.545999988913536,
-                "creation_time": 1688309985.0642164,
+                "steps": 5999973,
+                "file_path": "results/Pyramids1/Pyramids/Pyramids-5999973.onnx",
+                "reward": 1.819000020623207,
+                "creation_time": 1688313505.9541245,
                 "auxillary_file_paths": [
-                    "results/Pyramids1/Pyramids/Pyramids-2999948.pt"
+                    "results/Pyramids1/Pyramids/Pyramids-5999973.pt"
                 ]
             },
             {
-                "steps": 3000076,
-                "file_path": "results/Pyramids1/Pyramids/Pyramids-3000076.onnx",
-                "reward": 1.545999988913536,
-                "creation_time": 1688309985.1272125,
+                "steps": 6000101,
+                "file_path": "results/Pyramids1/Pyramids/Pyramids-6000101.onnx",
+                "reward": 1.819000020623207,
+                "creation_time": 1688313506.0156755,
                 "auxillary_file_paths": [
-                    "results/Pyramids1/Pyramids/Pyramids-3000076.pt"
+                    "results/Pyramids1/Pyramids/Pyramids-6000101.pt"
                 ]
             }
         ],
         "final_checkpoint": {
-            "steps": 3000076,
+            "steps": 6000101,
             "file_path": "results/Pyramids1/Pyramids.onnx",
-            "reward": 1.545999988913536,
-            "creation_time": 1688309985.1272125,
+            "reward": 1.819000020623207,
+            "creation_time": 1688313506.0156755,
             "auxillary_file_paths": [
-                "results/Pyramids1/Pyramids/Pyramids-3000076.pt"
+                "results/Pyramids1/Pyramids/Pyramids-6000101.pt"
             ]
         }
     },