More steps

Pyramids.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ec43f872f8a2ba76ba90fb40567cf96cb000e1c292fb406cef5770a629959ef
+oid sha256:61addf8b75d96e03986c3aca29b5277d018a453440f0260269cd24651bfd05f3
 size 1420238

Pyramids/Pyramids-1499961.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ed8ea0572b8ecd019838d0cf94182b7049e85ef8e8f6a202bc38806e5a12183
+size 1420238

Pyramids/Pyramids-1499961.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cf610f1f738d69307c3a15e48b8e604d608c62396bbe03c302641d2e02943e5
+size 8655228

Pyramids/Pyramids-1999982.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c5ccf299be5904f8572aad0d3ee91092a1d2ad90e1014b10ecd43fca04b5a30
+size 1420238

Pyramids/Pyramids-1999982.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70ac272c1c656d3d658e96a6a355e31fe07023738f3413861ff9c773dc5e54dc
+size 8655228

Pyramids/Pyramids-2499899.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78f4e5453ba2be78124aef6c3f481d677198bc313a169e40e1a2da76a79b2655
+size 1420238

Pyramids/Pyramids-2499899.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51d5a77ec8cde3992ef2d32449e39dbd77ac75534002910ee57bf8e6bd1d3e4d
+size 8655228

Pyramids/Pyramids-2999991.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61addf8b75d96e03986c3aca29b5277d018a453440f0260269cd24651bfd05f3
+size 1420238

Pyramids/Pyramids-2999991.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bd98e0aa334c635de2b33302ed61c9109318ae3f2dfbf2a380154d19ae37fc8
+size 8655228

Pyramids/Pyramids-3000119.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61addf8b75d96e03986c3aca29b5277d018a453440f0260269cd24651bfd05f3
+size 1420238

Pyramids/Pyramids-3000119.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:387d0860893d674449d23d714bf347a318726b4aec7e2f5ada66e5b5fab339a1
+size 8655228

Pyramids/checkpoint.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f071394f6be8159fda1f280f7af6f3e88914f18f9c8a0f8fa3f7dbe9e107b21
+oid sha256:04a655294bfe8aeb41e84d04d53f5adc2871eb663ef1476e7489859921179e2f
 size 8654754

Pyramids/events.out.tfevents.1726851847.Spyros.56222.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:800891ddc0023c9f01df146eef6251239639c765fe8f152b00f9d72a3b72c674
+size 942123

config.json

@@ -1 +1 @@
-{"default_settings": null, "behaviors": {"Pyramids": {"trainer_type": "ppo", "hyperparameters": {"batch_size": 128, "buffer_size": 2048, "learning_rate": 0.0003, "beta": 0.01, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 3, "shared_critic": false, "learning_rate_schedule": "linear", "beta_schedule": "linear", "epsilon_schedule": "linear"}, "checkpoint_interval": 500000, "network_settings": {"normalize": false, "hidden_units": 512, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.99, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}, "rnd": {"gamma": 0.99, "strength": 0.01, "network_settings": {"normalize": false, "hidden_units": 64, "num_layers": 3, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "learning_rate": 0.0001, "encoding_size": null}}, "init_path": null, "keep_checkpoints": 5, "even_checkpoints": false, "max_steps": 1000000, "time_horizon": 128, "summary_freq": 30000, "threaded": false, "self_play": null, "behavioral_cloning": null}}, "env_settings": {"env_path": "./training-envs-executables/linux/Pyramids/Pyramids", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "timeout_wait": 60, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true, "no_graphics_monitor": false}, "environment_parameters": null, "checkpoint_settings": {"run_id": "Pyramids Training", "initialize_from": null, "load_model": false, "resume": false, "force": true, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}
+{"default_settings": null, "behaviors": {"Pyramids": {"trainer_type": "ppo", "hyperparameters": {"batch_size": 128, "buffer_size": 2048, "learning_rate": 0.0003, "beta": 0.01, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 3, "shared_critic": false, "learning_rate_schedule": "linear", "beta_schedule": "linear", "epsilon_schedule": "linear"}, "checkpoint_interval": 500000, "network_settings": {"normalize": false, "hidden_units": 512, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.99, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}, "rnd": {"gamma": 0.99, "strength": 0.01, "network_settings": {"normalize": false, "hidden_units": 64, "num_layers": 3, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "learning_rate": 0.0001, "encoding_size": null}}, "init_path": null, "keep_checkpoints": 5, "even_checkpoints": false, "max_steps": 3000000, "time_horizon": 128, "summary_freq": 30000, "threaded": false, "self_play": null, "behavioral_cloning": null}}, "env_settings": {"env_path": "./training-envs-executables/linux/Pyramids/Pyramids", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "timeout_wait": 60, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true, "no_graphics_monitor": false}, "environment_parameters": null, "checkpoint_settings": {"run_id": "Pyramids Training", "initialize_from": null, "load_model": false, "resume": false, "force": true, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}

configuration.yaml

@@ -51,7 +51,7 @@ behaviors:
     init_path: null
     keep_checkpoints: 5
     even_checkpoints: false
-    max_steps: 1000000
+    max_steps: 3000000
     time_horizon: 128
     summary_freq: 30000
     threaded: false

run_logs/Player-0.log

@@ -1,7 +1,7 @@
 Mono path[0] = '/home/smitsis/deep_rl/unit5/ml-agents/training-envs-executables/linux/Pyramids/Pyramids_Data/Managed'
 Mono config path = '/home/smitsis/deep_rl/unit5/ml-agents/training-envs-executables/linux/Pyramids/Pyramids_Data/MonoBleedingEdge/etc'
 Found 2 interfaces on host : 0) 10.255.255.254 1) 172.17.157.33
-Multi-casting "[IP] 10.255.255.254 [Port] 55215 [Flags] 2 [Guid] 1965059594 [EditorId] 764847374 [Version] 1048832 [Id] LinuxPlayer(13,172.17.157.33) [Debug] 0 [PackageName] LinuxPlayer [ProjectName] UnityEnvironment" to [225.0.0.222:54997]...
+Multi-casting "[IP] 10.255.255.254 [Port] 55050 [Flags] 2 [Guid] 816111151 [EditorId] 764847374 [Version] 1048832 [Id] LinuxPlayer(13,172.17.157.33) [Debug] 0 [PackageName] LinuxPlayer [ProjectName] UnityEnvironment" to [225.0.0.222:54997]...
 Preloaded 'lib_burst_generated.so'
 Preloaded 'libgrpc_csharp_ext.x64.so'
 Initialize engine version: 2021.3.5f1 (40eb3a945986)
@@ -33,7 +33,7 @@ ALSA lib pcm.c:2664:(snd_pcm_open_noupdate) Unknown PCM default
 FMOD failed to initialize the output device.: "Error initializing output device. " (60)
 FMOD initialized on nosound output
 Begin MonoManager ReloadAssembly
-- Completed reload, in  0.147 seconds
+- Completed reload, in  0.107 seconds
 ERROR: Shader Sprites/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Sprites/Mask shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Legacy Shaders/VertexLit shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
@@ -65,7 +65,7 @@ ERROR: Shader Standard shader is not supported on this GPU (none of subshaders/f
 WARNING: Shader Unsupported: 'Standard' - All subshaders removed
 WARNING: Shader Did you use #pragma only_renderers and omit this platform?
 WARNING: Shader If subshaders removal was intentional, you may have forgotten turning Fallback off?
-UnloadTime: 1.480281 ms
+UnloadTime: 1.351582 ms
 ERROR: Shader UI/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 requesting resize 84 x 84
 Setting up 10 worker threads for Enlighten.
@@ -73,7 +73,7 @@ PlayerConnection::CleanupMemory Statistics:
 [ALLOC_TEMP_TLS] TLS Allocator
   StackAllocators : 
     [ALLOC_TEMP_MAIN]
-      Peak usage frame count: [16.0 KB-32.0 KB]: 10192 frames, [32.0 KB-64.0 KB]: 7094 frames, [64.0 KB-128.0 KB]: 1482 frames, [2.0 MB-4.0 MB]: 1 frames
+      Peak usage frame count: [16.0 KB-32.0 KB]: 8689 frames, [32.0 KB-64.0 KB]: 35188 frames, [64.0 KB-128.0 KB]: 12386 frames, [2.0 MB-4.0 MB]: 1 frames
       Initial Block Size 4.0 MB
       Current Block Size 4.0 MB
       Peak Allocated Bytes 2.0 MB
@@ -176,33 +176,28 @@ PlayerConnection::CleanupMemory Statistics:
     [ALLOC_TEMP_Job.Worker 2]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
-      Peak Allocated Bytes 0.8 KB
+      Peak Allocated Bytes 0 B
       Overflow Count 0
     [ALLOC_TEMP_Job.Worker 11]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
-      Peak Allocated Bytes 0 B
+      Peak Allocated Bytes 0.8 KB
       Overflow Count 0
     [ALLOC_TEMP_Job.Worker 6]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
       Peak Allocated Bytes 0 B
       Overflow Count 0
-    [ALLOC_TEMP_Job.Worker 0]
+    [ALLOC_TEMP_Job.Worker 5]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
       Peak Allocated Bytes 0 B
       Overflow Count 0
-    [ALLOC_TEMP_Job.Worker 5]
+    [ALLOC_TEMP_Job.Worker 0]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
       Peak Allocated Bytes 0 B
       Overflow Count 0
-    [ALLOC_TEMP_Background Job.Worker 9]
-      Initial Block Size 32.0 KB
-      Current Block Size 32.0 KB
-      Peak Allocated Bytes 0 B
-      Overflow Count 0
     [ALLOC_TEMP_EnlightenWorker] x 10
       Initial Block Size 64.0 KB
       Current Block Size 64.0 KB
@@ -213,6 +208,11 @@ PlayerConnection::CleanupMemory Statistics:
       Current Block Size 256.0 KB
       Peak Allocated Bytes 0 B
       Overflow Count 0
+    [ALLOC_TEMP_Background Job.Worker 9]
+      Initial Block Size 32.0 KB
+      Current Block Size 32.0 KB
+      Peak Allocated Bytes 0 B
+      Overflow Count 0
     [ALLOC_TEMP_Loading.AsyncRead]
       Initial Block Size 64.0 KB
       Current Block Size 64.0 KB
@@ -294,28 +294,28 @@ PlayerConnection::CleanupMemory Statistics:
       Peak Allocated Bytes 0 B
       Overflow Count 0
 [ALLOC_MEMORYPROFILER]
-  Peak usage frame count: [0.5 MB-1.0 MB]: 299 frames, [1.0 MB-2.0 MB]: 18470 frames
+  Peak usage frame count: [0.5 MB-1.0 MB]: 299 frames, [1.0 MB-2.0 MB]: 55965 frames
   Requested Block Size 1.0 MB
   Peak Block count 2
   Peak Allocated memory 1.6 MB
   Peak Large allocation bytes 0 B
 [ALLOC_DEFAULT] Dual Thread Allocator
-  Peak main deferred allocation count 13249
+  Peak main deferred allocation count 11683
     [ALLOC_BUCKET]
       Large Block size 4.0 MB
       Used Block count 1
       Peak Allocated bytes 2.0 MB
     [ALLOC_DEFAULT_MAIN]
-      Peak usage frame count: [8.0 MB-16.0 MB]: 1 frames, [16.0 MB-32.0 MB]: 18768 frames
+      Peak usage frame count: [8.0 MB-16.0 MB]: 1 frames, [16.0 MB-32.0 MB]: 56263 frames
       Requested Block Size 16.0 MB
       Peak Block count 3
-      Peak Allocated memory 31.6 MB
+      Peak Allocated memory 31.2 MB
       Peak Large allocation bytes 0 B
     [ALLOC_DEFAULT_THREAD]
-      Peak usage frame count: [16.0 MB-32.0 MB]: 18769 frames
+      Peak usage frame count: [16.0 MB-32.0 MB]: 56264 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
-      Peak Allocated memory 30.0 MB
+      Peak Allocated memory 30.7 MB
       Peak Large allocation bytes 16.0 MB
 [ALLOC_TEMP_JOB_1_FRAME]
   Initial Block Size 2.0 MB
@@ -344,13 +344,13 @@ PlayerConnection::CleanupMemory Statistics:
       Used Block count 1
       Peak Allocated bytes 2.0 MB
     [ALLOC_GFX_MAIN]
-      Peak usage frame count: [32.0 KB-64.0 KB]: 18768 frames, [64.0 KB-128.0 KB]: 1 frames
+      Peak usage frame count: [32.0 KB-64.0 KB]: 56263 frames, [64.0 KB-128.0 KB]: 1 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 67.0 KB
       Peak Large allocation bytes 0 B
     [ALLOC_GFX_THREAD]
-      Peak usage frame count: [64.0 KB-128.0 KB]: 18769 frames
+      Peak usage frame count: [64.0 KB-128.0 KB]: 56264 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 67.7 KB
@@ -362,13 +362,13 @@ PlayerConnection::CleanupMemory Statistics:
       Used Block count 1
       Peak Allocated bytes 2.0 MB
     [ALLOC_CACHEOBJECTS_MAIN]
-      Peak usage frame count: [0.5 MB-1.0 MB]: 299 frames, [1.0 MB-2.0 MB]: 18470 frames
+      Peak usage frame count: [0.5 MB-1.0 MB]: 299 frames, [1.0 MB-2.0 MB]: 55965 frames
       Requested Block Size 4.0 MB
       Peak Block count 1
       Peak Allocated memory 1.4 MB
       Peak Large allocation bytes 0 B
     [ALLOC_CACHEOBJECTS_THREAD]
-      Peak usage frame count: [1.0 MB-2.0 MB]: 18768 frames, [4.0 MB-8.0 MB]: 1 frames
+      Peak usage frame count: [1.0 MB-2.0 MB]: 56263 frames, [4.0 MB-8.0 MB]: 1 frames
       Requested Block Size 4.0 MB
       Peak Block count 2
       Peak Allocated memory 4.7 MB
@@ -380,19 +380,19 @@ PlayerConnection::CleanupMemory Statistics:
       Used Block count 1
       Peak Allocated bytes 2.0 MB
     [ALLOC_TYPETREE_MAIN]
-      Peak usage frame count: [0-1.0 KB]: 18769 frames
+      Peak usage frame count: [0-1.0 KB]: 56264 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 1.0 KB
       Peak Large allocation bytes 0 B
     [ALLOC_TYPETREE_THREAD]
-      Peak usage frame count: [1.0 KB-2.0 KB]: 18769 frames
+      Peak usage frame count: [1.0 KB-2.0 KB]: 56264 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 1.9 KB
       Peak Large allocation bytes 0 B
 [ALLOC_PROFILER]
-  Peak usage frame count: [32.0 KB-64.0 KB]: 18769 frames
+  Peak usage frame count: [32.0 KB-64.0 KB]: 56264 frames
   Requested Block Size 16.0 MB
   Peak Block count 1
   Peak Allocated memory 43.8 KB
@@ -401,4 +401,4 @@ PlayerConnection::CleanupMemory Statistics:
       Large Block size 4.0 MB
       Used Block count 1
       Peak Allocated bytes 396 B
-##utp:{"type":"MemoryLeaks","version":2,"phase":"Immediate","time":1726847151659,"processId":31458,"allocatedMemory":2209098,"memoryLabels":[{"Default":9033},{"Permanent":1264},{"NewDelete":227599},{"Thread":34964},{"Manager":26827},{"VertexData":12},{"Geometry":280},{"Texture":16},{"Shader":69173},{"Material":24},{"GfxDevice":35248},{"Animation":304},{"Audio":3976},{"Physics":288},{"Serialization":216},{"Input":9176},{"JobScheduler":200},{"Mono":40},{"ScriptingNativeRuntime":216},{"BaseObject":1609212},{"Resource":592},{"Renderer":1936},{"Transform":48},{"File":800},{"WebCam":24},{"Culling":40},{"Terrain":953},{"Wind":24},{"String":3542},{"DynamicArray":30868},{"HashMap":7680},{"Utility":8704},{"PoolAlloc":1160},{"TypeTree":1792},{"ScriptManager":80},{"RuntimeInitializeOnLoadManager":72},{"SpriteAtlas":112},{"GI":3272},{"Unet":16},{"Director":7760},{"WebRequest":720},{"VR":45473},{"SceneManager":424},{"Video":32},{"LazyScriptCache":32},{"NativeArray":384},{"Camera":25},{"Secure":1},{"SerializationCache":624},{"APIUpdating":5872},{"Subsystems":384},{"VirtualTexturing":57552},{"AssetReference":32}]}
+##utp:{"type":"MemoryLeaks","version":2,"phase":"Immediate","time":1726855533975,"processId":56285,"allocatedMemory":2209098,"memoryLabels":[{"Default":9033},{"Permanent":1264},{"NewDelete":227599},{"Thread":34964},{"Manager":26827},{"VertexData":12},{"Geometry":280},{"Texture":16},{"Shader":69173},{"Material":24},{"GfxDevice":35248},{"Animation":304},{"Audio":3976},{"Physics":288},{"Serialization":216},{"Input":9176},{"JobScheduler":200},{"Mono":40},{"ScriptingNativeRuntime":216},{"BaseObject":1609212},{"Resource":592},{"Renderer":1936},{"Transform":48},{"File":800},{"WebCam":24},{"Culling":40},{"Terrain":953},{"Wind":24},{"String":3542},{"DynamicArray":30868},{"HashMap":7680},{"Utility":8704},{"PoolAlloc":1160},{"TypeTree":1792},{"ScriptManager":80},{"RuntimeInitializeOnLoadManager":72},{"SpriteAtlas":112},{"GI":3272},{"Unet":16},{"Director":7760},{"WebRequest":720},{"VR":45473},{"SceneManager":424},{"Video":32},{"LazyScriptCache":32},{"NativeArray":384},{"Camera":25},{"Secure":1},{"SerializationCache":624},{"APIUpdating":5872},{"Subsystems":384},{"VirtualTexturing":57552},{"AssetReference":32}]}

run_logs/timers.json

@@ -2,189 +2,189 @@
     "name": "root",
     "gauges": {
         "Pyramids.Policy.Entropy.mean": {
-            "value": 0.6287628412246704,
-            "min": 0.6287628412246704,
-            "max": 1.4757317304611206,
-            "count": 33
+            "value": 0.24078963696956635,
+            "min": 0.23496748507022858,
+            "max": 1.483778953552246,
+            "count": 100
         },
         "Pyramids.Policy.Entropy.sum": {
-            "value": 18832.705078125,
-            "min": 18832.705078125,
-            "max": 44767.796875,
-            "count": 33
+            "value": 7165.8994140625,
+            "min": 6983.625,
+            "max": 45011.91796875,
+            "count": 100
         },
         "Pyramids.Step.mean": {
-            "value": 989955.0,
-            "min": 29952.0,
-            "max": 989955.0,
-            "count": 33
+            "value": 2999991.0,
+            "min": 29921.0,
+            "max": 2999991.0,
+            "count": 100
         },
         "Pyramids.Step.sum": {
-            "value": 989955.0,
-            "min": 29952.0,
-            "max": 989955.0,
-            "count": 33
+            "value": 2999991.0,
+            "min": 29921.0,
+            "max": 2999991.0,
+            "count": 100
         },
         "Pyramids.Policy.ExtrinsicValueEstimate.mean": {
-            "value": 0.09492431581020355,
-            "min": -0.10491795092821121,
-            "max": 0.14662635326385498,
-            "count": 33
+            "value": 0.7674157619476318,
+            "min": -0.09803099185228348,
+            "max": 0.8211163282394409,
+            "count": 100
         },
         "Pyramids.Policy.ExtrinsicValueEstimate.sum": {
-            "value": 23.351381301879883,
-            "min": -25.180307388305664,
-            "max": 37.096466064453125,
-            "count": 33
+            "value": 227.9224853515625,
+            "min": -23.625469207763672,
+            "max": 243.05043029785156,
+            "count": 100
         },
         "Pyramids.Policy.RndValueEstimate.mean": {
-            "value": 0.009129628539085388,
-            "min": 0.0007445396622642875,
-            "max": 0.23194332420825958,
-            "count": 33
+            "value": 0.011679839342832565,
+            "min": -0.013532638549804688,
+            "max": 0.16018971800804138,
+            "count": 100
         },
         "Pyramids.Policy.RndValueEstimate.sum": {
-            "value": 2.2458887100219727,
-            "min": 0.1868794560432434,
-            "max": 55.66639709472656,
-            "count": 33
+            "value": 3.468912363052368,
+            "min": -3.545551300048828,
+            "max": 38.60572052001953,
+            "count": 100
         },
         "Pyramids.Losses.PolicyLoss.mean": {
-            "value": 0.06850323201212508,
-            "min": 0.06562200021371278,
-            "max": 0.07101916855680726,
-            "count": 33
+            "value": 0.06641950456608813,
+            "min": 0.06311546766406131,
+            "max": 0.07402042788129501,
+            "count": 100
         },
         "Pyramids.Losses.PolicyLoss.sum": {
-            "value": 0.959045248169751,
-            "min": 0.47074977868024515,
-            "max": 1.0468329841187725,
-            "count": 33
+            "value": 0.9298730639252338,
+            "min": 0.5753839198201949,
+            "max": 1.0777585393253841,
+            "count": 100
         },
         "Pyramids.Losses.ValueLoss.mean": {
-            "value": 0.0066629826944211105,
-            "min": 3.9189990934013224e-05,
-            "max": 0.007202525958467161,
-            "count": 33
+            "value": 0.015420854397968217,
+            "min": 5.832491022696869e-05,
+            "max": 0.016372939554414538,
+            "count": 100
         },
         "Pyramids.Losses.ValueLoss.sum": {
-            "value": 0.09328175772189555,
-            "min": 0.0005094698821421719,
-            "max": 0.10083536341854026,
-            "count": 33
+            "value": 0.21589196157155505,
+            "min": 0.0008165487431775616,
+            "max": 0.23471099033486098,
+            "count": 100
         },
         "Pyramids.Policy.LearningRate.mean": {
-            "value": 7.723683139757143e-06,
-            "min": 7.723683139757143e-06,
-            "max": 0.00029515063018788575,
-            "count": 33
+            "value": 1.4231852399238081e-06,
+            "min": 1.4231852399238081e-06,
+            "max": 0.00029828116307294584,
+            "count": 100
         },
         "Pyramids.Policy.LearningRate.sum": {
-            "value": 0.0001081315639566,
-            "min": 0.0001081315639566,
-            "max": 0.003139058053647399,
-            "count": 33
+            "value": 1.9924593358933315e-05,
+            "min": 1.9924593358933315e-05,
+            "max": 0.003969295676901467,
+            "count": 100
         },
         "Pyramids.Policy.Epsilon.mean": {
-            "value": 0.1025745285714286,
-            "min": 0.1025745285714286,
-            "max": 0.19838354285714285,
-            "count": 33
+            "value": 0.1004743619047619,
+            "min": 0.1004743619047619,
+            "max": 0.1994270541666667,
+            "count": 100
         },
         "Pyramids.Policy.Epsilon.sum": {
-            "value": 1.4360434000000004,
-            "min": 1.3886848,
-            "max": 2.3463526000000003,
-            "count": 33
+            "value": 1.4066410666666667,
+            "min": 1.4066410666666667,
+            "max": 2.7524638,
+            "count": 100
         },
         "Pyramids.Policy.Beta.mean": {
-            "value": 0.00026719540428571435,
-            "min": 0.00026719540428571435,
-            "max": 0.00983851593142857,
-            "count": 33
+            "value": 5.738875428571426e-05,
+            "min": 5.738875428571426e-05,
+            "max": 0.009942762711249999,
+            "count": 100
         },
         "Pyramids.Policy.Beta.sum": {
-            "value": 0.0037407356600000006,
-            "min": 0.0037407356600000006,
-            "max": 0.10466062474000003,
-            "count": 33
+            "value": 0.0008034425599999996,
+            "min": 0.0008034425599999996,
+            "max": 0.13231754348,
+            "count": 100
         },
         "Pyramids.Losses.RNDLoss.mean": {
-            "value": 0.011561497114598751,
-            "min": 0.011561497114598751,
-            "max": 0.3893832564353943,
-            "count": 33
+            "value": 0.004307860042899847,
+            "min": 0.004134449642151594,
+            "max": 0.27409908175468445,
+            "count": 100
         },
         "Pyramids.Losses.RNDLoss.sum": {
-            "value": 0.16186095774173737,
-            "min": 0.16186095774173737,
-            "max": 2.7256827354431152,
-            "count": 33
+            "value": 0.060310039669275284,
+            "min": 0.058274008333683014,
+            "max": 2.1927926540374756,
+            "count": 100
         },
         "Pyramids.Environment.EpisodeLength.mean": {
-            "value": 786.4,
-            "min": 696.3170731707318,
+            "value": 229.5151515151515,
+            "min": 227.515625,
             "max": 999.0,
-            "count": 33
+            "count": 100
         },
         "Pyramids.Environment.EpisodeLength.sum": {
-            "value": 27524.0,
-            "min": 15984.0,
-            "max": 32910.0,
-            "count": 33
+            "value": 30296.0,
+            "min": 16464.0,
+            "max": 32522.0,
+            "count": 100
         },
         "Pyramids.Environment.CumulativeReward.mean": {
-            "value": 0.35610281292881285,
-            "min": -1.0000000521540642,
-            "max": 0.5717999580793265,
-            "count": 33
+            "value": 1.7704848367156405,
+            "min": -0.9997097287447222,
+            "max": 1.7704848367156405,
+            "count": 100
         },
         "Pyramids.Environment.CumulativeReward.sum": {
-            "value": 12.46359845250845,
-            "min": -31.99920167028904,
-            "max": 23.443798281252384,
-            "count": 33
+            "value": 233.70399844646454,
+            "min": -30.991001591086388,
+            "max": 233.70399844646454,
+            "count": 100
         },
         "Pyramids.Policy.ExtrinsicReward.mean": {
-            "value": 0.35610281292881285,
-            "min": -1.0000000521540642,
-            "max": 0.5717999580793265,
-            "count": 33
+            "value": 1.7704848367156405,
+            "min": -0.9997097287447222,
+            "max": 1.7704848367156405,
+            "count": 100
         },
         "Pyramids.Policy.ExtrinsicReward.sum": {
-            "value": 12.46359845250845,
-            "min": -31.99920167028904,
-            "max": 23.443798281252384,
-            "count": 33
+            "value": 233.70399844646454,
+            "min": -30.991001591086388,
+            "max": 233.70399844646454,
+            "count": 100
         },
         "Pyramids.Policy.RndReward.mean": {
-            "value": 0.09512017368605094,
-            "min": 0.09183706338565874,
-            "max": 7.138329115696251,
-            "count": 33
+            "value": 0.010253159306750715,
+            "min": 0.01014670295417162,
+            "max": 5.14729900395169,
+            "count": 100
         },
         "Pyramids.Policy.RndReward.sum": {
-            "value": 3.329206079011783,
-            "min": 3.329206079011783,
-            "max": 114.21326585114002,
-            "count": 33
+            "value": 1.3534170284910942,
+            "min": 1.2856663103229948,
+            "max": 87.50408306717873,
+            "count": 100
         },
         "Pyramids.IsTraining.mean": {
             "value": 1.0,
             "min": 1.0,
             "max": 1.0,
-            "count": 33
+            "count": 100
         },
         "Pyramids.IsTraining.sum": {
             "value": 1.0,
             "min": 1.0,
             "max": 1.0,
-            "count": 33
+            "count": 100
         }
     },
     "metadata": {
         "timer_format_version": "0.1.0",
-        "start_time_seconds": "1726845918",
+        "start_time_seconds": "1726851844",
         "python_version": "3.10.12 (main, Sep 11 2024, 15:47:36) [GCC 11.4.0]",
         "command_line_arguments": "/home/smitsis/deep_rl/unit5/.venv/bin/mlagents-learn ./config/ppo/PyramidsRND.yaml --env=./training-envs-executables/linux/Pyramids/Pyramids --run-id=Pyramids Training --no-graphics --force",
         "mlagents_version": "1.1.0.dev0",
@@ -192,59 +192,59 @@
         "communication_protocol_version": "1.5.0",
         "pytorch_version": "2.4.1+cu121",
         "numpy_version": "1.23.5",
-        "end_time_seconds": "1726847151"
+        "end_time_seconds": "1726855534"
     },
-    "total": 1233.0665486219987,
+    "total": 3689.1127825900003,
     "count": 1,
-    "self": 0.32305689199984045,
+    "self": 0.4760124200020073,
     "children": {
         "run_training.setup": {
-            "total": 0.014216781999493833,
+            "total": 0.02995310000005702,
             "count": 1,
-            "self": 0.014216781999493833
+            "self": 0.02995310000005702
         },
         "TrainerController.start_learning": {
-            "total": 1232.7292749479993,
+            "total": 3688.606817069998,
             "count": 1,
-            "self": 1.1693049939394768,
+            "self": 3.351448298042669,
             "children": {
                 "TrainerController._reset_env": {
-                    "total": 3.1676304559996424,
+                    "total": 3.4157981459993607,
                     "count": 1,
-                    "self": 3.1676304559996424
+                    "self": 3.4157981459993607
                 },
                 "TrainerController.advance": {
-                    "total": 1228.3244182690587,
-                    "count": 63139,
-                    "self": 0.9860291470222364,
+                    "total": 3681.785385775953,
+                    "count": 193375,
+                    "self": 2.852028546583824,
                     "children": {
                         "env_step": {
-                            "total": 662.6237612770274,
-                            "count": 63139,
-                            "self": 573.4121740682613,
+                            "total": 2067.0188927489744,
+                            "count": 193375,
+                            "self": 1818.4640877506263,
                             "children": {
                                 "SubprocessEnvManager._take_step": {
-                                    "total": 88.53688027890894,
-                                    "count": 63139,
-                                    "self": 3.0738960809485434,
+                                    "total": 246.5984943047515,
+                                    "count": 193375,
+                                    "self": 8.955711231759778,
                                     "children": {
                                         "TorchPolicy.evaluate": {
-                                            "total": 85.4629841979604,
-                                            "count": 62561,
-                                            "self": 85.4629841979604
+                                            "total": 237.64278307299173,
+                                            "count": 187545,
+                                            "self": 237.64278307299173
                                         }
                                     }
                                 },
                                 "workers": {
-                                    "total": 0.674706929857166,
-                                    "count": 63139,
+                                    "total": 1.9563106935966061,
+                                    "count": 193375,
                                     "self": 0.0,
                                     "children": {
                                         "worker_root": {
-                                            "total": 1231.1805352508718,
-                                            "count": 63139,
+                                            "total": 3684.438881399403,
+                                            "count": 193375,
                                             "is_parallel": true,
-                                            "self": 735.2333133738912,
+                                            "self": 2091.789580488021,
                                             "children": {
                                                 "run_training.setup": {
                                                     "total": 0.0,
@@ -253,48 +253,48 @@
                                                     "self": 0.0,
                                                     "children": {
                                                         "steps_from_proto": {
-                                                            "total": 0.004558552999696985,
+                                                            "total": 0.0038262239995674463,
                                                             "count": 1,
                                                             "is_parallel": true,
-                                                            "self": 0.0018460510000295471,
+                                                            "self": 0.0012316629981796723,
                                                             "children": {
                                                                 "_process_rank_one_or_two_observation": {
-                                                                    "total": 0.002712501999667438,
+                                                                    "total": 0.002594561001387774,
                                                                     "count": 8,
                                                                     "is_parallel": true,
-                                                                    "self": 0.002712501999667438
+                                                                    "self": 0.002594561001387774
                                                                 }
                                                             }
                                                         },
                                                         "UnityEnvironment.step": {
-                                                            "total": 0.032818505999784975,
+                                                            "total": 0.024337621000086074,
                                                             "count": 1,
                                                             "is_parallel": true,
-                                                            "self": 0.00023017599960439838,
+                                                            "self": 0.0001588909999554744,
                                                             "children": {
                                                                 "UnityEnvironment._generate_step_input": {
-                                                                    "total": 0.0003087009999944712,
+                                                                    "total": 0.0002929189995484194,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.0003087009999944712
+                                                                    "self": 0.0002929189995484194
                                                                 },
                                                                 "communicator.exchange": {
-                                                                    "total": 0.03138544000012189,
+                                                                    "total": 0.023449497999536106,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.03138544000012189
+                                                                    "self": 0.023449497999536106
                                                                 },
                                                                 "steps_from_proto": {
-                                                                    "total": 0.0008941890000642161,
+                                                                    "total": 0.0004363130010460736,
                                                                     "count": 1,
                                                                     "is_parallel": true,
-                                                                    "self": 0.00021586000002571382,
+                                                                    "self": 0.00010668600043572951,
                                                                     "children": {
                                                                         "_process_rank_one_or_two_observation": {
-                                                                            "total": 0.0006783290000385023,
+                                                                            "total": 0.0003296270006103441,
                                                                             "count": 8,
                                                                             "is_parallel": true,
-                                                                            "self": 0.0006783290000385023
+                                                                            "self": 0.0003296270006103441
                                                                         }
                                                                     }
                                                                 }
@@ -303,34 +303,34 @@
                                                     }
                                                 },
                                                 "UnityEnvironment.step": {
-                                                    "total": 495.9472218769806,
-                                                    "count": 63138,
+                                                    "total": 1592.649300911382,
+                                                    "count": 193374,
                                                     "is_parallel": true,
-                                                    "self": 11.066111933129832,
+                                                    "self": 33.29139924104493,
                                                     "children": {
                                                         "UnityEnvironment._generate_step_input": {
-                                                            "total": 10.439097855049113,
-                                                            "count": 63138,
+                                                            "total": 31.90276924813952,
+                                                            "count": 193374,
                                                             "is_parallel": true,
-                                                            "self": 10.439097855049113
+                                                            "self": 31.90276924813952
                                                         },
                                                         "communicator.exchange": {
-                                                            "total": 440.2152833349619,
-                                                            "count": 63138,
+                                                            "total": 1423.4529963051518,
+                                                            "count": 193374,
                                                             "is_parallel": true,
-                                                            "self": 440.2152833349619
+                                                            "self": 1423.4529963051518
                                                         },
                                                         "steps_from_proto": {
-                                                            "total": 34.22672875383978,
-                                                            "count": 63138,
+                                                            "total": 104.00213611704567,
+                                                            "count": 193374,
                                                             "is_parallel": true,
-                                                            "self": 8.270993221273784,
+                                                            "self": 25.156676909145972,
                                                             "children": {
                                                                 "_process_rank_one_or_two_observation": {
-                                                                    "total": 25.955735532565996,
-                                                                    "count": 505104,
+                                                                    "total": 78.8454592078997,
+                                                                    "count": 1546992,
                                                                     "is_parallel": true,
-                                                                    "self": 25.955735532565996
+                                                                    "self": 78.8454592078997
                                                                 }
                                                             }
                                                         }
@@ -343,31 +343,31 @@
                             }
                         },
                         "trainer_advance": {
-                            "total": 564.7146278450091,
-                            "count": 63139,
-                            "self": 1.8404336940038775,
+                            "total": 1611.9144644803946,
+                            "count": 193375,
+                            "self": 5.991415485596008,
                             "children": {
                                 "process_trajectory": {
-                                    "total": 81.2656869600005,
-                                    "count": 63139,
-                                    "self": 81.11396476400114,
+                                    "total": 246.4258911207835,
+                                    "count": 193375,
+                                    "self": 246.08724975578662,
                                     "children": {
                                         "RLTrainer._checkpoint": {
-                                            "total": 0.15172219599935488,
-                                            "count": 2,
-                                            "self": 0.15172219599935488
+                                            "total": 0.33864136499687447,
+                                            "count": 6,
+                                            "self": 0.33864136499687447
                                         }
                                     }
                                 },
                                 "_update_policy": {
-                                    "total": 481.6085071910047,
-                                    "count": 433,
-                                    "self": 235.24576882302335,
+                                    "total": 1359.497157874015,
+                                    "count": 1398,
+                                    "self": 661.1802310612275,
                                     "children": {
                                         "TorchPPOOptimizer.update": {
-                                            "total": 246.36273836798136,
-                                            "count": 22854,
-                                            "self": 246.36273836798136
+                                            "total": 698.3169268127876,
+                                            "count": 68376,
+                                            "self": 698.3169268127876
                                         }
                                     }
                                 }
@@ -376,19 +376,19 @@
                     }
                 },
                 "trainer_threads": {
-                    "total": 8.330007403856143e-07,
+                    "total": 5.05002390127629e-07,
                     "count": 1,
-                    "self": 8.330007403856143e-07
+                    "self": 5.05002390127629e-07
                 },
                 "TrainerController._save_models": {
-                    "total": 0.0679203960007726,
+                    "total": 0.054184345000976464,
                     "count": 1,
-                    "self": 0.002301239999724203,
+                    "self": 0.0010695109995140228,
                     "children": {
                         "RLTrainer._checkpoint": {
-                            "total": 0.0656191560010484,
+                            "total": 0.05311483400146244,
                             "count": 1,
-                            "self": 0.0656191560010484
+                            "self": 0.05311483400146244
                         }
                     }
                 }

run_logs/training_status.json

@@ -2,40 +2,58 @@
     "Pyramids": {
         "checkpoints": [
             {
-                "steps": 499991,
-                "file_path": "results/Pyramids Training/Pyramids/Pyramids-499991.onnx",
+                "steps": 1499961,
+                "file_path": "results/Pyramids Training/Pyramids/Pyramids-1499961.onnx",
+                "reward": 1.5452499836683273,
+                "creation_time": 1726853652.3375618,
+                "auxillary_file_paths": [
+                    "results/Pyramids Training/Pyramids/Pyramids-1499961.pt"
+                ]
+            },
+            {
+                "steps": 1999982,
+                "file_path": "results/Pyramids Training/Pyramids/Pyramids-1999982.onnx",
+                "reward": 1.670500010251999,
+                "creation_time": 1726854273.8090847,
+                "auxillary_file_paths": [
+                    "results/Pyramids Training/Pyramids/Pyramids-1999982.pt"
+                ]
+            },
+            {
+                "steps": 2499899,
+                "file_path": "results/Pyramids Training/Pyramids/Pyramids-2499899.onnx",
                 "reward": null,
-                "creation_time": 1726846551.6174712,
+                "creation_time": 1726854895.155915,
                 "auxillary_file_paths": [
-                    "results/Pyramids Training/Pyramids/Pyramids-499991.pt"
+                    "results/Pyramids Training/Pyramids/Pyramids-2499899.pt"
                 ]
             },
             {
-                "steps": 999942,
-                "file_path": "results/Pyramids Training/Pyramids/Pyramids-999942.onnx",
-                "reward": 0.6706666275858879,
-                "creation_time": 1726847151.2899451,
+                "steps": 2999991,
+                "file_path": "results/Pyramids Training/Pyramids/Pyramids-2999991.onnx",
+                "reward": 1.8064999729394913,
+                "creation_time": 1726855533.5001256,
                 "auxillary_file_paths": [
-                    "results/Pyramids Training/Pyramids/Pyramids-999942.pt"
+                    "results/Pyramids Training/Pyramids/Pyramids-2999991.pt"
                 ]
             },
             {
-                "steps": 1000070,
-                "file_path": "results/Pyramids Training/Pyramids/Pyramids-1000070.onnx",
-                "reward": 0.6706666275858879,
-                "creation_time": 1726847151.3658912,
+                "steps": 3000119,
+                "file_path": "results/Pyramids Training/Pyramids/Pyramids-3000119.onnx",
+                "reward": 1.8064999729394913,
+                "creation_time": 1726855533.5592263,
                 "auxillary_file_paths": [
-                    "results/Pyramids Training/Pyramids/Pyramids-1000070.pt"
+                    "results/Pyramids Training/Pyramids/Pyramids-3000119.pt"
                 ]
             }
         ],
         "final_checkpoint": {
-            "steps": 1000070,
+            "steps": 3000119,
             "file_path": "results/Pyramids Training/Pyramids.onnx",
-            "reward": 0.6706666275858879,
-            "creation_time": 1726847151.3658912,
+            "reward": 1.8064999729394913,
+            "creation_time": 1726855533.5592263,
             "auxillary_file_paths": [
-                "results/Pyramids Training/Pyramids/Pyramids-1000070.pt"
+                "results/Pyramids Training/Pyramids/Pyramids-3000119.pt"
             ]
         }
     },