Improve

SnowballTarget.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5fe42b3139d3f684385b7a96101ebff90abaed5f093080d8b4572dce9f81a5fc
+oid sha256:712e3f2259d2108fceba279af513f556698961b2989bb20e7131cf8ba4927f46
 size 650646

SnowballTarget/SnowballTarget-149960.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05c79706806b79c28b3db1429b0d50684d711b3400e2114206c26236cdc0f7fa
+size 650646

SnowballTarget/SnowballTarget-149960.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:421ed1d3639fef4fb4e13731c9d7badca96eab1b447026cc87e6d6a74d1a0a49
+size 3850075

SnowballTarget/SnowballTarget-199984.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:712e3f2259d2108fceba279af513f556698961b2989bb20e7131cf8ba4927f46
+size 650646

SnowballTarget/SnowballTarget-199984.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8876ec977fdd842f38dd4b56786879f56674af2f5aa80ee7fefc05434c6b3282
+size 3850075

SnowballTarget/SnowballTarget-200112.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:712e3f2259d2108fceba279af513f556698961b2989bb20e7131cf8ba4927f46
+size 650646

SnowballTarget/SnowballTarget-200112.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98bde74f2e068b5d5cdb03ab8906bad97f0486feefeced150dfc944b72504b89
+size 3850075

SnowballTarget/SnowballTarget-49936.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a12c251f22c7cdb79d76deede62a3ef0a840b0b96b669a29ecc990a4ac15aef
+size 650646

SnowballTarget/SnowballTarget-49936.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:133f4451543be169686027d315b3ae291e094fd8466f6f138f8db980d3e007be
+size 3850000

SnowballTarget/SnowballTarget-99936.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e1923f803721251597f92af3e83c849bc10429a0a73923a2051cb4dc166e818
+size 650646

SnowballTarget/SnowballTarget-99936.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7b68a2b4e10698b361cd7d461171c8bb69f5c180cdd7de2f68072e3d1177bef
+size 3850000

SnowballTarget/checkpoint.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f20a59a5487789ab1751d354692dbb32f3f8474a17e1e0a1a0db8b9bb3786900
+oid sha256:4b06fe5508f7ec2913f3043b3c6c3cca5aabc2a7786ec9a5142d1fa146f9023a
 size 3849250

SnowballTarget/events.out.tfevents.1722593986.a8df1cf79192.5029.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:612266705990ace7a4bcee1f86655867193dced871973e4b7a81ec2314443929
+size 16616

config.json

@@ -1 +1 @@
-{"default_settings": null, "behaviors": {"SnowballTarget": {"trainer_type": "ppo", "hyperparameters": {"batch_size": 128, "buffer_size": 2048, "learning_rate": 0.0003, "beta": 0.005, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 3, "shared_critic": false, "learning_rate_schedule": "linear", "beta_schedule": "linear", "epsilon_schedule": "linear"}, "checkpoint_interval": 50000, "network_settings": {"normalize": false, "hidden_units": 256, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.99, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}}, "init_path": null, "keep_checkpoints": 10, "even_checkpoints": false, "max_steps": 2000, "time_horizon": 64, "summary_freq": 10000, "threaded": true, "self_play": null, "behavioral_cloning": null}}, "env_settings": {"env_path": "./training-envs-executables/linux/SnowballTarget/SnowballTarget", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "timeout_wait": 60, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true, "no_graphics_monitor": false}, "environment_parameters": null, "checkpoint_settings": {"run_id": "SnowballTarget1", "initialize_from": null, "load_model": false, "resume": true, "force": false, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}
+{"default_settings": null, "behaviors": {"SnowballTarget": {"trainer_type": "ppo", "hyperparameters": {"batch_size": 128, "buffer_size": 2048, "learning_rate": 0.0003, "beta": 0.005, "epsilon": 0.2, "lambd": 0.95, "num_epoch": 3, "shared_critic": false, "learning_rate_schedule": "linear", "beta_schedule": "linear", "epsilon_schedule": "linear"}, "checkpoint_interval": 50000, "network_settings": {"normalize": false, "hidden_units": 256, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}, "reward_signals": {"extrinsic": {"gamma": 0.99, "strength": 1.0, "network_settings": {"normalize": false, "hidden_units": 128, "num_layers": 2, "vis_encode_type": "simple", "memory": null, "goal_conditioning_type": "hyper", "deterministic": false}}}, "init_path": null, "keep_checkpoints": 10, "even_checkpoints": false, "max_steps": 200000, "time_horizon": 64, "summary_freq": 10000, "threaded": true, "self_play": null, "behavioral_cloning": null}}, "env_settings": {"env_path": "./training-envs-executables/linux/SnowballTarget/SnowballTarget", "env_args": null, "base_port": 5005, "num_envs": 1, "num_areas": 1, "timeout_wait": 60, "seed": -1, "max_lifetime_restarts": 10, "restarts_rate_limit_n": 1, "restarts_rate_limit_period_s": 60}, "engine_settings": {"width": 84, "height": 84, "quality_level": 5, "time_scale": 20, "target_frame_rate": -1, "capture_frame_rate": 60, "no_graphics": true, "no_graphics_monitor": false}, "environment_parameters": null, "checkpoint_settings": {"run_id": "SnowballTarget1", "initialize_from": null, "load_model": false, "resume": true, "force": false, "train_model": false, "inference": false, "results_dir": "results"}, "torch_settings": {"device": null}, "debug": false}

configuration.yaml

@@ -38,7 +38,7 @@ behaviors:
     init_path: null
     keep_checkpoints: 10
     even_checkpoints: false
-    max_steps: 2000
+    max_steps: 200000
     time_horizon: 64
     summary_freq: 10000
     threaded: true

run_logs/Player-0.log

@@ -31,7 +31,7 @@ ALSA lib pcm.c:2664:(snd_pcm_open_noupdate) Unknown PCM default
 FMOD failed to initialize the output device.: "Error initializing output device. " (60)
 FMOD initialized on nosound output
 Begin MonoManager ReloadAssembly
-- Completed reload, in  0.082 seconds
+- Completed reload, in  0.084 seconds
 ERROR: Shader Sprites/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Sprites/Mask shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 ERROR: Shader Legacy Shaders/VertexLit shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
@@ -42,7 +42,7 @@ ERROR: Shader Standard shader is not supported on this GPU (none of subshaders/f
 WARNING: Shader Unsupported: 'Standard' - All subshaders removed
 WARNING: Shader Did you use #pragma only_renderers and omit this platform?
 WARNING: Shader If subshaders removal was intentional, you may have forgotten turning Fallback off?
-UnloadTime: 0.692270 ms
+UnloadTime: 0.651153 ms
 ERROR: Shader UI/Default shader is not supported on this GPU (none of subshaders/fallbacks are suitable)
 requesting resize 84 x 84
 Setting up 1 worker threads for Enlighten.
@@ -50,7 +50,7 @@ Memory Statistics:
 [ALLOC_TEMP_TLS] TLS Allocator
   StackAllocators : 
     [ALLOC_TEMP_MAIN]
-      Peak usage frame count: [2.0 MB-4.0 MB]: 1 frames
+      Peak usage frame count: [8.0 KB-16.0 KB]: 4808 frames, [16.0 KB-32.0 KB]: 81 frames, [2.0 MB-4.0 MB]: 1 frames
       Initial Block Size 4.0 MB
       Current Block Size 4.0 MB
       Peak Allocated Bytes 2.0 MB
@@ -78,7 +78,7 @@ Memory Statistics:
     [ALLOC_TEMP_Job.Worker 0]
       Initial Block Size 256.0 KB
       Current Block Size 256.0 KB
-      Peak Allocated Bytes 2.8 KB
+      Peak Allocated Bytes 3.6 KB
       Overflow Count 0
     [ALLOC_TEMP_Background Job.Worker 10]
       Initial Block Size 32.0 KB
@@ -166,22 +166,22 @@ Memory Statistics:
       Peak Allocated Bytes 0 B
       Overflow Count 0
 [ALLOC_DEFAULT] Dual Thread Allocator
-  Peak main deferred allocation count 3
+  Peak main deferred allocation count 44
     [ALLOC_BUCKET]
       Large Block size 4.0 MB
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_DEFAULT_MAIN]
-      Peak usage frame count: [4.0 MB-8.0 MB]: 1 frames
+      Peak usage frame count: [4.0 MB-8.0 MB]: 4890 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
-      Peak Allocated memory 4.6 MB
+      Peak Allocated memory 6.6 MB
       Peak Large allocation bytes 0 B
     [ALLOC_DEFAULT_THREAD]
-      Peak usage frame count: [16.0 MB-32.0 MB]: 1 frames
+      Peak usage frame count: [16.0 MB-32.0 MB]: 4890 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
-      Peak Allocated memory 17.3 MB
+      Peak Allocated memory 17.7 MB
       Peak Large allocation bytes 16.0 MB
 [ALLOC_TEMP_JOB_1_FRAME]
   Initial Block Size 2.0 MB
@@ -195,7 +195,7 @@ Memory Statistics:
   Overflow Count (full) 0
 [ALLOC_TEMP_JOB_4_FRAMES (JobTemp)]
   Initial Block Size 2.0 MB
-  Used Block Count 1
+  Used Block Count 2
   Overflow Count (too large) 0
   Overflow Count (full) 0
 [ALLOC_TEMP_JOB_ASYNC (Background)]
@@ -210,13 +210,13 @@ Memory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_GFX_MAIN]
-      Peak usage frame count: [64.0 KB-128.0 KB]: 1 frames
+      Peak usage frame count: [32.0 KB-64.0 KB]: 4231 frames, [64.0 KB-128.0 KB]: 659 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
-      Peak Allocated memory 65.6 KB
+      Peak Allocated memory 67.3 KB
       Peak Large allocation bytes 0 B
     [ALLOC_GFX_THREAD]
-      Peak usage frame count: [32.0 KB-64.0 KB]: 1 frames
+      Peak usage frame count: [32.0 KB-64.0 KB]: 4890 frames
       Requested Block Size 16.0 MB
       Peak Block count 1
       Peak Allocated memory 39.6 KB
@@ -228,16 +228,16 @@ Memory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_CACHEOBJECTS_MAIN]
-      Peak usage frame count: [0.5 MB-1.0 MB]: 1 frames
+      Peak usage frame count: [0.5 MB-1.0 MB]: 4890 frames
       Requested Block Size 4.0 MB
       Peak Block count 1
-      Peak Allocated memory 0.5 MB
+      Peak Allocated memory 0.6 MB
       Peak Large allocation bytes 0 B
     [ALLOC_CACHEOBJECTS_THREAD]
-      Peak usage frame count: [2.0 MB-4.0 MB]: 1 frames
+      Peak usage frame count: [0.5 MB-1.0 MB]: 4889 frames, [2.0 MB-4.0 MB]: 1 frames
       Requested Block Size 4.0 MB
       Peak Block count 1
-      Peak Allocated memory 2.3 MB
+      Peak Allocated memory 2.2 MB
       Peak Large allocation bytes 0 B
 [ALLOC_TYPETREE] Dual Thread Allocator
   Peak main deferred allocation count 0
@@ -246,13 +246,13 @@ Memory Statistics:
       Used Block count 1
       Peak Allocated bytes 1.0 MB
     [ALLOC_TYPETREE_MAIN]
-      Peak usage frame count: [0-1.0 KB]: 1 frames
+      Peak usage frame count: [0-1.0 KB]: 4890 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 1.0 KB
       Peak Large allocation bytes 0 B
     [ALLOC_TYPETREE_THREAD]
-      Peak usage frame count: [1.0 KB-2.0 KB]: 1 frames
+      Peak usage frame count: [1.0 KB-2.0 KB]: 4890 frames
       Requested Block Size 2.0 MB
       Peak Block count 1
       Peak Allocated memory 1.7 KB

run_logs/timers.json

@@ -1,8 +1,154 @@
 {
     "name": "root",
+    "gauges": {
+        "SnowballTarget.Policy.Entropy.mean": {
+            "value": 1.0248702764511108,
+            "min": 1.0233978033065796,
+            "max": 2.6493778228759766,
+            "count": 18
+        },
+        "SnowballTarget.Policy.Entropy.sum": {
+            "value": 9977.1123046875,
+            "min": 9977.1123046875,
+            "max": 25267.1171875,
+            "count": 18
+        },
+        "SnowballTarget.Step.mean": {
+            "value": 199984.0,
+            "min": 29992.0,
+            "max": 199984.0,
+            "count": 18
+        },
+        "SnowballTarget.Step.sum": {
+            "value": 199984.0,
+            "min": 29992.0,
+            "max": 199984.0,
+            "count": 18
+        },
+        "SnowballTarget.Policy.ExtrinsicValueEstimate.mean": {
+            "value": 12.989912986755371,
+            "min": 2.4948792457580566,
+            "max": 13.001709938049316,
+            "count": 18
+        },
+        "SnowballTarget.Policy.ExtrinsicValueEstimate.sum": {
+            "value": 2662.93212890625,
+            "min": 434.1090087890625,
+            "max": 2662.93212890625,
+            "count": 18
+        },
+        "SnowballTarget.Environment.EpisodeLength.mean": {
+            "value": 199.0,
+            "min": 199.0,
+            "max": 199.0,
+            "count": 18
+        },
+        "SnowballTarget.Environment.EpisodeLength.sum": {
+            "value": 10945.0,
+            "min": 8756.0,
+            "max": 10945.0,
+            "count": 18
+        },
+        "SnowballTarget.Losses.PolicyLoss.mean": {
+            "value": 0.06182343586103752,
+            "min": 0.06052512708782017,
+            "max": 0.07680597960072691,
+            "count": 18
+        },
+        "SnowballTarget.Losses.PolicyLoss.sum": {
+            "value": 0.3091171793051876,
+            "min": 0.26256851684051,
+            "max": 0.36497568320962326,
+            "count": 18
+        },
+        "SnowballTarget.Losses.ValueLoss.mean": {
+            "value": 0.2130128609783509,
+            "min": 0.18682205450593256,
+            "max": 0.2755992733234284,
+            "count": 18
+        },
+        "SnowballTarget.Losses.ValueLoss.sum": {
+            "value": 1.0650643048917545,
+            "min": 0.7472882180237302,
+            "max": 1.3488157207474991,
+            "count": 18
+        },
+        "SnowballTarget.Policy.LearningRate.mean": {
+            "value": 7.620097459999994e-06,
+            "min": 7.620097459999994e-06,
+            "max": 0.00026007001331,
+            "count": 18
+        },
+        "SnowballTarget.Policy.LearningRate.sum": {
+            "value": 3.810048729999997e-05,
+            "min": 3.810048729999997e-05,
+            "max": 0.0011601001132999999,
+            "count": 18
+        },
+        "SnowballTarget.Policy.Epsilon.mean": {
+            "value": 0.10254,
+            "min": 0.10254,
+            "max": 0.18669000000000002,
+            "count": 18
+        },
+        "SnowballTarget.Policy.Epsilon.sum": {
+            "value": 0.5127,
+            "min": 0.42996,
+            "max": 0.8866999999999999,
+            "count": 18
+        },
+        "SnowballTarget.Policy.Beta.mean": {
+            "value": 0.0001367459999999999,
+            "min": 0.0001367459999999999,
+            "max": 0.004335831,
+            "count": 18
+        },
+        "SnowballTarget.Policy.Beta.sum": {
+            "value": 0.0006837299999999995,
+            "min": 0.0006837299999999995,
+            "max": 0.019346330000000002,
+            "count": 18
+        },
+        "SnowballTarget.Environment.CumulativeReward.mean": {
+            "value": 25.236363636363638,
+            "min": 8.023809523809524,
+            "max": 25.836363636363636,
+            "count": 18
+        },
+        "SnowballTarget.Environment.CumulativeReward.sum": {
+            "value": 1388.0,
+            "min": 337.0,
+            "max": 1421.0,
+            "count": 18
+        },
+        "SnowballTarget.Policy.ExtrinsicReward.mean": {
+            "value": 25.236363636363638,
+            "min": 8.023809523809524,
+            "max": 25.836363636363636,
+            "count": 18
+        },
+        "SnowballTarget.Policy.ExtrinsicReward.sum": {
+            "value": 1388.0,
+            "min": 337.0,
+            "max": 1421.0,
+            "count": 18
+        },
+        "SnowballTarget.IsTraining.mean": {
+            "value": 1.0,
+            "min": 1.0,
+            "max": 1.0,
+            "count": 18
+        },
+        "SnowballTarget.IsTraining.sum": {
+            "value": 1.0,
+            "min": 1.0,
+            "max": 1.0,
+            "count": 18
+        }
+    },
     "metadata": {
         "timer_format_version": "0.1.0",
-        "start_time_seconds": "1722593821",
+        "start_time_seconds": "1722593986",
         "python_version": "3.10.12 (main, Jul 29 2024, 16:56:48) [GCC 11.4.0]",
         "command_line_arguments": "/usr/local/bin/mlagents-learn ./config/ppo/SnowballTarget.yaml --env=./training-envs-executables/linux/SnowballTarget/SnowballTarget --run-id=SnowballTarget1 --no-graphics --resume",
         "mlagents_version": "1.1.0.dev0",
@@ -10,31 +156,162 @@
         "communication_protocol_version": "1.5.0",
         "pytorch_version": "2.3.1+cu121",
         "numpy_version": "1.23.5",
-        "end_time_seconds": "1722593824"
+        "end_time_seconds": "1722594388"
     },
-    "total": 2.8245287739999867,
+    "total": 402.14017314800003,
     "count": 1,
-    "self": 0.5370103089999247,
+    "self": 0.47645351199980723,
     "children": {
         "run_training.setup": {
-            "total": 0.05382296699997369,
+            "total": 0.05271483400019861,
             "count": 1,
-            "self": 0.05382296699997369
+            "self": 0.05271483400019861
         },
         "TrainerController.start_learning": {
-            "total": 2.2336954980000883,
+            "total": 401.611004802,
             "count": 1,
-            "self": 0.0002561400001468428,
+            "self": 0.5080185079914372,
             "children": {
                 "TrainerController._reset_env": {
-                    "total": 1.9355054259999633,
+                    "total": 1.9157861460000731,
                     "count": 1,
-                    "self": 1.9355054259999633
+                    "self": 1.9157861460000731
+                },
+                "TrainerController.advance": {
+                    "total": 399.0280675060085,
+                    "count": 16298,
+                    "self": 0.231761505992381,
+                    "children": {
+                        "env_step": {
+                            "total": 398.7963060000161,
+                            "count": 16298,
+                            "self": 257.3976601529878,
+                            "children": {
+                                "SubprocessEnvManager._take_step": {
+                                    "total": 141.15487900101402,
+                                    "count": 16298,
+                                    "self": 1.3216700969931026,
+                                    "children": {
+                                        "TorchPolicy.evaluate": {
+                                            "total": 139.83320890402092,
+                                            "count": 16298,
+                                            "self": 139.83320890402092
+                                        }
+                                    }
+                                },
+                                "workers": {
+                                    "total": 0.24376684601429588,
+                                    "count": 16298,
+                                    "self": 0.0,
+                                    "children": {
+                                        "worker_root": {
+                                            "total": 400.4621210340165,
+                                            "count": 16298,
+                                            "is_parallel": true,
+                                            "self": 202.58089878202077,
+                                            "children": {
+                                                "run_training.setup": {
+                                                    "total": 0.0,
+                                                    "count": 0,
+                                                    "is_parallel": true,
+                                                    "self": 0.0,
+                                                    "children": {
+                                                        "steps_from_proto": {
+                                                            "total": 0.0021687190001102863,
+                                                            "count": 1,
+                                                            "is_parallel": true,
+                                                            "self": 0.0006749759998001537,
+                                                            "children": {
+                                                                "_process_rank_one_or_two_observation": {
+                                                                    "total": 0.0014937430003101326,
+                                                                    "count": 10,
+                                                                    "is_parallel": true,
+                                                                    "self": 0.0014937430003101326
+                                                                }
+                                                            }
+                                                        },
+                                                        "UnityEnvironment.step": {
+                                                            "total": 0.07031695199998467,
+                                                            "count": 1,
+                                                            "is_parallel": true,
+                                                            "self": 0.0006866130001981219,
+                                                            "children": {
+                                                                "UnityEnvironment._generate_step_input": {
+                                                                    "total": 0.0003499219999412162,
+                                                                    "count": 1,
+                                                                    "is_parallel": true,
+                                                                    "self": 0.0003499219999412162
+                                                                },
+                                                                "communicator.exchange": {
+                                                                    "total": 0.06749808599988683,
+                                                                    "count": 1,
+                                                                    "is_parallel": true,
+                                                                    "self": 0.06749808599988683
+                                                                },
+                                                                "steps_from_proto": {
+                                                                    "total": 0.0017823309999585035,
+                                                                    "count": 1,
+                                                                    "is_parallel": true,
+                                                                    "self": 0.00034167799958595424,
+                                                                    "children": {
+                                                                        "_process_rank_one_or_two_observation": {
+                                                                            "total": 0.0014406530003725493,
+                                                                            "count": 10,
+                                                                            "is_parallel": true,
+                                                                            "self": 0.0014406530003725493
+                                                                        }
+                                                                    }
+                                                                }
+                                                            }
+                                                        }
+                                                    }
+                                                },
+                                                "UnityEnvironment.step": {
+                                                    "total": 197.88122225199572,
+                                                    "count": 16297,
+                                                    "is_parallel": true,
+                                                    "self": 8.868798117969845,
+                                                    "children": {
+                                                        "UnityEnvironment._generate_step_input": {
+                                                            "total": 4.791149939013167,
+                                                            "count": 16297,
+                                                            "is_parallel": true,
+                                                            "self": 4.791149939013167
+                                                        },
+                                                        "communicator.exchange": {
+                                                            "total": 153.98308474799887,
+                                                            "count": 16297,
+                                                            "is_parallel": true,
+                                                            "self": 153.98308474799887
+                                                        },
+                                                        "steps_from_proto": {
+                                                            "total": 30.238189447013838,
+                                                            "count": 16297,
+                                                            "is_parallel": true,
+                                                            "self": 5.63628823898739,
+                                                            "children": {
+                                                                "_process_rank_one_or_two_observation": {
+                                                                    "total": 24.601901208026447,
+                                                                    "count": 162970,
+                                                                    "is_parallel": true,
+                                                                    "self": 24.601901208026447
+                                                                }
+                                                            }
+                                                        }
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
                 },
                 "trainer_threads": {
-                    "total": 1.6825999978209438e-05,
+                    "total": 0.00015016800011835585,
                     "count": 1,
-                    "self": 1.6825999978209438e-05,
+                    "self": 0.00015016800011835585,
                     "children": {
                         "thread_root": {
                             "total": 0.0,
@@ -43,16 +320,38 @@
                             "self": 0.0,
                             "children": {
                                 "trainer_advance": {
-                                    "total": 0.0002180029999863109,
-                                    "count": 1,
+                                    "total": 394.0747794568442,
+                                    "count": 604609,
                                     "is_parallel": true,
-                                    "self": 2.5026000002981164e-05,
+                                    "self": 12.610627955976042,
                                     "children": {
                                         "process_trajectory": {
-                                            "total": 0.00019297699998332973,
-                                            "count": 1,
+                                            "total": 218.03464418686826,
+                                            "count": 604609,
+                                            "is_parallel": true,
+                                            "self": 217.08632586886802,
+                                            "children": {
+                                                "RLTrainer._checkpoint": {
+                                                    "total": 0.948318318000247,
+                                                    "count": 4,
+                                                    "is_parallel": true,
+                                                    "self": 0.948318318000247
+                                                }
+                                            }
+                                        },
+                                        "_update_policy": {
+                                            "total": 163.42950731399992,
+                                            "count": 81,
                                             "is_parallel": true,
-                                            "self": 0.00019297699998332973
+                                            "self": 50.42302629799565,
+                                            "children": {
+                                                "TorchPPOOptimizer.update": {
+                                                    "total": 113.00648101600427,
+                                                    "count": 4128,
+                                                    "is_parallel": true,
+                                                    "self": 113.00648101600427
+                                                }
+                                            }
                                         }
                                     }
                                 }
@@ -61,14 +360,14 @@
                     }
                 },
                 "TrainerController._save_models": {
-                    "total": 0.2979171059999999,
+                    "total": 0.15898247399991305,
                     "count": 1,
-                    "self": 0.001102231999993819,
+                    "self": 0.0015238260000387527,
                     "children": {
                         "RLTrainer._checkpoint": {
-                            "total": 0.29681487400000606,
+                            "total": 0.1574586479998743,
                             "count": 1,
-                            "self": 0.29681487400000606
+                            "self": 0.1574586479998743
                         }
                     }
                 }

run_logs/training_status.json

@@ -18,15 +18,60 @@
                 "auxillary_file_paths": [
                     "results/SnowballTarget1/SnowballTarget/SnowballTarget-21208.pt"
                 ]
+            },
+            {
+                "steps": 49936,
+                "file_path": "results/SnowballTarget1/SnowballTarget/SnowballTarget-49936.onnx",
+                "reward": 12.454545454545455,
+                "creation_time": 1722594054.0695908,
+                "auxillary_file_paths": [
+                    "results/SnowballTarget1/SnowballTarget/SnowballTarget-49936.pt"
+                ]
+            },
+            {
+                "steps": 99936,
+                "file_path": "results/SnowballTarget1/SnowballTarget/SnowballTarget-99936.onnx",
+                "reward": 19.636363636363637,
+                "creation_time": 1722594164.6592822,
+                "auxillary_file_paths": [
+                    "results/SnowballTarget1/SnowballTarget/SnowballTarget-99936.pt"
+                ]
+            },
+            {
+                "steps": 149960,
+                "file_path": "results/SnowballTarget1/SnowballTarget/SnowballTarget-149960.onnx",
+                "reward": 23.818181818181817,
+                "creation_time": 1722594274.679816,
+                "auxillary_file_paths": [
+                    "results/SnowballTarget1/SnowballTarget/SnowballTarget-149960.pt"
+                ]
+            },
+            {
+                "steps": 199984,
+                "file_path": "results/SnowballTarget1/SnowballTarget/SnowballTarget-199984.onnx",
+                "reward": 25.818181818181817,
+                "creation_time": 1722594387.5177155,
+                "auxillary_file_paths": [
+                    "results/SnowballTarget1/SnowballTarget/SnowballTarget-199984.pt"
+                ]
+            },
+            {
+                "steps": 200112,
+                "file_path": "results/SnowballTarget1/SnowballTarget/SnowballTarget-200112.onnx",
+                "reward": 25.818181818181817,
+                "creation_time": 1722594387.7154431,
+                "auxillary_file_paths": [
+                    "results/SnowballTarget1/SnowballTarget/SnowballTarget-200112.pt"
+                ]
             }
         ],
         "final_checkpoint": {
-            "steps": 21208,
+            "steps": 200112,
             "file_path": "results/SnowballTarget1/SnowballTarget.onnx",
-            "reward": null,
-            "creation_time": 1722593823.6294467,
+            "reward": 25.818181818181817,
+            "creation_time": 1722594387.7154431,
             "auxillary_file_paths": [
-                "results/SnowballTarget1/SnowballTarget/SnowballTarget-21208.pt"
+                "results/SnowballTarget1/SnowballTarget/SnowballTarget-200112.pt"
             ]
         }
     },