Upload training_log.json with huggingface_hub

training_log.json

@@ -1,1442 +1,2522 @@
 [
   {
     "update": 5,
-    "global_step": 10240,
-    "num_episodes": 5,
-    "mean_reward": -2.121724510192871,
-    "mean_length": 337.8,
-    "loss": 2.8168609142303467,
-    "sps": 228.7546335682378
+    "global_step": 20480,
+    "num_episodes": 12,
+    "mean_reward": 2.9954163233439126,
+    "mean_length": 301.75,
+    "loss": 0.4707077145576477,
+    "sps": 2897.2289066048047
   },
   {
     "update": 10,
-    "global_step": 20480,
-    "num_episodes": 18,
-    "mean_reward": -1.9787551561991374,
-    "mean_length": 282.94444444444446,
-    "loss": 0.21835778653621674,
-    "sps": 1443.696434210723
+    "global_step": 40960,
+    "num_episodes": 12,
+    "mean_reward": 2.9954163233439126,
+    "mean_length": 301.75,
+    "loss": 1.0232033729553223,
+    "sps": 2951.2458688255706
   },
   {
     "update": 15,
-    "global_step": 30720,
-    "num_episodes": 18,
-    "mean_reward": -1.9787551561991374,
-    "mean_length": 282.94444444444446,
-    "loss": 0.2217431664466858,
-    "sps": 2444.104440024094
+    "global_step": 61440,
+    "num_episodes": 12,
+    "mean_reward": 2.9954163233439126,
+    "mean_length": 301.75,
+    "loss": 1.3049204349517822,
+    "sps": 2822.044769995029
   },
   {
     "update": 20,
-    "global_step": 40960,
-    "num_episodes": 22,
-    "mean_reward": 0.6390173651955344,
-    "mean_length": 1597.090909090909,
-    "loss": 0.5035408735275269,
-    "sps": 769.1328942825502
+    "global_step": 81920,
+    "num_episodes": 16,
+    "mean_reward": 6.446762174367905,
+    "mean_length": 2105.625,
+    "loss": 3.682100772857666,
+    "sps": 2013.3169607307623
   },
   {
     "update": 25,
-    "global_step": 51200,
-    "num_episodes": 27,
-    "mean_reward": 0.33698498761212387,
-    "mean_length": 1341.111111111111,
-    "loss": 0.16991910338401794,
-    "sps": 2449.665853907546
+    "global_step": 102400,
+    "num_episodes": 24,
+    "mean_reward": 14.11154419183731,
+    "mean_length": 3507.7083333333335,
+    "loss": 0.5709385275840759,
+    "sps": 2704.0544416242456
   },
   {
     "update": 30,
-    "global_step": 61440,
-    "num_episodes": 29,
-    "mean_reward": 0.5858152652608937,
-    "mean_length": 1594.9310344827586,
-    "loss": 0.6312248110771179,
-    "sps": 2363.5499282399865
+    "global_step": 122880,
+    "num_episodes": 24,
+    "mean_reward": 14.11154419183731,
+    "mean_length": 3507.7083333333335,
+    "loss": 0.9411047697067261,
+    "sps": 2703.677829940517
   },
   {
     "update": 35,
-    "global_step": 71680,
-    "num_episodes": 29,
-    "mean_reward": 0.5858152652608937,
-    "mean_length": 1594.9310344827586,
-    "loss": 0.4890201985836029,
-    "sps": 2440.672480568945
+    "global_step": 143360,
+    "num_episodes": 24,
+    "mean_reward": 14.11154419183731,
+    "mean_length": 3507.7083333333335,
+    "loss": 0.18034808337688446,
+    "sps": 2660.7833455377177
   },
   {
     "update": 40,
-    "global_step": 81920,
-    "num_episodes": 31,
-    "mean_reward": 1.2613418640628937,
-    "mean_length": 1822.0,
-    "loss": 0.42574575543403625,
-    "sps": 940.0717318496648
+    "global_step": 163840,
+    "num_episodes": 29,
+    "mean_reward": 33.75182229074939,
+    "mean_length": 4627.068965517241,
+    "loss": 0.449077308177948,
+    "sps": 781.6860032595372
   },
   {
     "update": 45,
-    "global_step": 92160,
-    "num_episodes": 55,
-    "mean_reward": 2.8606436122547496,
-    "mean_length": 1469.2727272727273,
-    "loss": 0.5428293943405151,
-    "sps": 1479.6348998060785
+    "global_step": 184320,
+    "num_episodes": 43,
+    "mean_reward": 31.471195509267407,
+    "mean_length": 3927.2093023255816,
+    "loss": 2.4075186252593994,
+    "sps": 1640.7038600574003
   },
   {
     "update": 50,
-    "global_step": 102400,
-    "num_episodes": 58,
-    "mean_reward": 3.2600761939739358,
-    "mean_length": 1568.051724137931,
-    "loss": 0.6221064329147339,
-    "sps": 2412.4523346060355
+    "global_step": 204800,
+    "num_episodes": 43,
+    "mean_reward": 31.471195509267407,
+    "mean_length": 3927.2093023255816,
+    "loss": 0.7558565735816956,
+    "sps": 2296.3021079759415
   },
   {
     "update": 55,
-    "global_step": 112640,
-    "num_episodes": 58,
-    "mean_reward": 3.2600761939739358,
-    "mean_length": 1568.051724137931,
-    "loss": 0.794475257396698,
-    "sps": 2405.0877911326343
+    "global_step": 225280,
+    "num_episodes": 43,
+    "mean_reward": 31.471195509267407,
+    "mean_length": 3927.2093023255816,
+    "loss": 0.004491906613111496,
+    "sps": 1918.4604744440246
   },
   {
     "update": 60,
-    "global_step": 122880,
-    "num_episodes": 58,
-    "mean_reward": 3.2600761939739358,
-    "mean_length": 1568.051724137931,
-    "loss": 1.0936992168426514,
-    "sps": 2358.874803208204
+    "global_step": 245760,
+    "num_episodes": 45,
+    "mean_reward": 33.261741065979,
+    "mean_length": 4197.111111111111,
+    "loss": 0.35625988245010376,
+    "sps": 2061.6452443134217
   },
   {
     "update": 65,
-    "global_step": 133120,
-    "num_episodes": 62,
-    "mean_reward": 7.5118274227265385,
-    "mean_length": 1954.4516129032259,
-    "loss": 1.877166986465454,
-    "sps": 2402.6606220225244
+    "global_step": 266240,
+    "num_episodes": 56,
+    "mean_reward": 39.252455030168804,
+    "mean_length": 4465.803571428572,
+    "loss": 0.26919102668762207,
+    "sps": 2690.170793323002
   },
   {
     "update": 70,
-    "global_step": 143360,
-    "num_episodes": 63,
-    "mean_reward": 9.12682440924266,
-    "mean_length": 2082.15873015873,
-    "loss": 0.55415278673172,
-    "sps": 2417.4656020731263
+    "global_step": 286720,
+    "num_episodes": 56,
+    "mean_reward": 39.252455030168804,
+    "mean_length": 4465.803571428572,
+    "loss": 0.1174827367067337,
+    "sps": 2143.112234819028
   },
   {
     "update": 75,
-    "global_step": 153600,
-    "num_episodes": 63,
-    "mean_reward": 9.12682440924266,
-    "mean_length": 2082.15873015873,
-    "loss": 0.6123632788658142,
-    "sps": 2415.520694055524
+    "global_step": 307200,
+    "num_episodes": 56,
+    "mean_reward": 39.252455030168804,
+    "mean_length": 4465.803571428572,
+    "loss": -0.09781338274478912,
+    "sps": 2208.779203581181
   },
   {
     "update": 80,
-    "global_step": 163840,
-    "num_episodes": 63,
-    "mean_reward": 9.12682440924266,
-    "mean_length": 2082.15873015873,
-    "loss": 0.3689587116241455,
-    "sps": 2464.6240808517623
+    "global_step": 327680,
+    "num_episodes": 62,
+    "mean_reward": 38.99283899030378,
+    "mean_length": 4527.532258064516,
+    "loss": 1.440595030784607,
+    "sps": 655.150017301736
   },
   {
     "update": 85,
-    "global_step": 174080,
-    "num_episodes": 70,
-    "mean_reward": 9.247220189230783,
-    "mean_length": 2310.557142857143,
-    "loss": 0.3228328824043274,
-    "sps": 2445.2962822104055
+    "global_step": 348160,
+    "num_episodes": 84,
+    "mean_reward": 36.50451924119677,
+    "mean_length": 4002.7380952380954,
+    "loss": 1.0692782402038574,
+    "sps": 1057.9744994181403
   },
   {
     "update": 90,
-    "global_step": 184320,
-    "num_episodes": 76,
-    "mean_reward": 9.205821903128372,
-    "mean_length": 2268.3026315789475,
-    "loss": 0.6627960205078125,
-    "sps": 453.4183326926133
+    "global_step": 368640,
+    "num_episodes": 86,
+    "mean_reward": 35.66498793003171,
+    "mean_length": 3935.0232558139537,
+    "loss": -0.0837952047586441,
+    "sps": 2123.204801099428
   },
   {
     "update": 95,
-    "global_step": 194560,
-    "num_episodes": 80,
-    "mean_reward": 9.297839653491973,
-    "mean_length": 2165.9375,
-    "loss": 1.1097347736358643,
-    "sps": 2424.5065546850346
+    "global_step": 389120,
+    "num_episodes": 86,
+    "mean_reward": 35.66498793003171,
+    "mean_length": 3935.0232558139537,
+    "loss": 0.02618391066789627,
+    "sps": 1904.8344116389594
   },
   {
     "update": 100,
-    "global_step": 204800,
-    "num_episodes": 80,
-    "mean_reward": 9.297839653491973,
-    "mean_length": 2165.9375,
-    "loss": 0.38644012808799744,
-    "sps": 2386.6710469869367
+    "global_step": 409600,
+    "num_episodes": 88,
+    "mean_reward": 35.91715085506439,
+    "mean_length": 4072.8636363636365,
+    "loss": -0.004935186356306076,
+    "sps": 1928.296801932442
   },
   {
     "update": 105,
-    "global_step": 215040,
-    "num_episodes": 87,
-    "mean_reward": 11.394574307847297,
-    "mean_length": 2339.057471264368,
-    "loss": 0.7209265828132629,
-    "sps": 2442.8262241524835
+    "global_step": 430080,
+    "num_episodes": 100,
+    "mean_reward": 33.11026698112488,
+    "mean_length": 4118.01,
+    "loss": 1.716380000114441,
+    "sps": 755.5801175010796
   },
   {
     "update": 110,
-    "global_step": 225280,
-    "num_episodes": 87,
-    "mean_reward": 11.394574307847297,
-    "mean_length": 2339.057471264368,
-    "loss": 2.1218621730804443,
-    "sps": 2449.9984290228053
+    "global_step": 450560,
+    "num_episodes": 102,
+    "mean_reward": 33.19641512870788,
+    "mean_length": 4218.5,
+    "loss": 0.0785534530878067,
+    "sps": 2175.9855223665277
   },
   {
     "update": 115,
-    "global_step": 235520,
-    "num_episodes": 90,
-    "mean_reward": 12.198311275906033,
-    "mean_length": 2375.788888888889,
-    "loss": 0.5268573760986328,
-    "sps": 2438.632604283566
+    "global_step": 471040,
+    "num_episodes": 102,
+    "mean_reward": 33.19641512870788,
+    "mean_length": 4218.5,
+    "loss": -0.07273395359516144,
+    "sps": 1787.2507488200727
   },
   {
     "update": 120,
-    "global_step": 245760,
-    "num_episodes": 90,
-    "mean_reward": 12.198311275906033,
-    "mean_length": 2375.788888888889,
-    "loss": 2.0001754760742188,
-    "sps": 2413.1198849334774
+    "global_step": 491520,
+    "num_episodes": 103,
+    "mean_reward": 33.58080266952515,
+    "mean_length": 4314.65,
+    "loss": -0.026257965713739395,
+    "sps": 1805.9278386526794
   },
   {
     "update": 125,
-    "global_step": 256000,
-    "num_episodes": 105,
-    "mean_reward": 12.691367435455323,
-    "mean_length": 2446.14,
-    "loss": 0.6336287260055542,
-    "sps": 619.7035033065293
+    "global_step": 512000,
+    "num_episodes": 110,
+    "mean_reward": 37.310110912323,
+    "mean_length": 4794.35,
+    "loss": 1.8378688097000122,
+    "sps": 1715.9048394677898
   },
   {
     "update": 130,
-    "global_step": 266240,
-    "num_episodes": 105,
-    "mean_reward": 12.691367435455323,
-    "mean_length": 2446.14,
-    "loss": 0.40229156613349915,
-    "sps": 2411.1751921586524
+    "global_step": 532480,
+    "num_episodes": 115,
+    "mean_reward": 38.11976575374603,
+    "mean_length": 4792.66,
+    "loss": 0.0826515257358551,
+    "sps": 2480.658396587235
   },
   {
     "update": 135,
-    "global_step": 276480,
-    "num_episodes": 109,
-    "mean_reward": 13.886511907577514,
-    "mean_length": 2526.42,
-    "loss": 0.45471617579460144,
-    "sps": 2450.150074118317
+    "global_step": 552960,
+    "num_episodes": 115,
+    "mean_reward": 38.11976575374603,
+    "mean_length": 4792.66,
+    "loss": -0.058959588408470154,
+    "sps": 2558.7887965900827
   },
   {
     "update": 140,
-    "global_step": 286720,
-    "num_episodes": 109,
-    "mean_reward": 13.886511907577514,
-    "mean_length": 2526.42,
-    "loss": 0.4323250949382782,
-    "sps": 2378.2401254524298
+    "global_step": 573440,
+    "num_episodes": 117,
+    "mean_reward": 40.18981074810028,
+    "mean_length": 4792.66,
+    "loss": 0.01938764378428459,
+    "sps": 2511.6128893797145
   },
   {
     "update": 145,
-    "global_step": 296960,
-    "num_episodes": 121,
-    "mean_reward": 14.005602750778198,
-    "mean_length": 2623.1,
-    "loss": 0.5164788365364075,
-    "sps": 1792.6912302713768
+    "global_step": 593920,
+    "num_episodes": 124,
+    "mean_reward": 51.76217389583588,
+    "mean_length": 4803.44,
+    "loss": 20.378520965576172,
+    "sps": 909.3232748780807
   },
   {
     "update": 150,
-    "global_step": 307200,
-    "num_episodes": 121,
-    "mean_reward": 14.005602750778198,
-    "mean_length": 2623.1,
-    "loss": 0.8702018857002258,
-    "sps": 2425.950637318183
+    "global_step": 614400,
+    "num_episodes": 126,
+    "mean_reward": 51.032718253135684,
+    "mean_length": 4803.44,
+    "loss": 0.11053133755922318,
+    "sps": 2492.3194028833077
   },
   {
     "update": 155,
-    "global_step": 317440,
-    "num_episodes": 129,
-    "mean_reward": 14.87307053565979,
-    "mean_length": 2527.32,
-    "loss": 0.1953483670949936,
-    "sps": 2394.950934358607
+    "global_step": 634880,
+    "num_episodes": 126,
+    "mean_reward": 51.032718253135684,
+    "mean_length": 4803.44,
+    "loss": 0.31901270151138306,
+    "sps": 2322.6918424224104
   },
   {
     "update": 160,
-    "global_step": 327680,
+    "global_step": 655360,
     "num_episodes": 129,
-    "mean_reward": 14.87307053565979,
-    "mean_length": 2527.32,
-    "loss": 0.19992592930793762,
-    "sps": 2424.040624859784
+    "mean_reward": 49.292411608695986,
+    "mean_length": 4705.26,
+    "loss": 0.6963619589805603,
+    "sps": 1560.7635575105323
   },
   {
     "update": 165,
-    "global_step": 337920,
-    "num_episodes": 139,
-    "mean_reward": 16.099841012954712,
-    "mean_length": 2525.93,
-    "loss": 0.4464109241962433,
-    "sps": 1319.6508401584945
+    "global_step": 675840,
+    "num_episodes": 138,
+    "mean_reward": 51.79567018985748,
+    "mean_length": 5005.94,
+    "loss": 3.423638343811035,
+    "sps": 1566.1339446711113
   },
   {
     "update": 170,
-    "global_step": 348160,
+    "global_step": 696320,
     "num_episodes": 139,
-    "mean_reward": 16.099841012954712,
-    "mean_length": 2525.93,
-    "loss": 0.10373280942440033,
-    "sps": 2348.0679386786046
+    "mean_reward": 52.29565020084381,
+    "mean_length": 5005.94,
+    "loss": 5.795687675476074,
+    "sps": 1548.0157210393158
   },
   {
     "update": 175,
-    "global_step": 358400,
-    "num_episodes": 142,
-    "mean_reward": 17.19686735153198,
-    "mean_length": 2624.69,
-    "loss": 0.8095718026161194,
-    "sps": 2386.2480483453646
+    "global_step": 716800,
+    "num_episodes": 139,
+    "mean_reward": 52.29565020084381,
+    "mean_length": 5005.94,
+    "loss": 11.365696907043457,
+    "sps": 1678.7844476146372
   },
   {
     "update": 180,
-    "global_step": 368640,
-    "num_episodes": 142,
-    "mean_reward": 17.19686735153198,
-    "mean_length": 2624.69,
-    "loss": 0.009909076616168022,
-    "sps": 2301.590679746089
+    "global_step": 737280,
+    "num_episodes": 140,
+    "mean_reward": 53.179930477142335,
+    "mean_length": 5098.77,
+    "loss": -0.05775236710906029,
+    "sps": 1668.2296340431747
   },
   {
     "update": 185,
-    "global_step": 378880,
-    "num_episodes": 148,
-    "mean_reward": 19.868873529434204,
-    "mean_length": 2907.66,
-    "loss": 1.7244997024536133,
-    "sps": 2280.4958488818847
+    "global_step": 757760,
+    "num_episodes": 149,
+    "mean_reward": 70.75448143482208,
+    "mean_length": 5195.77,
+    "loss": 73.1705322265625,
+    "sps": 858.1752320250257
   },
   {
     "update": 190,
-    "global_step": 389120,
-    "num_episodes": 148,
-    "mean_reward": 19.868873529434204,
-    "mean_length": 2907.66,
-    "loss": 4.516170978546143,
-    "sps": 2419.553348740851
+    "global_step": 778240,
+    "num_episodes": 150,
+    "mean_reward": 71.51151569843292,
+    "mean_length": 5295.07,
+    "loss": 0.24444621801376343,
+    "sps": 1363.6491070768768
   },
   {
     "update": 195,
-    "global_step": 399360,
-    "num_episodes": 152,
-    "mean_reward": 20.717632093429565,
-    "mean_length": 3008.33,
-    "loss": 3.401921272277832,
-    "sps": 575.3298806579921
+    "global_step": 798720,
+    "num_episodes": 150,
+    "mean_reward": 71.51151569843292,
+    "mean_length": 5295.07,
+    "loss": 0.8110665082931519,
+    "sps": 1453.3363954580439
   },
   {
     "update": 200,
-    "global_step": 409600,
-    "num_episodes": 153,
-    "mean_reward": 20.717632093429565,
-    "mean_length": 3008.33,
-    "loss": 0.8132290244102478,
-    "sps": 2450.768029758656
+    "global_step": 819200,
+    "num_episodes": 151,
+    "mean_reward": 72.04212862491607,
+    "mean_length": 5392.97,
+    "loss": 0.1485036462545395,
+    "sps": 1399.4723333291518
   },
   {
     "update": 205,
-    "global_step": 419840,
-    "num_episodes": 162,
-    "mean_reward": 19.608867626190186,
-    "mean_length": 2909.79,
-    "loss": 1.867810845375061,
-    "sps": 621.2303877555788
+    "global_step": 839680,
+    "num_episodes": 157,
+    "mean_reward": 98.28625455379486,
+    "mean_length": 5585.11,
+    "loss": 23.002685546875,
+    "sps": 240.5853655120997
   },
   {
     "update": 210,
-    "global_step": 430080,
-    "num_episodes": 167,
-    "mean_reward": 19.028235759735107,
-    "mean_length": 2619.36,
-    "loss": 0.8309984803199768,
-    "sps": 2358.6319149765754
+    "global_step": 860160,
+    "num_episodes": 160,
+    "mean_reward": 99.03613055229187,
+    "mean_length": 5599.07,
+    "loss": 56.14109802246094,
+    "sps": 963.6583162438803
   },
   {
     "update": 215,
-    "global_step": 440320,
-    "num_episodes": 169,
-    "mean_reward": 19.63686834335327,
-    "mean_length": 2715.53,
-    "loss": 0.17013275623321533,
-    "sps": 1381.2913503327266
+    "global_step": 880640,
+    "num_episodes": 160,
+    "mean_reward": 99.03613055229187,
+    "mean_length": 5599.07,
+    "loss": 0.288861483335495,
+    "sps": 1135.1194345027134
   },
   {
     "update": 220,
-    "global_step": 450560,
-    "num_episodes": 170,
-    "mean_reward": 19.778069705963134,
-    "mean_length": 2619.36,
-    "loss": -0.013407116755843163,
-    "sps": 2225.527904161147
+    "global_step": 901120,
+    "num_episodes": 162,
+    "mean_reward": 100.13229321479797,
+    "mean_length": 5596.55,
+    "loss": 0.12560202181339264,
+    "sps": 1065.000301525082
   },
   {
     "update": 225,
-    "global_step": 460800,
-    "num_episodes": 172,
-    "mean_reward": 21.13547016143799,
-    "mean_length": 2718.42,
-    "loss": 0.14364227652549744,
-    "sps": 2345.342614298781
+    "global_step": 921600,
+    "num_episodes": 173,
+    "mean_reward": 126.59835072040558,
+    "mean_length": 5771.42,
+    "loss": 18.769105911254883,
+    "sps": 484.9667540902585
   },
   {
     "update": 230,
-    "global_step": 471040,
-    "num_episodes": 174,
-    "mean_reward": 22.236838788986205,
-    "mean_length": 2819.34,
-    "loss": 0.017919447273015976,
-    "sps": 2408.348085488877
+    "global_step": 942080,
+    "num_episodes": 175,
+    "mean_reward": 133.06522512435913,
+    "mean_length": 5865.35,
+    "loss": 2.0742053985595703,
+    "sps": 485.5770281538178
   },
   {
     "update": 235,
-    "global_step": 481280,
-    "num_episodes": 174,
-    "mean_reward": 22.236838788986205,
-    "mean_length": 2819.34,
-    "loss": 2.6431455612182617,
-    "sps": 2403.528538830992
+    "global_step": 962560,
+    "num_episodes": 175,
+    "mean_reward": 133.06522512435913,
+    "mean_length": 5865.35,
+    "loss": 1.1434893608093262,
+    "sps": 476.861101571966
   },
   {
     "update": 240,
-    "global_step": 491520,
-    "num_episodes": 178,
-    "mean_reward": 21.810234026908873,
-    "mean_length": 2928.26,
-    "loss": 0.2556005120277405,
-    "sps": 2373.833411448431
+    "global_step": 983040,
+    "num_episodes": 177,
+    "mean_reward": 136.24841757774354,
+    "mean_length": 6062.18,
+    "loss": 2.5528671741485596,
+    "sps": 519.697272037427
   },
   {
     "update": 245,
-    "global_step": 501760,
+    "global_step": 1003520,
     "num_episodes": 181,
-    "mean_reward": 23.416525321006773,
-    "mean_length": 3026.48,
-    "loss": 3.2893059253692627,
-    "sps": 2362.3097327902087
+    "mean_reward": 162.3702562904358,
+    "mean_length": 6350.36,
+    "loss": 2.4525704383850098,
+    "sps": 2612.0970180967397
   },
   {
     "update": 250,
-    "global_step": 512000,
-    "num_episodes": 182,
-    "mean_reward": 23.666512427330016,
-    "mean_length": 3026.48,
-    "loss": 4.634244441986084,
-    "sps": 2410.981639631563
+    "global_step": 1024000,
+    "num_episodes": 183,
+    "mean_reward": 163.440665807724,
+    "mean_length": 6546.01,
+    "loss": 1.370378017425537,
+    "sps": 1843.3577332235466
   },
   {
     "update": 255,
-    "global_step": 522240,
-    "num_episodes": 182,
-    "mean_reward": 23.666512427330016,
-    "mean_length": 3026.48,
-    "loss": 1.1017451286315918,
-    "sps": 2373.168403411863
+    "global_step": 1044480,
+    "num_episodes": 183,
+    "mean_reward": 163.440665807724,
+    "mean_length": 6546.01,
+    "loss": 3.996934652328491,
+    "sps": 1664.629871283792
   },
   {
     "update": 260,
-    "global_step": 532480,
+    "global_step": 1064960,
     "num_episodes": 188,
-    "mean_reward": 24.153521933555602,
-    "mean_length": 3032.56,
-    "loss": 0.058166228234767914,
-    "sps": 549.3090191152681
+    "mean_reward": 164.5874024486542,
+    "mean_length": 6629.5,
+    "loss": 38.7066535949707,
+    "sps": 275.2025358552949
   },
   {
     "update": 265,
-    "global_step": 542720,
-    "num_episodes": 200,
-    "mean_reward": 24.202102789878847,
-    "mean_length": 2827.14,
-    "loss": 3.6524577140808105,
-    "sps": 2408.936350313052
+    "global_step": 1085440,
+    "num_episodes": 195,
+    "mean_reward": 178.587217502594,
+    "mean_length": 6716.14,
+    "loss": 48.47567367553711,
+    "sps": 202.49850145253117
   },
   {
     "update": 270,
-    "global_step": 552960,
-    "num_episodes": 203,
-    "mean_reward": 25.780276198387146,
-    "mean_length": 2924.21,
-    "loss": -0.03986305743455887,
-    "sps": 2384.8038232666454
+    "global_step": 1105920,
+    "num_episodes": 195,
+    "mean_reward": 178.587217502594,
+    "mean_length": 6716.14,
+    "loss": 65.4755859375,
+    "sps": 336.3470793114214
   },
   {
     "update": 275,
-    "global_step": 563200,
-    "num_episodes": 203,
-    "mean_reward": 25.780276198387146,
-    "mean_length": 2924.21,
-    "loss": 0.009281929582357407,
-    "sps": 2245.2843029907467
+    "global_step": 1126400,
+    "num_episodes": 195,
+    "mean_reward": 178.587217502594,
+    "mean_length": 6716.14,
+    "loss": 122.97542572021484,
+    "sps": 364.01088862747804
   },
   {
     "update": 280,
-    "global_step": 573440,
-    "num_episodes": 204,
-    "mean_reward": 26.63669138431549,
-    "mean_length": 3023.78,
-    "loss": 0.1620613932609558,
-    "sps": 2155.8537501850938
+    "global_step": 1146880,
+    "num_episodes": 200,
+    "mean_reward": 217.83971637248993,
+    "mean_length": 6897.39,
+    "loss": 16.919631958007812,
+    "sps": 523.4729171393346
   },
   {
     "update": 285,
-    "global_step": 583680,
-    "num_episodes": 208,
-    "mean_reward": 27.24311915874481,
-    "mean_length": 3124.7,
-    "loss": 0.7804455161094666,
-    "sps": 2336.540118617715
+    "global_step": 1167360,
+    "num_episodes": 209,
+    "mean_reward": 269.6603216218948,
+    "mean_length": 6691.86,
+    "loss": 37.87599182128906,
+    "sps": 815.1007717041923
   },
   {
     "update": 290,
-    "global_step": 593920,
-    "num_episodes": 210,
-    "mean_reward": 27.990238661766053,
-    "mean_length": 3122.84,
-    "loss": -0.02732567861676216,
-    "sps": 2446.515068539969
+    "global_step": 1187840,
+    "num_episodes": 209,
+    "mean_reward": 269.6603216218948,
+    "mean_length": 6691.86,
+    "loss": 93.80265808105469,
+    "sps": 677.6266786573581
   },
   {
     "update": 295,
-    "global_step": 604160,
-    "num_episodes": 210,
-    "mean_reward": 27.990238661766053,
-    "mean_length": 3122.84,
-    "loss": 0.026210829615592957,
-    "sps": 2401.0219632291446
+    "global_step": 1208320,
+    "num_episodes": 209,
+    "mean_reward": 269.6603216218948,
+    "mean_length": 6691.86,
+    "loss": 2.2752561569213867,
+    "sps": 796.2728392649855
   },
   {
     "update": 300,
-    "global_step": 614400,
+    "global_step": 1228800,
     "num_episodes": 212,
-    "mean_reward": 28.8441028547287,
-    "mean_length": 3221.9,
-    "loss": 0.018815483897924423,
-    "sps": 1787.985104867478
+    "mean_reward": 282.0548305273056,
+    "mean_length": 6888.72,
+    "loss": 44.09903335571289,
+    "sps": 435.2776190122807
   },
   {
     "update": 305,
-    "global_step": 624640,
-    "num_episodes": 219,
-    "mean_reward": 29.601799569129945,
-    "mean_length": 3224.78,
-    "loss": -0.01376257836818695,
-    "sps": 2307.873344880848
+    "global_step": 1249280,
+    "num_episodes": 218,
+    "mean_reward": 326.81962947368623,
+    "mean_length": 6987.94,
+    "loss": 7.751248836517334,
+    "sps": 605.8631845420061
   },
   {
     "update": 310,
-    "global_step": 634880,
-    "num_episodes": 223,
-    "mean_reward": 30.112299256324768,
-    "mean_length": 3222.81,
-    "loss": -0.13672460615634918,
-    "sps": 2376.26116166583
+    "global_step": 1269760,
+    "num_episodes": 218,
+    "mean_reward": 326.81962947368623,
+    "mean_length": 6987.94,
+    "loss": 1.2491610050201416,
+    "sps": 609.0143093574485
   },
   {
     "update": 315,
-    "global_step": 645120,
-    "num_episodes": 223,
-    "mean_reward": 30.112299256324768,
-    "mean_length": 3222.81,
-    "loss": 0.24129757285118103,
-    "sps": 2301.537029038706
+    "global_step": 1290240,
+    "num_episodes": 218,
+    "mean_reward": 326.81962947368623,
+    "mean_length": 6987.94,
+    "loss": 0.18734177947044373,
+    "sps": 969.6236925007502
   },
   {
     "update": 320,
-    "global_step": 655360,
-    "num_episodes": 224,
-    "mean_reward": 30.965999827384948,
-    "mean_length": 3320.52,
-    "loss": 0.3078324496746063,
-    "sps": 2359.266120378419
+    "global_step": 1310720,
+    "num_episodes": 222,
+    "mean_reward": 354.5139421653748,
+    "mean_length": 6981.29,
+    "loss": 5.763525009155273,
+    "sps": 471.35135085304955
   },
   {
     "update": 325,
-    "global_step": 665600,
-    "num_episodes": 230,
-    "mean_reward": 31.090740485191347,
-    "mean_length": 3419.33,
-    "loss": 0.002869449555873871,
-    "sps": 2345.5360183144344
+    "global_step": 1331200,
+    "num_episodes": 235,
+    "mean_reward": 373.75524856090544,
+    "mean_length": 6657.2,
+    "loss": 8.977479934692383,
+    "sps": 286.61651802589716
   },
   {
     "update": 330,
-    "global_step": 675840,
-    "num_episodes": 233,
-    "mean_reward": 31.204454226493837,
-    "mean_length": 3514.15,
-    "loss": 0.8130831718444824,
-    "sps": 2348.1224969342966
+    "global_step": 1351680,
+    "num_episodes": 235,
+    "mean_reward": 373.75524856090544,
+    "mean_length": 6657.2,
+    "loss": 4.4195051193237305,
+    "sps": 540.9233917238028
   },
   {
     "update": 335,
-    "global_step": 686080,
-    "num_episodes": 233,
-    "mean_reward": 31.204454226493837,
-    "mean_length": 3514.15,
-    "loss": -0.10025477409362793,
-    "sps": 2203.7073854518803
+    "global_step": 1372160,
+    "num_episodes": 235,
+    "mean_reward": 373.75524856090544,
+    "mean_length": 6657.2,
+    "loss": 1.350094199180603,
+    "sps": 577.9686556262599
   },
   {
     "update": 340,
-    "global_step": 696320,
-    "num_episodes": 236,
-    "mean_reward": 30.600520968437195,
-    "mean_length": 3419.33,
-    "loss": 0.11422204971313477,
-    "sps": 1223.5079096329264
+    "global_step": 1392640,
+    "num_episodes": 240,
+    "mean_reward": 407.44437908649445,
+    "mean_length": 6657.92,
+    "loss": 32.73158264160156,
+    "sps": 852.1673008615437
   },
   {
     "update": 345,
-    "global_step": 706560,
-    "num_episodes": 244,
-    "mean_reward": 29.116036429405213,
-    "mean_length": 3431.39,
-    "loss": 0.3314732015132904,
-    "sps": 604.6873121204968
+    "global_step": 1413120,
+    "num_episodes": 250,
+    "mean_reward": 424.2497792673111,
+    "mean_length": 6448.66,
+    "loss": 40.15397262573242,
+    "sps": 149.27274368511317
   },
   {
     "update": 350,
-    "global_step": 716800,
-    "num_episodes": 248,
-    "mean_reward": 27.764892373085022,
-    "mean_length": 3338.43,
-    "loss": -0.08348219841718674,
-    "sps": 2334.652125215495
+    "global_step": 1433600,
+    "num_episodes": 250,
+    "mean_reward": 424.2497792673111,
+    "mean_length": 6448.66,
+    "loss": 1.2196906805038452,
+    "sps": 385.0138381705491
   },
   {
     "update": 355,
-    "global_step": 727040,
-    "num_episodes": 248,
-    "mean_reward": 27.764892373085022,
-    "mean_length": 3338.43,
-    "loss": -0.011928500607609749,
-    "sps": 2266.898774941684
+    "global_step": 1454080,
+    "num_episodes": 250,
+    "mean_reward": 424.2497792673111,
+    "mean_length": 6448.66,
+    "loss": 0.6893861889839172,
+    "sps": 364.4585616900348
   },
   {
     "update": 360,
-    "global_step": 737280,
-    "num_episodes": 251,
-    "mean_reward": 27.271975588798522,
-    "mean_length": 3335.03,
-    "loss": 0.16195061802864075,
-    "sps": 511.5670473274587
+    "global_step": 1474560,
+    "num_episodes": 255,
+    "mean_reward": 406.7418563890457,
+    "mean_length": 6349.6,
+    "loss": 21.265850067138672,
+    "sps": 309.03058801935407
   },
   {
     "update": 365,
-    "global_step": 747520,
+    "global_step": 1495040,
     "num_episodes": 261,
-    "mean_reward": 25.41567024707794,
-    "mean_length": 3238.35,
-    "loss": -0.17278775572776794,
-    "sps": 810.6621085529184
+    "mean_reward": 472.512622590065,
+    "mean_length": 6333.64,
+    "loss": 1.368093490600586,
+    "sps": 1030.3650029243877
   },
   {
     "update": 370,
-    "global_step": 757760,
-    "num_episodes": 274,
-    "mean_reward": 21.383243069648742,
-    "mean_length": 2943.09,
-    "loss": -0.017205242067575455,
-    "sps": 619.3760616179322
+    "global_step": 1515520,
+    "num_episodes": 261,
+    "mean_reward": 472.512622590065,
+    "mean_length": 6333.64,
+    "loss": 0.6811725497245789,
+    "sps": 988.2291227242658
   },
   {
     "update": 375,
-    "global_step": 768000,
-    "num_episodes": 274,
-    "mean_reward": 21.383243069648742,
-    "mean_length": 2943.09,
-    "loss": 0.08961383998394012,
-    "sps": 2236.482803815745
+    "global_step": 1536000,
+    "num_episodes": 261,
+    "mean_reward": 472.512622590065,
+    "mean_length": 6333.64,
+    "loss": 0.5682379603385925,
+    "sps": 972.3493096974811
   },
   {
     "update": 380,
-    "global_step": 778240,
-    "num_episodes": 274,
-    "mean_reward": 21.383243069648742,
-    "mean_length": 2943.09,
-    "loss": 4.401951313018799,
-    "sps": 2147.399362575019
+    "global_step": 1556480,
+    "num_episodes": 272,
+    "mean_reward": 484.873091211319,
+    "mean_length": 6237.44,
+    "loss": 3.223947048187256,
+    "sps": 2004.102630433895
   },
   {
     "update": 385,
-    "global_step": 788480,
-    "num_episodes": 278,
-    "mean_reward": 22.28250171661377,
-    "mean_length": 3028.1,
-    "loss": 1.4729293584823608,
-    "sps": 2300.4546825723296
+    "global_step": 1576960,
+    "num_episodes": 277,
+    "mean_reward": 478.27076225280763,
+    "mean_length": 6236.56,
+    "loss": 1.7760697603225708,
+    "sps": 2832.6289658137994
   },
   {
     "update": 390,
-    "global_step": 798720,
-    "num_episodes": 280,
-    "mean_reward": 21.386626567840576,
-    "mean_length": 3127.16,
-    "loss": 0.06943056732416153,
-    "sps": 2341.8789187579932
+    "global_step": 1597440,
+    "num_episodes": 277,
+    "mean_reward": 478.27076225280763,
+    "mean_length": 6236.56,
+    "loss": -0.11427821964025497,
+    "sps": 2603.8164819807103
   },
   {
     "update": 395,
-    "global_step": 808960,
-    "num_episodes": 280,
-    "mean_reward": 21.386626567840576,
-    "mean_length": 3127.16,
-    "loss": 0.929630696773529,
-    "sps": 2232.925422842351
+    "global_step": 1617920,
+    "num_episodes": 279,
+    "mean_reward": 462.21016893386843,
+    "mean_length": 6137.5,
+    "loss": 0.8288934230804443,
+    "sps": 2392.5538344727297
   },
   {
     "update": 400,
-    "global_step": 819200,
-    "num_episodes": 280,
-    "mean_reward": 21.386626567840576,
-    "mean_length": 3127.16,
-    "loss": -0.035214584320783615,
-    "sps": 2187.4650402736006
+    "global_step": 1638400,
+    "num_episodes": 284,
+    "mean_reward": 454.1959769916534,
+    "mean_length": 6037.93,
+    "loss": 202.71636962890625,
+    "sps": 355.5497429467541
   },
   {
     "update": 405,
-    "global_step": 829440,
-    "num_episodes": 284,
-    "mean_reward": 21.765649213790894,
-    "mean_length": 3031.31,
-    "loss": 3.6039881706237793,
-    "sps": 2344.35431842087
+    "global_step": 1658880,
+    "num_episodes": 289,
+    "mean_reward": 454.0530729818344,
+    "mean_length": 6035.23,
+    "loss": 0.2099057286977768,
+    "sps": 869.6302546973046
   },
   {
     "update": 410,
-    "global_step": 839680,
-    "num_episodes": 288,
-    "mean_reward": 23.637630491256715,
-    "mean_length": 3126.66,
-    "loss": 0.8354597091674805,
-    "sps": 1416.6940593723025
+    "global_step": 1679360,
+    "num_episodes": 289,
+    "mean_reward": 454.0530729818344,
+    "mean_length": 6035.23,
+    "loss": 0.018864750862121582,
+    "sps": 837.2921854213746
   },
   {
     "update": 415,
-    "global_step": 849920,
-    "num_episodes": 288,
-    "mean_reward": 23.637630491256715,
-    "mean_length": 3126.66,
-    "loss": 1.1654760837554932,
-    "sps": 2444.537765977125
+    "global_step": 1699840,
+    "num_episodes": 290,
+    "mean_reward": 454.8029096364975,
+    "mean_length": 6035.23,
+    "loss": 0.08915512263774872,
+    "sps": 836.5661659973193
   },
   {
     "update": 420,
-    "global_step": 860160,
-    "num_episodes": 288,
-    "mean_reward": 23.637630491256715,
-    "mean_length": 3126.66,
-    "loss": 0.7069441676139832,
-    "sps": 2402.946945101056
+    "global_step": 1720320,
+    "num_episodes": 294,
+    "mean_reward": 490.9899611520767,
+    "mean_length": 6134.91,
+    "loss": 9.022570610046387,
+    "sps": 1353.2280787984882
   },
   {
     "update": 425,
-    "global_step": 870400,
-    "num_episodes": 296,
-    "mean_reward": 26.219414005279543,
-    "mean_length": 3426.79,
-    "loss": 1.0991233587265015,
-    "sps": 1584.3855114127348
+    "global_step": 1740800,
+    "num_episodes": 299,
+    "mean_reward": 470.29952450752256,
+    "mean_length": 6136.24,
+    "loss": 1.4281823635101318,
+    "sps": 731.0193791394075
   },
   {
     "update": 430,
-    "global_step": 880640,
-    "num_episodes": 302,
-    "mean_reward": 24.143256397247313,
-    "mean_length": 3329.74,
-    "loss": 0.5094761252403259,
-    "sps": 1460.1652484484139
+    "global_step": 1761280,
+    "num_episodes": 299,
+    "mean_reward": 470.29952450752256,
+    "mean_length": 6136.24,
+    "loss": 0.12717652320861816,
+    "sps": 972.3142548127942
   },
   {
     "update": 435,
-    "global_step": 890880,
-    "num_episodes": 302,
-    "mean_reward": 24.143256397247313,
-    "mean_length": 3329.74,
-    "loss": 1.6765072345733643,
-    "sps": 2365.579425966714
+    "global_step": 1781760,
+    "num_episodes": 300,
+    "mean_reward": 470.8291971683502,
+    "mean_length": 6235.51,
+    "loss": 1.772779107093811,
+    "sps": 955.1582152126839
   },
   {
     "update": 440,
-    "global_step": 901120,
-    "num_episodes": 302,
-    "mean_reward": 24.143256397247313,
-    "mean_length": 3329.74,
-    "loss": -0.044437870383262634,
-    "sps": 2282.7314942544617
+    "global_step": 1802240,
+    "num_episodes": 305,
+    "mean_reward": 467.37178787708285,
+    "mean_length": 6335.05,
+    "loss": 45.16695022583008,
+    "sps": 263.8764177763192
   },
   {
     "update": 445,
-    "global_step": 911360,
-    "num_episodes": 308,
-    "mean_reward": 23.534629821777344,
-    "mean_length": 3238.0,
-    "loss": 0.4068017303943634,
-    "sps": 865.914540942848
+    "global_step": 1822720,
+    "num_episodes": 310,
+    "mean_reward": 468.54246531009676,
+    "mean_length": 6353.87,
+    "loss": 45.204833984375,
+    "sps": 765.4018779843883
   },
   {
     "update": 450,
-    "global_step": 921600,
-    "num_episodes": 314,
-    "mean_reward": 23.246068153381348,
-    "mean_length": 3138.43,
-    "loss": 0.32406821846961975,
-    "sps": 830.3529260738604
+    "global_step": 1843200,
+    "num_episodes": 310,
+    "mean_reward": 468.54246531009676,
+    "mean_length": 6353.87,
+    "loss": 17.418542861938477,
+    "sps": 716.3527304338312
   },
   {
     "update": 455,
-    "global_step": 931840,
-    "num_episodes": 316,
-    "mean_reward": 22.337194681167603,
-    "mean_length": 3057.38,
-    "loss": 0.21466466784477234,
-    "sps": 2411.145412824079
+    "global_step": 1863680,
+    "num_episodes": 315,
+    "mean_reward": 448.0337993764877,
+    "mean_length": 6059.53,
+    "loss": 35.471492767333984,
+    "sps": 752.6285328468343
   },
   {
     "update": 460,
-    "global_step": 942080,
-    "num_episodes": 316,
-    "mean_reward": 22.337194681167603,
-    "mean_length": 3057.38,
-    "loss": 4.0019731521606445,
-    "sps": 2372.8288290677337
+    "global_step": 1884160,
+    "num_episodes": 323,
+    "mean_reward": 390.5069852733612,
+    "mean_length": 5868.34,
+    "loss": 22.768428802490234,
+    "sps": 1442.0900075832017
   },
   {
     "update": 465,
-    "global_step": 952320,
-    "num_episodes": 321,
-    "mean_reward": 22.935798473358155,
-    "mean_length": 3155.68,
-    "loss": 3.8567330837249756,
-    "sps": 796.2439424516102
+    "global_step": 1904640,
+    "num_episodes": 325,
+    "mean_reward": 391.32213854789734,
+    "mean_length": 6066.81,
+    "loss": 1.743547797203064,
+    "sps": 2177.279445045517
   },
   {
     "update": 470,
-    "global_step": 962560,
-    "num_episodes": 326,
-    "mean_reward": 23.01468356132507,
-    "mean_length": 3080.66,
-    "loss": 0.04481935873627663,
-    "sps": 2363.29177191282
+    "global_step": 1925120,
+    "num_episodes": 325,
+    "mean_reward": 391.32213854789734,
+    "mean_length": 6066.81,
+    "loss": -0.12528716027736664,
+    "sps": 2128.448401830331
   },
   {
     "update": 475,
-    "global_step": 972800,
-    "num_episodes": 329,
-    "mean_reward": 24.00417799949646,
-    "mean_length": 3081.1,
-    "loss": -0.09712222218513489,
-    "sps": 2323.0109350014427
+    "global_step": 1945600,
+    "num_episodes": 326,
+    "mean_reward": 386.4432405376434,
+    "mean_length": 6066.81,
+    "loss": -0.0739293247461319,
+    "sps": 2048.6650107889627
   },
   {
     "update": 480,
-    "global_step": 983040,
-    "num_episodes": 329,
-    "mean_reward": 24.00417799949646,
-    "mean_length": 3081.1,
-    "loss": 0.29562610387802124,
-    "sps": 2249.6786763931746
+    "global_step": 1966080,
+    "num_episodes": 331,
+    "mean_reward": 395.79872769355774,
+    "mean_length": 6362.12,
+    "loss": 1.8403631448745728,
+    "sps": 1177.4940640961918
   },
   {
     "update": 485,
-    "global_step": 993280,
-    "num_episodes": 330,
-    "mean_reward": 24.60798607826233,
-    "mean_length": 3177.27,
-    "loss": 0.02162332460284233,
-    "sps": 2222.1627608215495
+    "global_step": 1986560,
+    "num_episodes": 333,
+    "mean_reward": 396.0874492454529,
+    "mean_length": 6461.8,
+    "loss": 21.89581298828125,
+    "sps": 700.649536593067
   },
   {
     "update": 490,
-    "global_step": 1003520,
-    "num_episodes": 334,
-    "mean_reward": 25.107961702346802,
-    "mean_length": 3177.27,
-    "loss": 0.30668944120407104,
-    "sps": 2336.9965399128855
+    "global_step": 2007040,
+    "num_episodes": 333,
+    "mean_reward": 396.0874492454529,
+    "mean_length": 6461.8,
+    "loss": 17.010061264038086,
+    "sps": 728.107784506257
   },
   {
     "update": 495,
-    "global_step": 1013760,
-    "num_episodes": 337,
-    "mean_reward": 25.60792893409729,
-    "mean_length": 3175.92,
-    "loss": 0.40231630206108093,
-    "sps": 1175.8081438816857
+    "global_step": 2027520,
+    "num_episodes": 338,
+    "mean_reward": 396.22795082092284,
+    "mean_length": 6560.14,
+    "loss": 73.3307876586914,
+    "sps": 359.9910748355061
   },
   {
     "update": 500,
-    "global_step": 1024000,
-    "num_episodes": 337,
-    "mean_reward": 25.60792893409729,
-    "mean_length": 3175.92,
-    "loss": 0.33998745679855347,
-    "sps": 2419.5867438311334
+    "global_step": 2048000,
+    "num_episodes": 347,
+    "mean_reward": 393.5666734457016,
+    "mean_length": 6464.03,
+    "loss": 51.052146911621094,
+    "sps": 186.4407003799165
   },
   {
     "update": 505,
-    "global_step": 1034240,
-    "num_episodes": 340,
-    "mean_reward": 25.602466859817504,
-    "mean_length": 3171.49,
-    "loss": 0.09274712204933167,
-    "sps": 2409.938615394154
+    "global_step": 2068480,
+    "num_episodes": 353,
+    "mean_reward": 363.4766788816452,
+    "mean_length": 6267.46,
+    "loss": 10.636628150939941,
+    "sps": 391.3156150286931
   },
   {
     "update": 510,
-    "global_step": 1044480,
-    "num_episodes": 346,
-    "mean_reward": 25.201329984664916,
-    "mean_length": 3263.94,
-    "loss": 0.2879057228565216,
-    "sps": 976.7251772331928
+    "global_step": 2088960,
+    "num_episodes": 353,
+    "mean_reward": 363.4766788816452,
+    "mean_length": 6267.46,
+    "loss": 5.408469200134277,
+    "sps": 392.291526612045
   },
   {
     "update": 515,
-    "global_step": 1054720,
-    "num_episodes": 347,
-    "mean_reward": 25.555161905288696,
-    "mean_length": 3361.65,
-    "loss": 0.39604732394218445,
-    "sps": 2406.938365656791
+    "global_step": 2109440,
+    "num_episodes": 356,
+    "mean_reward": 340.8062537956238,
+    "mean_length": 6159.0,
+    "loss": 0.881823718547821,
+    "sps": 887.267479612318
   },
   {
     "update": 520,
-    "global_step": 1064960,
-    "num_episodes": 347,
-    "mean_reward": 25.555161905288696,
-    "mean_length": 3361.65,
-    "loss": 0.03350641578435898,
-    "sps": 2384.9415454427835
+    "global_step": 2129920,
+    "num_episodes": 368,
+    "mean_reward": 318.0571813583374,
+    "mean_length": 5973.21,
+    "loss": 29.853599548339844,
+    "sps": 776.9091606744039
   },
   {
     "update": 525,
-    "global_step": 1075200,
-    "num_episodes": 348,
-    "mean_reward": 26.658946142196655,
-    "mean_length": 3459.36,
-    "loss": 0.8593693971633911,
-    "sps": 2415.202845406159
+    "global_step": 2150400,
+    "num_episodes": 375,
+    "mean_reward": 277.7464536857605,
+    "mean_length": 5779.94,
+    "loss": 0.9935208559036255,
+    "sps": 2837.3122923609985
   },
   {
     "update": 530,
-    "global_step": 1085440,
-    "num_episodes": 355,
-    "mean_reward": 27.258036041259764,
-    "mean_length": 3553.8,
-    "loss": 0.4878949820995331,
-    "sps": 710.1710154431543
+    "global_step": 2170880,
+    "num_episodes": 375,
+    "mean_reward": 277.7464536857605,
+    "mean_length": 5779.94,
+    "loss": 0.16118717193603516,
+    "sps": 2674.823411889354
   },
   {
     "update": 535,
-    "global_step": 1095680,
-    "num_episodes": 368,
-    "mean_reward": 28.800746297836305,
-    "mean_length": 3371.19,
-    "loss": 0.21109388768672943,
-    "sps": 532.1724281533873
+    "global_step": 2191360,
+    "num_episodes": 379,
+    "mean_reward": 276.7646808052063,
+    "mean_length": 5783.77,
+    "loss": 0.6545084118843079,
+    "sps": 2718.7471360755962
   },
   {
     "update": 540,
-    "global_step": 1105920,
-    "num_episodes": 371,
-    "mean_reward": 29.045905199050903,
-    "mean_length": 3369.84,
-    "loss": 0.14603528380393982,
-    "sps": 2335.2550876461346
+    "global_step": 2211840,
+    "num_episodes": 387,
+    "mean_reward": 273.41629776477816,
+    "mean_length": 5491.19,
+    "loss": 0.009996440261602402,
+    "sps": 2662.424914783932
   },
   {
     "update": 545,
-    "global_step": 1116160,
-    "num_episodes": 373,
-    "mean_reward": 29.899697828292847,
-    "mean_length": 3466.01,
-    "loss": -0.04600970447063446,
-    "sps": 2305.921805837758
+    "global_step": 2232320,
+    "num_episodes": 393,
+    "mean_reward": 237.44847791194917,
+    "mean_length": 5288.65,
+    "loss": -0.07593612372875214,
+    "sps": 2721.3806445317573
   },
   {
     "update": 550,
-    "global_step": 1126400,
-    "num_episodes": 374,
-    "mean_reward": 31.003479852676392,
-    "mean_length": 3565.07,
-    "loss": 0.11138609051704407,
-    "sps": 2400.3530395714765
+    "global_step": 2252800,
+    "num_episodes": 393,
+    "mean_reward": 237.44847791194917,
+    "mean_length": 5288.65,
+    "loss": -0.013108465820550919,
+    "sps": 2496.4586032280413
   },
   {
     "update": 555,
-    "global_step": 1136640,
-    "num_episodes": 377,
-    "mean_reward": 31.149640073776244,
-    "mean_length": 3467.36,
-    "loss": 0.35408008098602295,
-    "sps": 1863.2471573876685
+    "global_step": 2273280,
+    "num_episodes": 397,
+    "mean_reward": 214.58075400829316,
+    "mean_length": 5193.33,
+    "loss": 0.026919692754745483,
+    "sps": 2688.731731235585
   },
   {
     "update": 560,
-    "global_step": 1146880,
-    "num_episodes": 378,
-    "mean_reward": 31.50347306251526,
-    "mean_length": 3563.53,
-    "loss": 0.03238815441727638,
-    "sps": 2410.263871749416
+    "global_step": 2293760,
+    "num_episodes": 407,
+    "mean_reward": 159.23116432189943,
+    "mean_length": 4793.65,
+    "loss": 2.2952957153320312,
+    "sps": 966.333276372859
   },
   {
     "update": 565,
-    "global_step": 1157120,
-    "num_episodes": 379,
-    "mean_reward": 31.503473825454712,
-    "mean_length": 3563.53,
-    "loss": 1.2669872045516968,
-    "sps": 2435.9597448206396
+    "global_step": 2314240,
+    "num_episodes": 413,
+    "mean_reward": 159.70496647834779,
+    "mean_length": 4791.54,
+    "loss": 1.3611968755722046,
+    "sps": 2550.010825605688
   },
   {
     "update": 570,
-    "global_step": 1167360,
-    "num_episodes": 381,
-    "mean_reward": 30.14463397026062,
-    "mean_length": 3465.82,
-    "loss": 1.1442028284072876,
-    "sps": 2364.850671069342
+    "global_step": 2334720,
+    "num_episodes": 413,
+    "mean_reward": 159.70496647834779,
+    "mean_length": 4791.54,
+    "loss": 1.439621090888977,
+    "sps": 2484.751376024715
   },
   {
     "update": 575,
-    "global_step": 1177600,
-    "num_episodes": 387,
-    "mean_reward": 27.80320453643799,
-    "mean_length": 3366.44,
-    "loss": 0.08564125001430511,
-    "sps": 566.4595364452883
+    "global_step": 2355200,
+    "num_episodes": 417,
+    "mean_reward": 158.6872748184204,
+    "mean_length": 4786.52,
+    "loss": 12.480733871459961,
+    "sps": 428.08037217860056
   },
   {
     "update": 580,
-    "global_step": 1187840,
-    "num_episodes": 389,
-    "mean_reward": 27.20208529472351,
-    "mean_length": 3266.87,
-    "loss": 0.14057211577892303,
-    "sps": 2297.6598442501095
+    "global_step": 2375680,
+    "num_episodes": 424,
+    "mean_reward": 126.5506838798523,
+    "mean_length": 4703.64,
+    "loss": 12.992765426635742,
+    "sps": 1524.6391897022352
   },
   {
     "update": 585,
-    "global_step": 1198080,
-    "num_episodes": 396,
-    "mean_reward": 26.080838441848755,
-    "mean_length": 3169.09,
-    "loss": 0.0875428318977356,
-    "sps": 2440.645435446852
+    "global_step": 2396160,
+    "num_episodes": 427,
+    "mean_reward": 131.45039747238158,
+    "mean_length": 4703.64,
+    "loss": 11.845376968383789,
+    "sps": 487.7409629471044
   },
   {
     "update": 590,
-    "global_step": 1208320,
-    "num_episodes": 403,
-    "mean_reward": 24.67230319023132,
-    "mean_length": 3069.72,
-    "loss": -0.005318094044923782,
-    "sps": 2415.1756827351583
+    "global_step": 2416640,
+    "num_episodes": 427,
+    "mean_reward": 131.45039747238158,
+    "mean_length": 4703.64,
+    "loss": 0.3306816816329956,
+    "sps": 906.10311953954
   },
   {
     "update": 595,
-    "global_step": 1218560,
-    "num_episodes": 406,
-    "mean_reward": 23.778743057250978,
-    "mean_length": 3164.03,
-    "loss": 0.07221215218305588,
-    "sps": 1535.4284659165353
+    "global_step": 2437120,
+    "num_episodes": 429,
+    "mean_reward": 123.15537308216095,
+    "mean_length": 4604.79,
+    "loss": 0.30884307622909546,
+    "sps": 938.566230501384
   },
   {
     "update": 600,
-    "global_step": 1228800,
-    "num_episodes": 413,
-    "mean_reward": 22.0106067276001,
-    "mean_length": 2967.58,
-    "loss": 0.35672813653945923,
-    "sps": 2410.0731702663725
+    "global_step": 2457600,
+    "num_episodes": 437,
+    "mean_reward": 133.3434053325653,
+    "mean_length": 4405.56,
+    "loss": 0.21275369822978973,
+    "sps": 1018.040863452185
   },
   {
     "update": 605,
-    "global_step": 1239040,
-    "num_episodes": 418,
-    "mean_reward": 20.706662797927855,
-    "mean_length": 2849.49,
-    "loss": 0.1961039900779724,
-    "sps": 2444.0300319234293
+    "global_step": 2478080,
+    "num_episodes": 442,
+    "mean_reward": 171.27508597373964,
+    "mean_length": 4501.98,
+    "loss": 0.47754910588264465,
+    "sps": 2289.461146828872
   },
   {
     "update": 610,
-    "global_step": 1249280,
-    "num_episodes": 420,
-    "mean_reward": 21.058296146392824,
-    "mean_length": 2949.06,
-    "loss": 0.20069828629493713,
-    "sps": 2409.835173874158
+    "global_step": 2498560,
+    "num_episodes": 442,
+    "mean_reward": 171.27508597373964,
+    "mean_length": 4501.98,
+    "loss": 0.5260030627250671,
+    "sps": 2191.020876073232
   },
   {
     "update": 615,
-    "global_step": 1259520,
-    "num_episodes": 420,
-    "mean_reward": 21.058296146392824,
-    "mean_length": 2949.06,
-    "loss": 0.6283493041992188,
-    "sps": 2351.264188304028
+    "global_step": 2519040,
+    "num_episodes": 446,
+    "mean_reward": 134.8440179824829,
+    "mean_length": 4406.13,
+    "loss": 0.03903695195913315,
+    "sps": 2094.500891632649
   },
   {
     "update": 620,
-    "global_step": 1269760,
-    "num_episodes": 436,
-    "mean_reward": 16.700364379882814,
-    "mean_length": 2538.45,
-    "loss": 0.06421943008899689,
-    "sps": 581.6206601898414
+    "global_step": 2539520,
+    "num_episodes": 462,
+    "mean_reward": 108.21314854621887,
+    "mean_length": 4325.47,
+    "loss": 7.972282409667969,
+    "sps": 1843.9384231501685
   },
   {
     "update": 625,
-    "global_step": 1280000,
-    "num_episodes": 437,
-    "mean_reward": 17.554159069061278,
-    "mean_length": 2634.62,
-    "loss": 0.025755397975444794,
-    "sps": 2336.2122156272303
+    "global_step": 2560000,
+    "num_episodes": 467,
+    "mean_reward": 103.93524898529053,
+    "mean_length": 4323.9,
+    "loss": 0.2857341468334198,
+    "sps": 2844.0073567316244
   },
   {
     "update": 630,
-    "global_step": 1290240,
-    "num_episodes": 438,
-    "mean_reward": 17.804676685333252,
-    "mean_length": 2634.62,
-    "loss": -0.039572782814502716,
-    "sps": 2353.5698753913625
+    "global_step": 2580480,
+    "num_episodes": 467,
+    "mean_reward": 103.93524898529053,
+    "mean_length": 4323.9,
+    "loss": -0.0014126598834991455,
+    "sps": 2662.6258687696713
   },
   {
     "update": 635,
-    "global_step": 1300480,
-    "num_episodes": 438,
-    "mean_reward": 17.804676685333252,
-    "mean_length": 2634.62,
-    "loss": -0.0706489086151123,
-    "sps": 2283.6836848035246
+    "global_step": 2600960,
+    "num_episodes": 469,
+    "mean_reward": 105.16953000545502,
+    "mean_length": 4425.22,
+    "loss": -0.11831197887659073,
+    "sps": 2712.08306677995
   },
   {
     "update": 640,
-    "global_step": 1310720,
-    "num_episodes": 441,
-    "mean_reward": 19.15775371551514,
-    "mean_length": 2733.68,
-    "loss": -0.09373792260885239,
-    "sps": 1474.6357445156093
+    "global_step": 2621440,
+    "num_episodes": 474,
+    "mean_reward": 110.30714406490326,
+    "mean_length": 4816.19,
+    "loss": 1.1699683666229248,
+    "sps": 2190.63868238328
   },
   {
     "update": 645,
-    "global_step": 1320960,
-    "num_episodes": 442,
-    "mean_reward": 20.157576084136963,
-    "mean_length": 2733.68,
-    "loss": 0.11351752281188965,
-    "sps": 2334.5626594322757
+    "global_step": 2641920,
+    "num_episodes": 478,
+    "mean_reward": 115.03492943286896,
+    "mean_length": 4720.8,
+    "loss": 0.8407511115074158,
+    "sps": 2477.6471668004287
   },
   {
     "update": 650,
-    "global_step": 1331200,
-    "num_episodes": 444,
-    "mean_reward": 20.267150926589967,
-    "mean_length": 2833.09,
-    "loss": -0.18438173830509186,
-    "sps": 2236.677306457589
+    "global_step": 2662400,
+    "num_episodes": 478,
+    "mean_reward": 115.03492943286896,
+    "mean_length": 4720.8,
+    "loss": 0.41076016426086426,
+    "sps": 2375.4220186429184
   },
   {
     "update": 655,
-    "global_step": 1341440,
-    "num_episodes": 444,
-    "mean_reward": 20.267150926589967,
-    "mean_length": 2833.09,
-    "loss": -0.07986398041248322,
-    "sps": 2210.239531580035
+    "global_step": 2682880,
+    "num_episodes": 480,
+    "mean_reward": 113.97553071975707,
+    "mean_length": 4722.63,
+    "loss": 1.8437230587005615,
+    "sps": 437.20362392139924
   },
   {
     "update": 660,
-    "global_step": 1351680,
-    "num_episodes": 447,
-    "mean_reward": 21.873516216278077,
-    "mean_length": 2929.77,
-    "loss": -0.05535223335027695,
-    "sps": 2385.7304437353546
+    "global_step": 2703360,
+    "num_episodes": 488,
+    "mean_reward": 121.07888405799866,
+    "mean_length": 4921.4,
+    "loss": 9.341552734375,
+    "sps": 433.1180294441828
   },
   {
     "update": 665,
-    "global_step": 1361920,
-    "num_episodes": 449,
-    "mean_reward": 21.519568309783935,
-    "mean_length": 2830.71,
-    "loss": 0.10434924066066742,
-    "sps": 2330.8125431773424
+    "global_step": 2723840,
+    "num_episodes": 493,
+    "mean_reward": 122.25540927648544,
+    "mean_length": 4927.03,
+    "loss": 3.2276527881622314,
+    "sps": 311.23534986570854
   },
   {
     "update": 670,
-    "global_step": 1372160,
-    "num_episodes": 451,
-    "mean_reward": 22.623257703781128,
-    "mean_length": 2927.07,
-    "loss": 0.29875361919403076,
-    "sps": 2388.3964178785764
+    "global_step": 2744320,
+    "num_episodes": 493,
+    "mean_reward": 122.25540927648544,
+    "mean_length": 4927.03,
+    "loss": 1.626711130142212,
+    "sps": 443.5605257662605
   },
   {
     "update": 675,
-    "global_step": 1382400,
-    "num_episodes": 451,
-    "mean_reward": 22.623257703781128,
-    "mean_length": 2927.07,
-    "loss": 0.01194491982460022,
-    "sps": 2360.8138719071453
+    "global_step": 2764800,
+    "num_episodes": 497,
+    "mean_reward": 128.49539656877516,
+    "mean_length": 4990.32,
+    "loss": 1.4737187623977661,
+    "sps": 557.1794951843984
   },
   {
     "update": 680,
-    "global_step": 1392640,
-    "num_episodes": 454,
-    "mean_reward": 23.234625358581543,
-    "mean_length": 3026.64,
-    "loss": 0.1673501431941986,
-    "sps": 1557.6244924988657
+    "global_step": 2785280,
+    "num_episodes": 501,
+    "mean_reward": 133.27627504587173,
+    "mean_length": 5189.29,
+    "loss": 3.2421000003814697,
+    "sps": 227.14390586968517
   },
   {
     "update": 685,
-    "global_step": 1402880,
-    "num_episodes": 458,
-    "mean_reward": 23.841174821853638,
-    "mean_length": 3124.86,
-    "loss": 0.026581741869449615,
-    "sps": 2296.8415092422506
+    "global_step": 2805760,
+    "num_episodes": 503,
+    "mean_reward": 134.89025341749192,
+    "mean_length": 5319.23,
+    "loss": 19.16809844970703,
+    "sps": 503.2482073037513
   },
   {
     "update": 690,
-    "global_step": 1413120,
-    "num_episodes": 459,
-    "mean_reward": 23.949846239089965,
-    "mean_length": 3221.03,
-    "loss": 1.637216329574585,
-    "sps": 413.9001244167228
+    "global_step": 2826240,
+    "num_episodes": 503,
+    "mean_reward": 134.89025341749192,
+    "mean_length": 5319.23,
+    "loss": 0.3121195435523987,
+    "sps": 1237.8635799099984
   },
   {
     "update": 695,
-    "global_step": 1423360,
-    "num_episodes": 465,
-    "mean_reward": 24.109339590072633,
-    "mean_length": 3246.44,
-    "loss": 0.4041682779788971,
-    "sps": 1973.9302703248002
+    "global_step": 2846720,
+    "num_episodes": 505,
+    "mean_reward": 140.31052175283432,
+    "mean_length": 5418.9,
+    "loss": 10.290376663208008,
+    "sps": 894.1160824447923
   },
   {
     "update": 700,
-    "global_step": 1433600,
-    "num_episodes": 466,
-    "mean_reward": 24.21024447441101,
-    "mean_length": 3344.24,
-    "loss": 0.13472315669059753,
-    "sps": 2279.0860535321326
+    "global_step": 2867200,
+    "num_episodes": 511,
+    "mean_reward": 142.8947832798958,
+    "mean_length": 5596.12,
+    "loss": 8.96629810333252,
+    "sps": 2766.323313198168
   },
   {
     "update": 705,
-    "global_step": 1443840,
-    "num_episodes": 468,
-    "mean_reward": 24.856354351043702,
-    "mean_length": 3440.41,
-    "loss": 0.6666802763938904,
-    "sps": 2315.9586381196755
+    "global_step": 2887680,
+    "num_episodes": 514,
+    "mean_reward": 143.8507553601265,
+    "mean_length": 5598.39,
+    "loss": -0.03336918354034424,
+    "sps": 2819.0861765762347
   },
   {
     "update": 710,
-    "global_step": 1454080,
-    "num_episodes": 468,
-    "mean_reward": 24.856354351043702,
-    "mean_length": 3440.41,
-    "loss": -0.15654346346855164,
-    "sps": 2242.2811278107365
+    "global_step": 2908160,
+    "num_episodes": 514,
+    "mean_reward": 143.8507553601265,
+    "mean_length": 5598.39,
+    "loss": 1.484508991241455,
+    "sps": 2804.209085249585
   },
   {
     "update": 715,
-    "global_step": 1464320,
-    "num_episodes": 477,
-    "mean_reward": 24.00254797935486,
-    "mean_length": 3342.7,
-    "loss": -0.14158941805362701,
-    "sps": 2301.1874356888225
+    "global_step": 2928640,
+    "num_episodes": 516,
+    "mean_reward": 146.61101296186447,
+    "mean_length": 5697.46,
+    "loss": 0.41980481147766113,
+    "sps": 2853.5154122563904
   },
   {
     "update": 720,
-    "global_step": 1474560,
-    "num_episodes": 480,
-    "mean_reward": 24.0372780418396,
-    "mean_length": 3146.96,
-    "loss": 0.10493358224630356,
-    "sps": 843.3732587375952
+    "global_step": 2949120,
+    "num_episodes": 521,
+    "mean_reward": 159.6500442767143,
+    "mean_length": 5978.65,
+    "loss": 3.3171966075897217,
+    "sps": 337.7495209248038
   },
   {
     "update": 725,
-    "global_step": 1484800,
-    "num_episodes": 481,
-    "mean_reward": 23.896179141998292,
-    "mean_length": 3244.67,
-    "loss": 0.01014188677072525,
-    "sps": 2387.5931465826584
+    "global_step": 2969600,
+    "num_episodes": 523,
+    "mean_reward": 165.79403271436692,
+    "mean_length": 6078.2,
+    "loss": 2.299358606338501,
+    "sps": 488.4530631699151
   },
   {
     "update": 730,
-    "global_step": 1495040,
-    "num_episodes": 481,
-    "mean_reward": 23.896179141998292,
-    "mean_length": 3244.67,
-    "loss": 1.5800291299819946,
-    "sps": 2199.482155508925
+    "global_step": 2990080,
+    "num_episodes": 523,
+    "mean_reward": 165.79403271436692,
+    "mean_length": 6078.2,
+    "loss": 2.0765156745910645,
+    "sps": 622.5255664645315
   },
   {
     "update": 735,
-    "global_step": 1505280,
-    "num_episodes": 486,
-    "mean_reward": 23.005251121520995,
-    "mean_length": 3344.42,
-    "loss": 2.476745128631592,
-    "sps": 555.9822817000502
+    "global_step": 3010560,
+    "num_episodes": 525,
+    "mean_reward": 165.0441936945915,
+    "mean_length": 6078.2,
+    "loss": 0.5185770988464355,
+    "sps": 912.7167424710788
   },
   {
     "update": 740,
-    "global_step": 1515520,
-    "num_episodes": 487,
-    "mean_reward": 22.755260047912596,
-    "mean_length": 3344.42,
-    "loss": 0.043340131640434265,
-    "sps": 2358.713519842451
+    "global_step": 3031040,
+    "num_episodes": 529,
+    "mean_reward": 169.13619287729264,
+    "mean_length": 6177.05,
+    "loss": 0.8914755582809448,
+    "sps": 2277.7924528126773
   },
   {
     "update": 745,
-    "global_step": 1525760,
-    "num_episodes": 489,
-    "mean_reward": 23.35637879371643,
-    "mean_length": 3442.64,
-    "loss": 0.8548387885093689,
-    "sps": 2384.7462230357132
+    "global_step": 3051520,
+    "num_episodes": 531,
+    "mean_reward": 182.76090200662614,
+    "mean_length": 6177.05,
+    "loss": -0.03897722810506821,
+    "sps": 2938.7255967172237
   },
   {
     "update": 750,
-    "global_step": 1536000,
-    "num_episodes": 489,
-    "mean_reward": 23.35637879371643,
-    "mean_length": 3442.64,
-    "loss": 0.06128331273794174,
-    "sps": 2389.02547098786
+    "global_step": 3072000,
+    "num_episodes": 531,
+    "mean_reward": 182.76090200662614,
+    "mean_length": 6177.05,
+    "loss": -0.07192348688840866,
+    "sps": 2896.8996334668986
   },
   {
     "update": 755,
-    "global_step": 1546240,
-    "num_episodes": 495,
-    "mean_reward": 23.45271695137024,
-    "mean_length": 3539.52,
-    "loss": 0.010654259473085403,
-    "sps": 451.1509132304138
+    "global_step": 3092480,
+    "num_episodes": 535,
+    "mean_reward": 176.57018686056136,
+    "mean_length": 6276.7,
+    "loss": 17.123456954956055,
+    "sps": 1361.0160818187624
   },
   {
     "update": 760,
-    "global_step": 1556480,
-    "num_episodes": 498,
-    "mean_reward": 24.20286971092224,
-    "mean_length": 3541.89,
-    "loss": -0.015714531764388084,
-    "sps": 2272.008455427166
+    "global_step": 3112960,
+    "num_episodes": 539,
+    "mean_reward": 143.45233328580855,
+    "mean_length": 6475.8,
+    "loss": 1.7440789937973022,
+    "sps": 431.96268578339885
   },
   {
     "update": 765,
-    "global_step": 1566720,
-    "num_episodes": 503,
-    "mean_reward": 24.520383558273316,
-    "mean_length": 3642.43,
-    "loss": 1.2781355381011963,
-    "sps": 2234.132232505493
+    "global_step": 3133440,
+    "num_episodes": 541,
+    "mean_reward": 143.19214475393295,
+    "mean_length": 6476.28,
+    "loss": 1.2793841361999512,
+    "sps": 694.1149265905251
   },
   {
     "update": 770,
-    "global_step": 1576960,
-    "num_episodes": 503,
-    "mean_reward": 24.520383558273316,
-    "mean_length": 3642.43,
-    "loss": 0.005558963865041733,
-    "sps": 2252.444167541167
+    "global_step": 3153920,
+    "num_episodes": 541,
+    "mean_reward": 143.19214475393295,
+    "mean_length": 6476.28,
+    "loss": 27.601778030395508,
+    "sps": 666.6380756689646
   },
   {
     "update": 775,
-    "global_step": 1587200,
-    "num_episodes": 508,
-    "mean_reward": 26.765025606155394,
-    "mean_length": 3642.43,
-    "loss": 1.5083853006362915,
-    "sps": 1001.7898928543148
+    "global_step": 3174400,
+    "num_episodes": 545,
+    "mean_reward": 160.25242776632308,
+    "mean_length": 6772.51,
+    "loss": 3.569728136062622,
+    "sps": 235.53664992575125
   },
   {
     "update": 780,
-    "global_step": 1597440,
-    "num_episodes": 512,
-    "mean_reward": 27.61695213317871,
-    "mean_length": 3739.96,
-    "loss": -0.03274097293615341,
-    "sps": 406.1884387297904
+    "global_step": 3194880,
+    "num_episodes": 548,
+    "mean_reward": 161.8272612643242,
+    "mean_length": 6969.88,
+    "loss": 1.578532338142395,
+    "sps": 345.17499217473494
   },
   {
     "update": 785,
-    "global_step": 1607680,
-    "num_episodes": 517,
-    "mean_reward": 29.239461097717285,
-    "mean_length": 3749.65,
-    "loss": 0.056141212582588196,
-    "sps": 711.222362259853
+    "global_step": 3215360,
+    "num_episodes": 550,
+    "mean_reward": 161.4852223944664,
+    "mean_length": 7031.97,
+    "loss": 6.10168981552124,
+    "sps": 321.34142444031875
   },
   {
     "update": 790,
-    "global_step": 1617920,
-    "num_episodes": 517,
-    "mean_reward": 29.239461097717285,
-    "mean_length": 3749.65,
-    "loss": 0.08803388476371765,
-    "sps": 2290.465987254353
+    "global_step": 3235840,
+    "num_episodes": 550,
+    "mean_reward": 161.4852223944664,
+    "mean_length": 7031.97,
+    "loss": 0.8625541925430298,
+    "sps": 739.6504838117347
   },
   {
     "update": 795,
-    "global_step": 1628160,
-    "num_episodes": 524,
-    "mean_reward": 28.533955936431884,
-    "mean_length": 3547.65,
-    "loss": 5.9130449295043945,
-    "sps": 1206.367279137959
+    "global_step": 3256320,
+    "num_episodes": 554,
+    "mean_reward": 162.4044385075569,
+    "mean_length": 7219.96,
+    "loss": 2.502009868621826,
+    "sps": 191.1365478881192
   },
   {
     "update": 800,
-    "global_step": 1638400,
-    "num_episodes": 528,
-    "mean_reward": 29.898164215087892,
-    "mean_length": 3657.29,
-    "loss": 0.8671517372131348,
-    "sps": 2301.788037202966
+    "global_step": 3276800,
+    "num_episodes": 563,
+    "mean_reward": 162.7963340115547,
+    "mean_length": 7220.73,
+    "loss": 8.730064392089844,
+    "sps": 386.9629605603807
+  },
+  {
+    "update": 805,
+    "global_step": 3297280,
+    "num_episodes": 566,
+    "mean_reward": 165.31176669359206,
+    "mean_length": 7360.77,
+    "loss": 48.92124557495117,
+    "sps": 705.0215184629429
+  },
+  {
+    "update": 810,
+    "global_step": 3317760,
+    "num_episodes": 566,
+    "mean_reward": 165.31176669359206,
+    "mean_length": 7360.77,
+    "loss": 0.6651354432106018,
+    "sps": 801.9215448692757
+  },
+  {
+    "update": 815,
+    "global_step": 3338240,
+    "num_episodes": 569,
+    "mean_reward": 180.35619497060776,
+    "mean_length": 7453.65,
+    "loss": 0.9045634865760803,
+    "sps": 2078.869268499869
+  },
+  {
+    "update": 820,
+    "global_step": 3358720,
+    "num_episodes": 575,
+    "mean_reward": 193.25147196531296,
+    "mean_length": 7258.9,
+    "loss": 5.477941989898682,
+    "sps": 1253.1294143150922
+  },
+  {
+    "update": 825,
+    "global_step": 3379200,
+    "num_episodes": 581,
+    "mean_reward": 185.37936263799668,
+    "mean_length": 7158.59,
+    "loss": 0.19656488299369812,
+    "sps": 3241.297612628403
+  },
+  {
+    "update": 830,
+    "global_step": 3399680,
+    "num_episodes": 581,
+    "mean_reward": 185.37936263799668,
+    "mean_length": 7158.59,
+    "loss": 0.5765475034713745,
+    "sps": 2869.6879773523438
+  },
+  {
+    "update": 835,
+    "global_step": 3420160,
+    "num_episodes": 583,
+    "mean_reward": 185.1090102362633,
+    "mean_length": 7158.59,
+    "loss": 2.0618038177490234,
+    "sps": 415.0631773970818
+  },
+  {
+    "update": 840,
+    "global_step": 3440640,
+    "num_episodes": 588,
+    "mean_reward": 189.7356357884407,
+    "mean_length": 7352.61,
+    "loss": 9.016275405883789,
+    "sps": 227.3230128869974
+  },
+  {
+    "update": 845,
+    "global_step": 3461120,
+    "num_episodes": 591,
+    "mean_reward": 189.1485662293434,
+    "mean_length": 7394.25,
+    "loss": 22.739429473876953,
+    "sps": 314.2023925066658
+  },
+  {
+    "update": 850,
+    "global_step": 3481600,
+    "num_episodes": 592,
+    "mean_reward": 189.12477667808534,
+    "mean_length": 7443.48,
+    "loss": 1.6546752452850342,
+    "sps": 957.4586504289063
+  },
+  {
+    "update": 855,
+    "global_step": 3502080,
+    "num_episodes": 594,
+    "mean_reward": 187.17996717453002,
+    "mean_length": 7507.25,
+    "loss": 6.797643184661865,
+    "sps": 412.71199087606476
+  },
+  {
+    "update": 860,
+    "global_step": 3522560,
+    "num_episodes": 596,
+    "mean_reward": 193.70433765888214,
+    "mean_length": 7638.31,
+    "loss": 0.2814023196697235,
+    "sps": 809.1402642140959
+  },
+  {
+    "update": 865,
+    "global_step": 3543040,
+    "num_episodes": 600,
+    "mean_reward": 192.16364938259125,
+    "mean_length": 7538.65,
+    "loss": 4.5296950340271,
+    "sps": 934.0269243194301
+  },
+  {
+    "update": 870,
+    "global_step": 3563520,
+    "num_episodes": 602,
+    "mean_reward": 191.83141272068025,
+    "mean_length": 7569.62,
+    "loss": 0.12454970180988312,
+    "sps": 459.3460462463923
+  },
+  {
+    "update": 875,
+    "global_step": 3584000,
+    "num_episodes": 605,
+    "mean_reward": 185.7972752714157,
+    "mean_length": 7412.33,
+    "loss": 0.010119002312421799,
+    "sps": 2984.2673687681013
+  },
+  {
+    "update": 880,
+    "global_step": 3604480,
+    "num_episodes": 607,
+    "mean_reward": 191.80364267826081,
+    "mean_length": 7423.25,
+    "loss": 1.5759342908859253,
+    "sps": 2498.994384783772
+  },
+  {
+    "update": 885,
+    "global_step": 3624960,
+    "num_episodes": 611,
+    "mean_reward": 197.66494411945342,
+    "mean_length": 7523.71,
+    "loss": 0.07411646842956543,
+    "sps": 2398.8293936193445
+  },
+  {
+    "update": 890,
+    "global_step": 3645440,
+    "num_episodes": 612,
+    "mean_reward": 199.16475445270538,
+    "mean_length": 7523.71,
+    "loss": -0.09818007051944733,
+    "sps": 2494.5939188365055
+  },
+  {
+    "update": 895,
+    "global_step": 3665920,
+    "num_episodes": 616,
+    "mean_reward": 198.68793545246123,
+    "mean_length": 7520.61,
+    "loss": 0.07828275859355927,
+    "sps": 1930.4566445688752
+  },
+  {
+    "update": 900,
+    "global_step": 3686400,
+    "num_episodes": 618,
+    "mean_reward": 198.65358025074005,
+    "mean_length": 7422.9,
+    "loss": 0.6579947471618652,
+    "sps": 2636.224514742254
+  },
+  {
+    "update": 905,
+    "global_step": 3706880,
+    "num_episodes": 622,
+    "mean_reward": 200.68803305149078,
+    "mean_length": 7420.23,
+    "loss": 1.256749153137207,
+    "sps": 2869.3812284251953
+  },
+  {
+    "update": 910,
+    "global_step": 3727360,
+    "num_episodes": 624,
+    "mean_reward": 198.5566199350357,
+    "mean_length": 7420.23,
+    "loss": 1.2612106800079346,
+    "sps": 2818.4419350605035
+  },
+  {
+    "update": 915,
+    "global_step": 3747840,
+    "num_episodes": 626,
+    "mean_reward": 205.93744747638704,
+    "mean_length": 7420.23,
+    "loss": 0.7723605036735535,
+    "sps": 2519.097807723044
+  },
+  {
+    "update": 920,
+    "global_step": 3768320,
+    "num_episodes": 628,
+    "mean_reward": 201.4210061311722,
+    "mean_length": 7420.23,
+    "loss": 7.346652507781982,
+    "sps": 2242.0312258079334
+  },
+  {
+    "update": 925,
+    "global_step": 3788800,
+    "num_episodes": 630,
+    "mean_reward": 189.0587257051468,
+    "mean_length": 7420.23,
+    "loss": 5.038862705230713,
+    "sps": 383.270562834207
+  },
+  {
+    "update": 930,
+    "global_step": 3809280,
+    "num_episodes": 633,
+    "mean_reward": 189.46342292308807,
+    "mean_length": 7360.68,
+    "loss": 0.39050325751304626,
+    "sps": 1858.9704973292514
+  },
+  {
+    "update": 935,
+    "global_step": 3829760,
+    "num_episodes": 635,
+    "mean_reward": 198.2463042974472,
+    "mean_length": 7459.69,
+    "loss": 53.42470169067383,
+    "sps": 1069.4430639241739
+  },
+  {
+    "update": 940,
+    "global_step": 3850240,
+    "num_episodes": 636,
+    "mean_reward": 205.62649121761322,
+    "mean_length": 7459.69,
+    "loss": 1.097938895225525,
+    "sps": 1447.2739010301416
+  },
+  {
+    "update": 945,
+    "global_step": 3870720,
+    "num_episodes": 638,
+    "mean_reward": 220.27144483089447,
+    "mean_length": 7459.69,
+    "loss": 0.30668070912361145,
+    "sps": 2118.085155562087
+  },
+  {
+    "update": 950,
+    "global_step": 3891200,
+    "num_episodes": 641,
+    "mean_reward": 228.60497048854828,
+    "mean_length": 7558.87,
+    "loss": 1.160091519355774,
+    "sps": 1340.3946598207713
+  },
+  {
+    "update": 955,
+    "global_step": 3911680,
+    "num_episodes": 643,
+    "mean_reward": 229.18769864559172,
+    "mean_length": 7558.87,
+    "loss": 1.3600318431854248,
+    "sps": 2017.4738503217943
+  },
+  {
+    "update": 960,
+    "global_step": 3932160,
+    "num_episodes": 645,
+    "mean_reward": 226.2710527563095,
+    "mean_length": 7558.87,
+    "loss": 0.5724247097969055,
+    "sps": 2190.27561016469
+  },
+  {
+    "update": 965,
+    "global_step": 3952640,
+    "num_episodes": 647,
+    "mean_reward": 233.28511716365813,
+    "mean_length": 7558.87,
+    "loss": 0.6445204019546509,
+    "sps": 425.1404956260307
+  },
+  {
+    "update": 970,
+    "global_step": 3973120,
+    "num_episodes": 649,
+    "mean_reward": 234.76467885494233,
+    "mean_length": 7558.87,
+    "loss": 5.908900737762451,
+    "sps": 291.87626703381966
+  },
+  {
+    "update": 975,
+    "global_step": 3993600,
+    "num_episodes": 651,
+    "mean_reward": 236.37012548923494,
+    "mean_length": 7592.95,
+    "loss": 16.730175018310547,
+    "sps": 386.3945177442575
+  },
+  {
+    "update": 980,
+    "global_step": 4014080,
+    "num_episodes": 655,
+    "mean_reward": 240.30179923057557,
+    "mean_length": 7654.13,
+    "loss": 27.565311431884766,
+    "sps": 653.7194172279133
+  },
+  {
+    "update": 985,
+    "global_step": 4034560,
+    "num_episodes": 656,
+    "mean_reward": 241.5855724811554,
+    "mean_length": 7750.3,
+    "loss": 1.2501119375228882,
+    "sps": 1185.5585164533727
+  },
+  {
+    "update": 990,
+    "global_step": 4055040,
+    "num_episodes": 657,
+    "mean_reward": 242.3764306640625,
+    "mean_length": 7849.96,
+    "loss": 0.11368373781442642,
+    "sps": 953.2503801655828
+  },
+  {
+    "update": 995,
+    "global_step": 4075520,
+    "num_episodes": 660,
+    "mean_reward": 266.251427526474,
+    "mean_length": 8045.38,
+    "loss": 5.969295978546143,
+    "sps": 384.7242265220645
+  },
+  {
+    "update": 1000,
+    "global_step": 4096000,
+    "num_episodes": 664,
+    "mean_reward": 282.5860444736481,
+    "mean_length": 8176.85,
+    "loss": 27.033348083496094,
+    "sps": 376.322863993888
+  },
+  {
+    "update": 1005,
+    "global_step": 4116480,
+    "num_episodes": 665,
+    "mean_reward": 283.86758942604064,
+    "mean_length": 8232.26,
+    "loss": 7.4467058181762695,
+    "sps": 374.6383841753984
+  },
+  {
+    "update": 1010,
+    "global_step": 4136960,
+    "num_episodes": 666,
+    "mean_reward": 294.7833961582184,
+    "mean_length": 8232.26,
+    "loss": 17.35069465637207,
+    "sps": 308.584037781751
+  },
+  {
+    "update": 1015,
+    "global_step": 4157440,
+    "num_episodes": 669,
+    "mean_reward": 281.23537853717806,
+    "mean_length": 8289.35,
+    "loss": 1.9592938423156738,
+    "sps": 576.3621863402531
+  },
+  {
+    "update": 1020,
+    "global_step": 4177920,
+    "num_episodes": 675,
+    "mean_reward": 275.6631519651413,
+    "mean_length": 8186.57,
+    "loss": 23.52161407470703,
+    "sps": 661.3091435538709
+  },
+  {
+    "update": 1025,
+    "global_step": 4198400,
+    "num_episodes": 678,
+    "mean_reward": 279.6017733335495,
+    "mean_length": 8183.74,
+    "loss": 29.961881637573242,
+    "sps": 559.0450859302445
+  },
+  {
+    "update": 1030,
+    "global_step": 4218880,
+    "num_episodes": 680,
+    "mean_reward": 303.8355294418335,
+    "mean_length": 8282.54,
+    "loss": 0.7290852069854736,
+    "sps": 1641.6323017599234
+  },
+  {
+    "update": 1035,
+    "global_step": 4239360,
+    "num_episodes": 682,
+    "mean_reward": 309.25065141677857,
+    "mean_length": 8380.25,
+    "loss": 7.585202693939209,
+    "sps": 290.23415915491415
+  },
+  {
+    "update": 1040,
+    "global_step": 4259840,
+    "num_episodes": 685,
+    "mean_reward": 307.12660905838015,
+    "mean_length": 8380.25,
+    "loss": 6.446464538574219,
+    "sps": 383.71739315963373
+  },
+  {
+    "update": 1045,
+    "global_step": 4280320,
+    "num_episodes": 687,
+    "mean_reward": 303.14754570007324,
+    "mean_length": 8380.25,
+    "loss": 9.89684772491455,
+    "sps": 694.9770930488776
+  },
+  {
+    "update": 1050,
+    "global_step": 4300800,
+    "num_episodes": 688,
+    "mean_reward": 303.04791251659395,
+    "mean_length": 8462.37,
+    "loss": 1.8083292245864868,
+    "sps": 1040.0758682399094
+  },
+  {
+    "update": 1055,
+    "global_step": 4321280,
+    "num_episodes": 690,
+    "mean_reward": 303.53817603588107,
+    "mean_length": 8516.23,
+    "loss": 1.01134192943573,
+    "sps": 3137.679652522162
+  },
+  {
+    "update": 1060,
+    "global_step": 4341760,
+    "num_episodes": 693,
+    "mean_reward": 309.7838798046112,
+    "mean_length": 8562.16,
+    "loss": 14.91142749786377,
+    "sps": 214.3493374321697
+  },
+  {
+    "update": 1065,
+    "global_step": 4362240,
+    "num_episodes": 695,
+    "mean_reward": 305.9636838245392,
+    "mean_length": 8596.65,
+    "loss": 25.306856155395508,
+    "sps": 520.1481805059935
+  },
+  {
+    "update": 1070,
+    "global_step": 4382720,
+    "num_episodes": 698,
+    "mean_reward": 303.673222618103,
+    "mean_length": 8440.95,
+    "loss": 1.5045527219772339,
+    "sps": 1217.974691455878
+  },
+  {
+    "update": 1075,
+    "global_step": 4403200,
+    "num_episodes": 701,
+    "mean_reward": 304.443781414032,
+    "mean_length": 8557.61,
+    "loss": 3.09621524810791,
+    "sps": 3334.805184943337
+  },
+  {
+    "update": 1080,
+    "global_step": 4423680,
+    "num_episodes": 702,
+    "mean_reward": 303.1939596557617,
+    "mean_length": 8557.61,
+    "loss": 0.5530338883399963,
+    "sps": 3357.0436968093577
+  },
+  {
+    "update": 1085,
+    "global_step": 4444160,
+    "num_episodes": 706,
+    "mean_reward": 297.1481484127045,
+    "mean_length": 8521.02,
+    "loss": 7.937706470489502,
+    "sps": 1838.4046275928872
+  },
+  {
+    "update": 1090,
+    "global_step": 4464640,
+    "num_episodes": 709,
+    "mean_reward": 294.23652092933656,
+    "mean_length": 8421.99,
+    "loss": 1.2086868286132812,
+    "sps": 2518.910916673558
+  },
+  {
+    "update": 1095,
+    "global_step": 4485120,
+    "num_episodes": 713,
+    "mean_reward": 291.4626713657379,
+    "mean_length": 8418.82,
+    "loss": 0.07519354671239853,
+    "sps": 2391.4890715912597
+  },
+  {
+    "update": 1100,
+    "global_step": 4505600,
+    "num_episodes": 714,
+    "mean_reward": 295.5940579319,
+    "mean_length": 8418.82,
+    "loss": -0.04633036255836487,
+    "sps": 2309.370211128856
+  },
+  {
+    "update": 1105,
+    "global_step": 4526080,
+    "num_episodes": 717,
+    "mean_reward": 308.55377667427064,
+    "mean_length": 8615.8,
+    "loss": 6.805793762207031,
+    "sps": 1932.5391546325443
+  },
+  {
+    "update": 1110,
+    "global_step": 4546560,
+    "num_episodes": 718,
+    "mean_reward": 308.8043742084503,
+    "mean_length": 8615.8,
+    "loss": 1.5425922870635986,
+    "sps": 3048.7004653828785
+  },
+  {
+    "update": 1115,
+    "global_step": 4567040,
+    "num_episodes": 723,
+    "mean_reward": 303.7773459148407,
+    "mean_length": 8522.71,
+    "loss": 0.27959662675857544,
+    "sps": 2995.9554614172757
+  },
+  {
+    "update": 1120,
+    "global_step": 4587520,
+    "num_episodes": 724,
+    "mean_reward": 303.2645093822479,
+    "mean_length": 8522.71,
+    "loss": -0.0757850706577301,
+    "sps": 3148.672000633405
+  },
+  {
+    "update": 1125,
+    "global_step": 4608000,
+    "num_episodes": 728,
+    "mean_reward": 295.8857748699188,
+    "mean_length": 8522.71,
+    "loss": 0.8416875004768372,
+    "sps": 1310.0236701271228
+  },
+  {
+    "update": 1130,
+    "global_step": 4628480,
+    "num_episodes": 728,
+    "mean_reward": 295.8857748699188,
+    "mean_length": 8522.71,
+    "loss": 3.5039658546447754,
+    "sps": 2038.3808422065272
+  },
+  {
+    "update": 1135,
+    "global_step": 4648960,
+    "num_episodes": 733,
+    "mean_reward": 288.808697681427,
+    "mean_length": 8383.94,
+    "loss": 0.9907295107841492,
+    "sps": 432.1509684368163
+  },
+  {
+    "update": 1140,
+    "global_step": 4669440,
+    "num_episodes": 734,
+    "mean_reward": 288.7881833076477,
+    "mean_length": 8383.94,
+    "loss": 5.849100112915039,
+    "sps": 743.4081065939373
+  },
+  {
+    "update": 1145,
+    "global_step": 4689920,
+    "num_episodes": 742,
+    "mean_reward": 248.96954289913177,
+    "mean_length": 8035.29,
+    "loss": 5.8911590576171875,
+    "sps": 439.3842370241788
+  },
+  {
+    "update": 1150,
+    "global_step": 4710400,
+    "num_episodes": 743,
+    "mean_reward": 247.8361682987213,
+    "mean_length": 7942.3,
+    "loss": 2.5104424953460693,
+    "sps": 2091.825912976944
+  },
+  {
+    "update": 1155,
+    "global_step": 4730880,
+    "num_episodes": 746,
+    "mean_reward": 247.5757204723358,
+    "mean_length": 7843.08,
+    "loss": 0.02520643174648285,
+    "sps": 2215.1389764020146
+  },
+  {
+    "update": 1160,
+    "global_step": 4751360,
+    "num_episodes": 748,
+    "mean_reward": 237.4100481700897,
+    "mean_length": 7843.08,
+    "loss": 3.045536756515503,
+    "sps": 1354.2367583329549
+  },
+  {
+    "update": 1165,
+    "global_step": 4771840,
+    "num_episodes": 753,
+    "mean_reward": 254.61865887641906,
+    "mean_length": 7743.82,
+    "loss": 18.703535079956055,
+    "sps": 1982.4466952604278
+  },
+  {
+    "update": 1170,
+    "global_step": 4792320,
+    "num_episodes": 753,
+    "mean_reward": 254.61865887641906,
+    "mean_length": 7743.82,
+    "loss": 0.33046531677246094,
+    "sps": 2095.2503659093586
+  },
+  {
+    "update": 1175,
+    "global_step": 4812800,
+    "num_episodes": 755,
+    "mean_reward": 253.8280854511261,
+    "mean_length": 7787.79,
+    "loss": 3.0896127223968506,
+    "sps": 422.3812735467675
+  },
+  {
+    "update": 1180,
+    "global_step": 4833280,
+    "num_episodes": 758,
+    "mean_reward": 264.60756063461304,
+    "mean_length": 7709.59,
+    "loss": -0.09283498674631119,
+    "sps": 2676.2813899197204
+  },
+  {
+    "update": 1185,
+    "global_step": 4853760,
+    "num_episodes": 763,
+    "mean_reward": 239.50925126552582,
+    "mean_length": 7671.32,
+    "loss": 0.09459003806114197,
+    "sps": 2538.8112544970772
+  },
+  {
+    "update": 1190,
+    "global_step": 4874240,
+    "num_episodes": 763,
+    "mean_reward": 239.50925126552582,
+    "mean_length": 7671.32,
+    "loss": 0.17775702476501465,
+    "sps": 2975.422989387635
+  },
+  {
+    "update": 1195,
+    "global_step": 4894720,
+    "num_episodes": 767,
+    "mean_reward": 218.03613798141478,
+    "mean_length": 7614.22,
+    "loss": 3.9349420070648193,
+    "sps": 417.34421892596595
+  },
+  {
+    "update": 1200,
+    "global_step": 4915200,
+    "num_episodes": 768,
+    "mean_reward": 218.03613859176636,
+    "mean_length": 7614.22,
+    "loss": 4.028732776641846,
+    "sps": 784.4513250410687
+  },
+  {
+    "update": 1205,
+    "global_step": 4935680,
+    "num_episodes": 773,
+    "mean_reward": 208.71825884342195,
+    "mean_length": 7713.33,
+    "loss": 10.78580093383789,
+    "sps": 734.3705869944031
+  },
+  {
+    "update": 1210,
+    "global_step": 4956160,
+    "num_episodes": 774,
+    "mean_reward": 208.632105717659,
+    "mean_length": 7808.29,
+    "loss": 3.5958919525146484,
+    "sps": 1305.6310352836174
+  },
+  {
+    "update": 1215,
+    "global_step": 4976640,
+    "num_episodes": 777,
+    "mean_reward": 204.77496297359465,
+    "mean_length": 7805.2,
+    "loss": 3.0023584365844727,
+    "sps": 2482.0292324013326
+  },
+  {
+    "update": 1220,
+    "global_step": 4997120,
+    "num_episodes": 779,
+    "mean_reward": 196.00456192016603,
+    "mean_length": 7808.44,
+    "loss": 11.420112609863281,
+    "sps": 2348.8046891139184
+  },
+  {
+    "update": 1225,
+    "global_step": 5017600,
+    "num_episodes": 786,
+    "mean_reward": 184.82681788921357,
+    "mean_length": 7520.75,
+    "loss": 9.389796257019043,
+    "sps": 403.47715369548195
+  },
+  {
+    "update": 1230,
+    "global_step": 5038080,
+    "num_episodes": 789,
+    "mean_reward": 185.08837336063385,
+    "mean_length": 7350.11,
+    "loss": 1.2659046649932861,
+    "sps": 740.0947252642347
+  },
+  {
+    "update": 1235,
+    "global_step": 5058560,
+    "num_episodes": 793,
+    "mean_reward": 179.18952730178833,
+    "mean_length": 7160.46,
+    "loss": 0.042632922530174255,
+    "sps": 2798.3588240027393
+  },
+  {
+    "update": 1240,
+    "global_step": 5079040,
+    "num_episodes": 794,
+    "mean_reward": 184.8410611152649,
+    "mean_length": 7160.46,
+    "loss": 1.8178130388259888,
+    "sps": 2741.015198709958
+  },
+  {
+    "update": 1245,
+    "global_step": 5099520,
+    "num_episodes": 799,
+    "mean_reward": 189.7154534816742,
+    "mean_length": 7117.64,
+    "loss": 2.4596240520477295,
+    "sps": 2848.528616655762
+  },
+  {
+    "update": 1250,
+    "global_step": 5120000,
+    "num_episodes": 802,
+    "mean_reward": 191.06395349502563,
+    "mean_length": 7035.96,
+    "loss": 3.05761981010437,
+    "sps": 2957.866130229398
+  },
+  {
+    "update": 1255,
+    "global_step": 5140480,
+    "num_episodes": 807,
+    "mean_reward": 184.37625044822693,
+    "mean_length": 6937.2,
+    "loss": 8.337796211242676,
+    "sps": 797.4964725831112
+  },
+  {
+    "update": 1260,
+    "global_step": 5160960,
+    "num_episodes": 811,
+    "mean_reward": 183.53424713134766,
+    "mean_length": 6839.0,
+    "loss": 4.347324371337891,
+    "sps": 2885.534959076487
+  },
+  {
+    "update": 1265,
+    "global_step": 5181440,
+    "num_episodes": 814,
+    "mean_reward": 179.14370263576507,
+    "mean_length": 6841.38,
+    "loss": 0.9887397289276123,
+    "sps": 2726.0079616540743
+  },
+  {
+    "update": 1270,
+    "global_step": 5201920,
+    "num_episodes": 816,
+    "mean_reward": 182.02527619838713,
+    "mean_length": 6841.38,
+    "loss": 3.567499876022339,
+    "sps": 2548.7726014650657
+  },
+  {
+    "update": 1275,
+    "global_step": 5222400,
+    "num_episodes": 818,
+    "mean_reward": 167.24890702724457,
+    "mean_length": 6841.38,
+    "loss": 1.4451591968536377,
+    "sps": 312.6484055556043
+  },
+  {
+    "update": 1280,
+    "global_step": 5242880,
+    "num_episodes": 821,
+    "mean_reward": 165.29594656467438,
+    "mean_length": 6837.89,
+    "loss": 1.7843670845031738,
+    "sps": 570.2931373528285
+  },
+  {
+    "update": 1285,
+    "global_step": 5263360,
+    "num_episodes": 828,
+    "mean_reward": 152.66287196159362,
+    "mean_length": 6498.54,
+    "loss": 5.005251407623291,
+    "sps": 272.3373610296092
+  },
+  {
+    "update": 1290,
+    "global_step": 5283840,
+    "num_episodes": 830,
+    "mean_reward": 153.68320658683777,
+    "mean_length": 6498.54,
+    "loss": 8.443866729736328,
+    "sps": 292.14330020270444
+  },
+  {
+    "update": 1295,
+    "global_step": 5304320,
+    "num_episodes": 831,
+    "mean_reward": 157.06462443351745,
+    "mean_length": 6498.54,
+    "loss": 2.2017781734466553,
+    "sps": 265.6513386532661
+  },
+  {
+    "update": 1300,
+    "global_step": 5324800,
+    "num_episodes": 834,
+    "mean_reward": 158.8159614944458,
+    "mean_length": 6680.51,
+    "loss": 6.665990352630615,
+    "sps": 563.3260777169
+  },
+  {
+    "update": 1305,
+    "global_step": 5345280,
+    "num_episodes": 840,
+    "mean_reward": 145.94160705566406,
+    "mean_length": 6594.4,
+    "loss": 10.99843978881836,
+    "sps": 1955.0578156421066
+  },
+  {
+    "update": 1310,
+    "global_step": 5365760,
+    "num_episodes": 843,
+    "mean_reward": 145.8064595746994,
+    "mean_length": 6590.29,
+    "loss": 0.6882038116455078,
+    "sps": 382.4471874145512
+  },
+  {
+    "update": 1315,
+    "global_step": 5386240,
+    "num_episodes": 844,
+    "mean_reward": 146.0564516401291,
+    "mean_length": 6590.29,
+    "loss": 0.8968511819839478,
+    "sps": 688.8494388200007
+  },
+  {
+    "update": 1320,
+    "global_step": 5406720,
+    "num_episodes": 848,
+    "mean_reward": 150.32687950611114,
+    "mean_length": 6662.46,
+    "loss": 25.992216110229492,
+    "sps": 658.8033472373343
+  },
+  {
+    "update": 1325,
+    "global_step": 5427200,
+    "num_episodes": 851,
+    "mean_reward": 146.97661718845367,
+    "mean_length": 6761.72,
+    "loss": 3.56770658493042,
+    "sps": 985.132708019695
+  },
+  {
+    "update": 1330,
+    "global_step": 5447680,
+    "num_episodes": 852,
+    "mean_reward": 134.18026546001434,
+    "mean_length": 6664.57,
+    "loss": 15.875215530395508,
+    "sps": 1092.9486199541875
+  },
+  {
+    "update": 1335,
+    "global_step": 5468160,
+    "num_episodes": 853,
+    "mean_reward": 130.54885493755341,
+    "mean_length": 6664.57,
+    "loss": 6.959790229797363,
+    "sps": 289.8487086871761
+  },
+  {
+    "update": 1340,
+    "global_step": 5488640,
+    "num_episodes": 860,
+    "mean_reward": 130.95773406505586,
+    "mean_length": 6485.88,
+    "loss": 20.052452087402344,
+    "sps": 486.39910125898484
+  },
+  {
+    "update": 1345,
+    "global_step": 5509120,
+    "num_episodes": 863,
+    "mean_reward": 142.73314903259276,
+    "mean_length": 6488.99,
+    "loss": 10.171817779541016,
+    "sps": 201.9510308985079
+  },
+  {
+    "update": 1350,
+    "global_step": 5529600,
+    "num_episodes": 865,
+    "mean_reward": 140.62244102954864,
+    "mean_length": 6427.8,
+    "loss": 1.1496386528015137,
+    "sps": 635.0260794777045
+  },
+  {
+    "update": 1355,
+    "global_step": 5550080,
+    "num_episodes": 865,
+    "mean_reward": 140.62244102954864,
+    "mean_length": 6427.8,
+    "loss": 4.384088039398193,
+    "sps": 882.9404372993679
+  },
+  {
+    "update": 1360,
+    "global_step": 5570560,
+    "num_episodes": 872,
+    "mean_reward": 147.4697220182419,
+    "mean_length": 6527.99,
+    "loss": 10.650382041931152,
+    "sps": 280.68259822115476
+  },
+  {
+    "update": 1365,
+    "global_step": 5591040,
+    "num_episodes": 875,
+    "mean_reward": 145.2843037557602,
+    "mean_length": 6358.27,
+    "loss": 38.430118560791016,
+    "sps": 324.7223637427747
+  },
+  {
+    "update": 1370,
+    "global_step": 5611520,
+    "num_episodes": 876,
+    "mean_reward": 145.53429505825042,
+    "mean_length": 6358.27,
+    "loss": 38.706214904785156,
+    "sps": 538.1413797129117
+  },
+  {
+    "update": 1375,
+    "global_step": 5632000,
+    "num_episodes": 877,
+    "mean_reward": 145.43353649616242,
+    "mean_length": 6431.14,
+    "loss": 16.998456954956055,
+    "sps": 824.6403047601932
+  },
+  {
+    "update": 1380,
+    "global_step": 5652480,
+    "num_episodes": 881,
+    "mean_reward": 154.26934564590454,
+    "mean_length": 6526.9,
+    "loss": 3.5649774074554443,
+    "sps": 844.5595148121038
+  },
+  {
+    "update": 1385,
+    "global_step": 5672960,
+    "num_episodes": 884,
+    "mean_reward": 161.71352871894837,
+    "mean_length": 6550.38,
+    "loss": 3.710602045059204,
+    "sps": 908.754059815621
+  },
+  {
+    "update": 1390,
+    "global_step": 5693440,
+    "num_episodes": 885,
+    "mean_reward": 163.49456966400146,
+    "mean_length": 6649.33,
+    "loss": 0.4263116717338562,
+    "sps": 2189.736809475642
+  },
+  {
+    "update": 1395,
+    "global_step": 5713920,
+    "num_episodes": 886,
+    "mean_reward": 176.25792336940765,
+    "mean_length": 6738.86,
+    "loss": 0.27514657378196716,
+    "sps": 2997.673240659507
+  },
+  {
+    "update": 1400,
+    "global_step": 5734400,
+    "num_episodes": 889,
+    "mean_reward": 178.37687824249267,
+    "mean_length": 6926.31,
+    "loss": 0.08463311195373535,
+    "sps": 3081.918148797146
   }
 ]