Upload training_log.json with huggingface_hub

training_log.json

@@ -592,5 +592,3875 @@
     "mean_length": 3142.3,
     "loss": 1779.034423828125,
     "sps": 167.8109154720703
+  },
+  {
+    "update": 335,
+    "global_step": 1372160,
+    "num_episodes": 378,
+    "mean_reward": 357.36821466445923,
+    "mean_length": 2510.76,
+    "loss": 33.58185958862305,
+    "sps": 451.94218989282257
+  },
+  {
+    "update": 340,
+    "global_step": 1392640,
+    "num_episodes": 388,
+    "mean_reward": 455.7884948730469,
+    "mean_length": 2739.63,
+    "loss": 16.64685821533203,
+    "sps": 273.0263210661101
+  },
+  {
+    "update": 345,
+    "global_step": 1413120,
+    "num_episodes": 390,
+    "mean_reward": 456.36600038528445,
+    "mean_length": 2831.63,
+    "loss": 53.800376892089844,
+    "sps": 262.8572795282316
+  },
+  {
+    "update": 350,
+    "global_step": 1433600,
+    "num_episodes": 392,
+    "mean_reward": 358.3022264003754,
+    "mean_length": 2855.26,
+    "loss": 275.63385009765625,
+    "sps": 446.63750460322206
+  },
+  {
+    "update": 355,
+    "global_step": 1454080,
+    "num_episodes": 407,
+    "mean_reward": 375.7198329591751,
+    "mean_length": 2911.33,
+    "loss": 8.481382369995117,
+    "sps": 367.7815116282032
+  },
+  {
+    "update": 360,
+    "global_step": 1474560,
+    "num_episodes": 417,
+    "mean_reward": 297.66134167194366,
+    "mean_length": 2867.69,
+    "loss": 23.24544334411621,
+    "sps": 538.1701556626739
+  },
+  {
+    "update": 365,
+    "global_step": 1495040,
+    "num_episodes": 423,
+    "mean_reward": 297.34756595134735,
+    "mean_length": 2864.35,
+    "loss": 4.300067901611328,
+    "sps": 967.5617761077667
+  },
+  {
+    "update": 370,
+    "global_step": 1515520,
+    "num_episodes": 426,
+    "mean_reward": 420.4760777759552,
+    "mean_length": 2769.38,
+    "loss": 0.5866292715072632,
+    "sps": 666.1817947182782
+  },
+  {
+    "update": 375,
+    "global_step": 1536000,
+    "num_episodes": 429,
+    "mean_reward": 420.90687354564665,
+    "mean_length": 2720.24,
+    "loss": 1.5303698778152466,
+    "sps": 387.94660115994657
+  },
+  {
+    "update": 380,
+    "global_step": 1556480,
+    "num_episodes": 447,
+    "mean_reward": 356.93565448284147,
+    "mean_length": 2600.92,
+    "loss": 1.487197995185852,
+    "sps": 563.1387661188227
+  },
+  {
+    "update": 385,
+    "global_step": 1576960,
+    "num_episodes": 452,
+    "mean_reward": 357.39739652633665,
+    "mean_length": 2840.12,
+    "loss": 139.934326171875,
+    "sps": 419.07382914990956
+  },
+  {
+    "update": 390,
+    "global_step": 1597440,
+    "num_episodes": 456,
+    "mean_reward": 320.4454140949249,
+    "mean_length": 2739.7,
+    "loss": 35.25250244140625,
+    "sps": 1220.6982535284524
+  },
+  {
+    "update": 395,
+    "global_step": 1617920,
+    "num_episodes": 456,
+    "mean_reward": 320.4454140949249,
+    "mean_length": 2739.7,
+    "loss": 3.3828349113464355,
+    "sps": 1342.5750178490377
+  },
+  {
+    "update": 400,
+    "global_step": 1638400,
+    "num_episodes": 464,
+    "mean_reward": 328.27054366588595,
+    "mean_length": 2746.2,
+    "loss": 9.8130521774292,
+    "sps": 1796.7926258567122
+  },
+  {
+    "update": 405,
+    "global_step": 1658880,
+    "num_episodes": 470,
+    "mean_reward": 469.22999305725097,
+    "mean_length": 2992.85,
+    "loss": 1.3559972047805786,
+    "sps": 692.6458346671639
+  },
+  {
+    "update": 410,
+    "global_step": 1679360,
+    "num_episodes": 490,
+    "mean_reward": 372.7216357469559,
+    "mean_length": 2776.16,
+    "loss": 1.9204107522964478,
+    "sps": 654.3363015594651
+  },
+  {
+    "update": 415,
+    "global_step": 1699840,
+    "num_episodes": 498,
+    "mean_reward": 355.0806073760986,
+    "mean_length": 2364.43,
+    "loss": 0.20851367712020874,
+    "sps": 1574.1015970289384
+  },
+  {
+    "update": 420,
+    "global_step": 1720320,
+    "num_episodes": 505,
+    "mean_reward": 357.1250058794022,
+    "mean_length": 2659.22,
+    "loss": 0.8760831356048584,
+    "sps": 725.5072374754453
+  },
+  {
+    "update": 425,
+    "global_step": 1740800,
+    "num_episodes": 519,
+    "mean_reward": 338.12661921024323,
+    "mean_length": 2572.09,
+    "loss": 0.7201396822929382,
+    "sps": 774.136034580983
+  },
+  {
+    "update": 430,
+    "global_step": 1761280,
+    "num_episodes": 520,
+    "mean_reward": 338.7818141031265,
+    "mean_length": 2670.69,
+    "loss": 0.2984998822212219,
+    "sps": 593.9586130082629
+  },
+  {
+    "update": 435,
+    "global_step": 1781760,
+    "num_episodes": 524,
+    "mean_reward": 200.51355738162994,
+    "mean_length": 2677.33,
+    "loss": 0.6558288931846619,
+    "sps": 563.7237889903446
+  },
+  {
+    "update": 440,
+    "global_step": 1802240,
+    "num_episodes": 531,
+    "mean_reward": 195.75862418174745,
+    "mean_length": 2684.96,
+    "loss": 0.6485836505889893,
+    "sps": 801.5054039510692
+  },
+  {
+    "update": 445,
+    "global_step": 1822720,
+    "num_episodes": 541,
+    "mean_reward": 182.37921036720275,
+    "mean_length": 2787.68,
+    "loss": 0.9738303422927856,
+    "sps": 552.0177086572941
+  },
+  {
+    "update": 450,
+    "global_step": 1843200,
+    "num_episodes": 544,
+    "mean_reward": 182.38307575702666,
+    "mean_length": 2886.96,
+    "loss": 0.35677605867385864,
+    "sps": 1078.6474451651457
+  },
+  {
+    "update": 455,
+    "global_step": 1863680,
+    "num_episodes": 545,
+    "mean_reward": 183.31554008960723,
+    "mean_length": 2983.98,
+    "loss": 2.5785653591156006,
+    "sps": 2238.2674979327135
+  },
+  {
+    "update": 460,
+    "global_step": 1884160,
+    "num_episodes": 552,
+    "mean_reward": 184.02062775611878,
+    "mean_length": 2848.2,
+    "loss": 2.8595657348632812,
+    "sps": 788.1771239252804
+  },
+  {
+    "update": 465,
+    "global_step": 1904640,
+    "num_episodes": 556,
+    "mean_reward": 185.80799278259278,
+    "mean_length": 3043.7,
+    "loss": 1.9609827995300293,
+    "sps": 577.1914698783215
+  },
+  {
+    "update": 470,
+    "global_step": 1925120,
+    "num_episodes": 562,
+    "mean_reward": 187.01453431129457,
+    "mean_length": 3143.24,
+    "loss": 3.339183807373047,
+    "sps": 1007.594504286708
+  },
+  {
+    "update": 475,
+    "global_step": 1945600,
+    "num_episodes": 566,
+    "mean_reward": 173.54088757038116,
+    "mean_length": 2963.92,
+    "loss": 0.6618616580963135,
+    "sps": 901.0594653949358
+  },
+  {
+    "update": 480,
+    "global_step": 1966080,
+    "num_episodes": 575,
+    "mean_reward": 33.9250515460968,
+    "mean_length": 2871.77,
+    "loss": 2.1673176288604736,
+    "sps": 637.9211744730668
+  },
+  {
+    "update": 485,
+    "global_step": 1986560,
+    "num_episodes": 586,
+    "mean_reward": 34.479522528648374,
+    "mean_length": 3064.33,
+    "loss": 0.5433505773544312,
+    "sps": 434.7028394813384
+  },
+  {
+    "update": 490,
+    "global_step": 2007040,
+    "num_episodes": 596,
+    "mean_reward": 35.79800989627838,
+    "mean_length": 3282.82,
+    "loss": 0.78948974609375,
+    "sps": 336.9612684589691
+  },
+  {
+    "update": 495,
+    "global_step": 2027520,
+    "num_episodes": 604,
+    "mean_reward": 33.232103657722476,
+    "mean_length": 2993.3,
+    "loss": 1.0133846998214722,
+    "sps": 825.0153267212829
+  },
+  {
+    "update": 500,
+    "global_step": 2048000,
+    "num_episodes": 608,
+    "mean_reward": 35.19134169101715,
+    "mean_length": 3091.99,
+    "loss": 2.315326690673828,
+    "sps": 1734.6708087716852
+  },
+  {
+    "update": 505,
+    "global_step": 2068480,
+    "num_episodes": 626,
+    "mean_reward": 34.19606147289276,
+    "mean_length": 2992.5,
+    "loss": 0.5885242223739624,
+    "sps": 326.9154194294925
+  },
+  {
+    "update": 510,
+    "global_step": 2088960,
+    "num_episodes": 635,
+    "mean_reward": 34.653258776664735,
+    "mean_length": 2903.6,
+    "loss": 0.5012519359588623,
+    "sps": 618.1195825840496
+  },
+  {
+    "update": 515,
+    "global_step": 2109440,
+    "num_episodes": 648,
+    "mean_reward": 31.289368829727174,
+    "mean_length": 2418.77,
+    "loss": 3.98177433013916,
+    "sps": 646.2346813472644
+  },
+  {
+    "update": 520,
+    "global_step": 2129920,
+    "num_episodes": 655,
+    "mean_reward": 31.192284369468688,
+    "mean_length": 2226.18,
+    "loss": 0.6265271306037903,
+    "sps": 1099.3023206141552
+  },
+  {
+    "update": 525,
+    "global_step": 2150400,
+    "num_episodes": 660,
+    "mean_reward": 31.34406463623047,
+    "mean_length": 2219.04,
+    "loss": 0.10834737122058868,
+    "sps": 1013.9364910908357
+  },
+  {
+    "update": 530,
+    "global_step": 2170880,
+    "num_episodes": 663,
+    "mean_reward": 32.43585729598999,
+    "mean_length": 2310.25,
+    "loss": 0.16828738152980804,
+    "sps": 2044.258556649656
+  },
+  {
+    "update": 535,
+    "global_step": 2191360,
+    "num_episodes": 673,
+    "mean_reward": 31.12276816368103,
+    "mean_length": 2188.44,
+    "loss": 0.30583497881889343,
+    "sps": 669.970940042682
+  },
+  {
+    "update": 540,
+    "global_step": 2211840,
+    "num_episodes": 688,
+    "mean_reward": 31.249693727493288,
+    "mean_length": 2081.05,
+    "loss": 0.17176635563373566,
+    "sps": 590.5779813457752
+  },
+  {
+    "update": 545,
+    "global_step": 2232320,
+    "num_episodes": 694,
+    "mean_reward": 33.27913472175598,
+    "mean_length": 2258.75,
+    "loss": 0.7694998979568481,
+    "sps": 738.4276136206116
+  },
+  {
+    "update": 550,
+    "global_step": 2252800,
+    "num_episodes": 698,
+    "mean_reward": 33.9422331905365,
+    "mean_length": 2351.64,
+    "loss": 0.15182125568389893,
+    "sps": 762.723220730318
+  },
+  {
+    "update": 555,
+    "global_step": 2273280,
+    "num_episodes": 700,
+    "mean_reward": 34.04119193077087,
+    "mean_length": 2350.76,
+    "loss": 0.032472074031829834,
+    "sps": 1245.5515732240913
+  },
+  {
+    "update": 560,
+    "global_step": 2293760,
+    "num_episodes": 712,
+    "mean_reward": 32.31116131782532,
+    "mean_length": 2355.04,
+    "loss": 0.5084341168403625,
+    "sps": 356.9310128125497
+  },
+  {
+    "update": 565,
+    "global_step": 2314240,
+    "num_episodes": 719,
+    "mean_reward": 32.47976410865784,
+    "mean_length": 2441.54,
+    "loss": 0.21268926560878754,
+    "sps": 649.2244348171059
+  },
+  {
+    "update": 570,
+    "global_step": 2334720,
+    "num_episodes": 728,
+    "mean_reward": 33.27659282684326,
+    "mean_length": 2530.94,
+    "loss": 0.9577451348304749,
+    "sps": 768.1072804561215
+  },
+  {
+    "update": 575,
+    "global_step": 2355200,
+    "num_episodes": 734,
+    "mean_reward": 31.69405174255371,
+    "mean_length": 2412.76,
+    "loss": 1.0152896642684937,
+    "sps": 1202.6180581266553
+  },
+  {
+    "update": 580,
+    "global_step": 2375680,
+    "num_episodes": 751,
+    "mean_reward": 32.80648866415024,
+    "mean_length": 2678.86,
+    "loss": 1.551302433013916,
+    "sps": 620.7046830826843
+  },
+  {
+    "update": 585,
+    "global_step": 2396160,
+    "num_episodes": 757,
+    "mean_reward": 29.564244215488433,
+    "mean_length": 2585.36,
+    "loss": 3.1353652477264404,
+    "sps": 488.49247923421996
+  },
+  {
+    "update": 590,
+    "global_step": 2416640,
+    "num_episodes": 758,
+    "mean_reward": 30.12968365430832,
+    "mean_length": 2685.11,
+    "loss": 1.9931682348251343,
+    "sps": 1344.82690419507
+  },
+  {
+    "update": 595,
+    "global_step": 2437120,
+    "num_episodes": 765,
+    "mean_reward": 28.753595340251923,
+    "mean_length": 2505.15,
+    "loss": 1.9217519760131836,
+    "sps": 447.9409443511792
+  },
+  {
+    "update": 600,
+    "global_step": 2457600,
+    "num_episodes": 777,
+    "mean_reward": 30.216613895893097,
+    "mean_length": 2711.46,
+    "loss": 18.632978439331055,
+    "sps": 404.03595919196334
+  },
+  {
+    "update": 605,
+    "global_step": 2478080,
+    "num_episodes": 795,
+    "mean_reward": 25.763007862567903,
+    "mean_length": 2472.96,
+    "loss": 0.08484360575675964,
+    "sps": 273.80019869630905
+  },
+  {
+    "update": 610,
+    "global_step": 2498560,
+    "num_episodes": 800,
+    "mean_reward": 25.233678991794587,
+    "mean_length": 2472.55,
+    "loss": 1.4983996152877808,
+    "sps": 842.5169450671826
+  },
+  {
+    "update": 615,
+    "global_step": 2519040,
+    "num_episodes": 800,
+    "mean_reward": 25.233678991794587,
+    "mean_length": 2472.55,
+    "loss": 2.3043322563171387,
+    "sps": 1633.9246020797414
+  },
+  {
+    "update": 620,
+    "global_step": 2539520,
+    "num_episodes": 810,
+    "mean_reward": 25.826587975025177,
+    "mean_length": 2510.48,
+    "loss": 0.21852731704711914,
+    "sps": 518.4307357532686
+  },
+  {
+    "update": 625,
+    "global_step": 2560000,
+    "num_episodes": 814,
+    "mean_reward": 25.645928237438202,
+    "mean_length": 2512.63,
+    "loss": 1.0583765506744385,
+    "sps": 581.0047205132955
+  },
+  {
+    "update": 630,
+    "global_step": 2580480,
+    "num_episodes": 823,
+    "mean_reward": 25.49915248632431,
+    "mean_length": 2527.36,
+    "loss": 1.795029878616333,
+    "sps": 537.504310438823
+  },
+  {
+    "update": 635,
+    "global_step": 2600960,
+    "num_episodes": 825,
+    "mean_reward": 26.937079560756683,
+    "mean_length": 2620.74,
+    "loss": 0.2923164963722229,
+    "sps": 1350.1934205312384
+  },
+  {
+    "update": 640,
+    "global_step": 2621440,
+    "num_episodes": 834,
+    "mean_reward": 29.924575612545013,
+    "mean_length": 2905.58,
+    "loss": 0.43789708614349365,
+    "sps": 280.4967413720513
+  },
+  {
+    "update": 645,
+    "global_step": 2641920,
+    "num_episodes": 836,
+    "mean_reward": 28.948164422512054,
+    "mean_length": 2820.95,
+    "loss": 1.5149576663970947,
+    "sps": 640.5808040483822
+  },
+  {
+    "update": 650,
+    "global_step": 2662400,
+    "num_episodes": 843,
+    "mean_reward": 35.76819623708725,
+    "mean_length": 3020.26,
+    "loss": 1.5319448709487915,
+    "sps": 573.556787061315
+  },
+  {
+    "update": 655,
+    "global_step": 2682880,
+    "num_episodes": 851,
+    "mean_reward": 35.12200962543488,
+    "mean_length": 2929.69,
+    "loss": 0.4334838390350342,
+    "sps": 721.898659363473
+  },
+  {
+    "update": 660,
+    "global_step": 2703360,
+    "num_episodes": 863,
+    "mean_reward": 34.17141466140747,
+    "mean_length": 2837.57,
+    "loss": 15.400986671447754,
+    "sps": 531.226152703821
+  },
+  {
+    "update": 665,
+    "global_step": 2723840,
+    "num_episodes": 871,
+    "mean_reward": 34.406737427711484,
+    "mean_length": 2847.12,
+    "loss": -0.022274762392044067,
+    "sps": 1102.2418013565505
+  },
+  {
+    "update": 670,
+    "global_step": 2744320,
+    "num_episodes": 880,
+    "mean_reward": 33.42318947315216,
+    "mean_length": 2744.42,
+    "loss": 0.21278248727321625,
+    "sps": 932.5471353242045
+  },
+  {
+    "update": 675,
+    "global_step": 2764800,
+    "num_episodes": 881,
+    "mean_reward": 35.02722130298614,
+    "mean_length": 2793.45,
+    "loss": 0.48318442702293396,
+    "sps": 849.2902333548969
+  },
+  {
+    "update": 680,
+    "global_step": 2785280,
+    "num_episodes": 894,
+    "mean_reward": 42.5996656703949,
+    "mean_length": 2993.97,
+    "loss": 0.9185795187950134,
+    "sps": 476.82647813725623
+  },
+  {
+    "update": 685,
+    "global_step": 2805760,
+    "num_episodes": 914,
+    "mean_reward": 40.827002143859865,
+    "mean_length": 2561.69,
+    "loss": 1.2765486240386963,
+    "sps": 255.71578525021312
+  },
+  {
+    "update": 690,
+    "global_step": 2826240,
+    "num_episodes": 925,
+    "mean_reward": 45.62547200202942,
+    "mean_length": 2469.27,
+    "loss": 2.3010942935943604,
+    "sps": 570.2368795975491
+  },
+  {
+    "update": 695,
+    "global_step": 2846720,
+    "num_episodes": 931,
+    "mean_reward": 43.9872697687149,
+    "mean_length": 2287.32,
+    "loss": 1.1779371500015259,
+    "sps": 532.4176047731756
+  },
+  {
+    "update": 700,
+    "global_step": 2867200,
+    "num_episodes": 943,
+    "mean_reward": 36.38923056125641,
+    "mean_length": 1897.85,
+    "loss": 0.6398267149925232,
+    "sps": 501.06106420522286
+  },
+  {
+    "update": 705,
+    "global_step": 2887680,
+    "num_episodes": 952,
+    "mean_reward": 38.85928546905517,
+    "mean_length": 2189.91,
+    "loss": 0.7464686632156372,
+    "sps": 358.482733937848
+  },
+  {
+    "update": 710,
+    "global_step": 2908160,
+    "num_episodes": 960,
+    "mean_reward": 38.50984364032745,
+    "mean_length": 2181.81,
+    "loss": 0.819063663482666,
+    "sps": 729.0695475685854
+  },
+  {
+    "update": 715,
+    "global_step": 2928640,
+    "num_episodes": 960,
+    "mean_reward": 38.50984364032745,
+    "mean_length": 2181.81,
+    "loss": 1.3829450607299805,
+    "sps": 2103.44152639025
+  },
+  {
+    "update": 720,
+    "global_step": 2949120,
+    "num_episodes": 961,
+    "mean_reward": 38.30655426502228,
+    "mean_length": 2181.81,
+    "loss": 1.3242486715316772,
+    "sps": 2180.924028748301
+  },
+  {
+    "update": 725,
+    "global_step": 2969600,
+    "num_episodes": 979,
+    "mean_reward": 37.64626069068909,
+    "mean_length": 2258.86,
+    "loss": 0.6890235543251038,
+    "sps": 512.7806225711571
+  },
+  {
+    "update": 730,
+    "global_step": 2990080,
+    "num_episodes": 993,
+    "mean_reward": 29.446645002365113,
+    "mean_length": 2178.52,
+    "loss": 1.424858570098877,
+    "sps": 397.19464657996275
+  },
+  {
+    "update": 735,
+    "global_step": 3010560,
+    "num_episodes": 994,
+    "mean_reward": 29.469437551498412,
+    "mean_length": 2182.57,
+    "loss": 1.9823501110076904,
+    "sps": 742.698176618862
+  },
+  {
+    "update": 740,
+    "global_step": 3031040,
+    "num_episodes": 995,
+    "mean_reward": 29.75782982826233,
+    "mean_length": 2282.33,
+    "loss": 0.30816513299942017,
+    "sps": 1825.6853767545124
+  },
+  {
+    "update": 745,
+    "global_step": 3051520,
+    "num_episodes": 998,
+    "mean_reward": 30.453793020248412,
+    "mean_length": 2473.99,
+    "loss": 3.092510938644409,
+    "sps": 1555.5380368163708
+  },
+  {
+    "update": 750,
+    "global_step": 3072000,
+    "num_episodes": 1010,
+    "mean_reward": 31.862056040763854,
+    "mean_length": 2872.71,
+    "loss": 3.4220612049102783,
+    "sps": 266.50791667174065
+  },
+  {
+    "update": 755,
+    "global_step": 3092480,
+    "num_episodes": 1019,
+    "mean_reward": 28.9614812707901,
+    "mean_length": 2699.59,
+    "loss": 1.2745823860168457,
+    "sps": 753.5067688712668
+  },
+  {
+    "update": 760,
+    "global_step": 3112960,
+    "num_episodes": 1020,
+    "mean_reward": 23.99162916660309,
+    "mean_length": 2699.59,
+    "loss": 0.8926928043365479,
+    "sps": 1444.3258733361963
+  },
+  {
+    "update": 765,
+    "global_step": 3133440,
+    "num_episodes": 1024,
+    "mean_reward": 30.025504064559936,
+    "mean_length": 2889.38,
+    "loss": 1.112790822982788,
+    "sps": 756.883715254125
+  },
+  {
+    "update": 770,
+    "global_step": 3153920,
+    "num_episodes": 1034,
+    "mean_reward": 36.13991012096405,
+    "mean_length": 3187.15,
+    "loss": 0.43345746397972107,
+    "sps": 419.40821109391015
+  },
+  {
+    "update": 775,
+    "global_step": 3174400,
+    "num_episodes": 1050,
+    "mean_reward": 32.24160755157471,
+    "mean_length": 2985.91,
+    "loss": 0.13569332659244537,
+    "sps": 317.4046328348688
+  },
+  {
+    "update": 780,
+    "global_step": 3194880,
+    "num_episodes": 1061,
+    "mean_reward": 30.67223885536194,
+    "mean_length": 2711.19,
+    "loss": 0.2064545899629593,
+    "sps": 303.1813456850613
+  },
+  {
+    "update": 785,
+    "global_step": 3215360,
+    "num_episodes": 1072,
+    "mean_reward": 29.705427560806275,
+    "mean_length": 2524.86,
+    "loss": 4.488283634185791,
+    "sps": 913.592632749929
+  },
+  {
+    "update": 790,
+    "global_step": 3235840,
+    "num_episodes": 1085,
+    "mean_reward": 32.955479860305786,
+    "mean_length": 2618.62,
+    "loss": 0.05505555868148804,
+    "sps": 607.269974208857
+  },
+  {
+    "update": 795,
+    "global_step": 3256320,
+    "num_episodes": 1097,
+    "mean_reward": 31.909876976013184,
+    "mean_length": 2214.73,
+    "loss": 1.9556808471679688,
+    "sps": 747.2261381867389
+  },
+  {
+    "update": 800,
+    "global_step": 3276800,
+    "num_episodes": 1106,
+    "mean_reward": 29.67423951148987,
+    "mean_length": 1915.87,
+    "loss": 2.1206791400909424,
+    "sps": 701.3988516159834
+  },
+  {
+    "update": 805,
+    "global_step": 3297280,
+    "num_episodes": 1110,
+    "mean_reward": 30.086654043197633,
+    "mean_length": 2005.75,
+    "loss": 0.5833985805511475,
+    "sps": 660.5544733393301
+  },
+  {
+    "update": 810,
+    "global_step": 3317760,
+    "num_episodes": 1115,
+    "mean_reward": 32.238635430336,
+    "mean_length": 2171.27,
+    "loss": 0.4146590232849121,
+    "sps": 1094.4092641494085
+  },
+  {
+    "update": 815,
+    "global_step": 3338240,
+    "num_episodes": 1120,
+    "mean_reward": 33.369399318695066,
+    "mean_length": 2262.48,
+    "loss": 2.585177183151245,
+    "sps": 901.8559204378539
+  },
+  {
+    "update": 820,
+    "global_step": 3358720,
+    "num_episodes": 1130,
+    "mean_reward": 27.537890434265137,
+    "mean_length": 2050.2,
+    "loss": 0.5106171369552612,
+    "sps": 838.9275291477041
+  },
+  {
+    "update": 825,
+    "global_step": 3379200,
+    "num_episodes": 1136,
+    "mean_reward": 23.029513311386108,
+    "mean_length": 2161.34,
+    "loss": 4.071342468261719,
+    "sps": 1617.4028966230667
+  },
+  {
+    "update": 830,
+    "global_step": 3399680,
+    "num_episodes": 1138,
+    "mean_reward": 23.361328144073486,
+    "mean_length": 2259.74,
+    "loss": 0.30649739503860474,
+    "sps": 489.67958732294835
+  },
+  {
+    "update": 835,
+    "global_step": 3420160,
+    "num_episodes": 1146,
+    "mean_reward": 29.256564235687257,
+    "mean_length": 2379.17,
+    "loss": 0.2716136574745178,
+    "sps": 789.1092351378779
+  },
+  {
+    "update": 840,
+    "global_step": 3440640,
+    "num_episodes": 1152,
+    "mean_reward": 30.681161608695984,
+    "mean_length": 2564.53,
+    "loss": 3.510643720626831,
+    "sps": 465.91684356996774
+  },
+  {
+    "update": 845,
+    "global_step": 3461120,
+    "num_episodes": 1157,
+    "mean_reward": 31.814847540855407,
+    "mean_length": 2661.46,
+    "loss": 0.2368174046278,
+    "sps": 970.5356676833453
+  },
+  {
+    "update": 850,
+    "global_step": 3481600,
+    "num_episodes": 1158,
+    "mean_reward": 32.19375528335571,
+    "mean_length": 2760.37,
+    "loss": 1.0439393520355225,
+    "sps": 2118.0253570743016
+  },
+  {
+    "update": 855,
+    "global_step": 3502080,
+    "num_episodes": 1167,
+    "mean_reward": 32.78835594654083,
+    "mean_length": 2934.75,
+    "loss": 0.11038707196712494,
+    "sps": 373.62636209376416
+  },
+  {
+    "update": 860,
+    "global_step": 3522560,
+    "num_episodes": 1171,
+    "mean_reward": 38.54492960453033,
+    "mean_length": 3055.33,
+    "loss": 0.27012330293655396,
+    "sps": 811.5381951612115
+  },
+  {
+    "update": 865,
+    "global_step": 3543040,
+    "num_episodes": 1182,
+    "mean_reward": 38.79421305179596,
+    "mean_length": 3135.05,
+    "loss": 0.8694961071014404,
+    "sps": 643.9358039841488
+  },
+  {
+    "update": 870,
+    "global_step": 3563520,
+    "num_episodes": 1187,
+    "mean_reward": 37.14653766155243,
+    "mean_length": 3053.45,
+    "loss": 1.2393195629119873,
+    "sps": 871.9844580529647
+  },
+  {
+    "update": 875,
+    "global_step": 3584000,
+    "num_episodes": 1194,
+    "mean_reward": 40.78861089706421,
+    "mean_length": 3345.54,
+    "loss": 0.6096416115760803,
+    "sps": 304.93330977862365
+  },
+  {
+    "update": 880,
+    "global_step": 3604480,
+    "num_episodes": 1201,
+    "mean_reward": 39.81052246570587,
+    "mean_length": 3252.28,
+    "loss": 0.6608580946922302,
+    "sps": 677.495925390962
+  },
+  {
+    "update": 885,
+    "global_step": 3624960,
+    "num_episodes": 1215,
+    "mean_reward": 38.70529013156891,
+    "mean_length": 3165.08,
+    "loss": 1.2741384506225586,
+    "sps": 854.5639976310788
+  },
+  {
+    "update": 890,
+    "global_step": 3645440,
+    "num_episodes": 1215,
+    "mean_reward": 38.70529013156891,
+    "mean_length": 3165.08,
+    "loss": 0.5457536578178406,
+    "sps": 1511.837849491964
+  },
+  {
+    "update": 895,
+    "global_step": 3665920,
+    "num_episodes": 1220,
+    "mean_reward": 39.244035544395445,
+    "mean_length": 3264.42,
+    "loss": 0.6924870014190674,
+    "sps": 556.2715786447075
+  },
+  {
+    "update": 900,
+    "global_step": 3686400,
+    "num_episodes": 1232,
+    "mean_reward": 46.50887234210968,
+    "mean_length": 3173.4,
+    "loss": 1.7522594928741455,
+    "sps": 699.9425408800316
+  },
+  {
+    "update": 905,
+    "global_step": 3706880,
+    "num_episodes": 1245,
+    "mean_reward": 41.68342576980591,
+    "mean_length": 2950.87,
+    "loss": 1.1043925285339355,
+    "sps": 375.42107758469984
+  },
+  {
+    "update": 910,
+    "global_step": 3727360,
+    "num_episodes": 1257,
+    "mean_reward": 38.028645734786984,
+    "mean_length": 2550.61,
+    "loss": 0.11873626708984375,
+    "sps": 856.8550488972196
+  },
+  {
+    "update": 915,
+    "global_step": 3747840,
+    "num_episodes": 1265,
+    "mean_reward": 38.954245281219485,
+    "mean_length": 2549.56,
+    "loss": 0.31198206543922424,
+    "sps": 690.3971885269259
+  },
+  {
+    "update": 920,
+    "global_step": 3768320,
+    "num_episodes": 1269,
+    "mean_reward": 34.42387727260589,
+    "mean_length": 2518.34,
+    "loss": 1.7575799226760864,
+    "sps": 1152.958968508612
+  },
+  {
+    "update": 925,
+    "global_step": 3788800,
+    "num_episodes": 1274,
+    "mean_reward": 34.58799042224884,
+    "mean_length": 2520.98,
+    "loss": 0.3646126985549927,
+    "sps": 1100.5795244909316
+  },
+  {
+    "update": 930,
+    "global_step": 3809280,
+    "num_episodes": 1276,
+    "mean_reward": 36.83985797405243,
+    "mean_length": 2620.63,
+    "loss": 0.5593718886375427,
+    "sps": 999.9135216052517
+  },
+  {
+    "update": 935,
+    "global_step": 3829760,
+    "num_episodes": 1292,
+    "mean_reward": 36.57479173660278,
+    "mean_length": 2520.6,
+    "loss": 0.009059503674507141,
+    "sps": 1097.323441932561
+  },
+  {
+    "update": 940,
+    "global_step": 3850240,
+    "num_episodes": 1297,
+    "mean_reward": 37.501826615333556,
+    "mean_length": 2614.74,
+    "loss": 0.686941385269165,
+    "sps": 1048.1224362748299
+  },
+  {
+    "update": 945,
+    "global_step": 3870720,
+    "num_episodes": 1302,
+    "mean_reward": 38.2632510137558,
+    "mean_length": 2706.87,
+    "loss": 1.56045663356781,
+    "sps": 984.6772159375175
+  },
+  {
+    "update": 950,
+    "global_step": 3891200,
+    "num_episodes": 1303,
+    "mean_reward": 37.39876995563507,
+    "mean_length": 2611.05,
+    "loss": 1.817370891571045,
+    "sps": 662.4908033005096
+  },
+  {
+    "update": 955,
+    "global_step": 3911680,
+    "num_episodes": 1312,
+    "mean_reward": 38.347609777450565,
+    "mean_length": 2799.94,
+    "loss": 1.7936173677444458,
+    "sps": 498.5900701113524
+  },
+  {
+    "update": 960,
+    "global_step": 3932160,
+    "num_episodes": 1319,
+    "mean_reward": 38.531728854179384,
+    "mean_length": 2800.38,
+    "loss": 0.8617715835571289,
+    "sps": 288.6257663929259
+  },
+  {
+    "update": 965,
+    "global_step": 3952640,
+    "num_episodes": 1325,
+    "mean_reward": 32.38980568408966,
+    "mean_length": 2706.94,
+    "loss": 0.410921573638916,
+    "sps": 729.5250188646075
+  },
+  {
+    "update": 970,
+    "global_step": 3973120,
+    "num_episodes": 1330,
+    "mean_reward": 33.02072194576263,
+    "mean_length": 2800.87,
+    "loss": 0.6215076446533203,
+    "sps": 913.5447323136685
+  },
+  {
+    "update": 975,
+    "global_step": 3993600,
+    "num_episodes": 1333,
+    "mean_reward": 34.48237750530243,
+    "mean_length": 2899.46,
+    "loss": 1.4754104614257812,
+    "sps": 857.6512350171981
+  },
+  {
+    "update": 980,
+    "global_step": 4014080,
+    "num_episodes": 1346,
+    "mean_reward": 35.15436544418335,
+    "mean_length": 3006.69,
+    "loss": 1.3220990896224976,
+    "sps": 275.0666378661058
+  },
+  {
+    "update": 985,
+    "global_step": 4034560,
+    "num_episodes": 1359,
+    "mean_reward": 36.966177105903625,
+    "mean_length": 3140.04,
+    "loss": 0.31860384345054626,
+    "sps": 723.735690218882
+  },
+  {
+    "update": 990,
+    "global_step": 4055040,
+    "num_episodes": 1368,
+    "mean_reward": 36.728344497680666,
+    "mean_length": 2959.73,
+    "loss": 5.1887078285217285,
+    "sps": 773.332744431613
+  },
+  {
+    "update": 995,
+    "global_step": 4075520,
+    "num_episodes": 1369,
+    "mean_reward": 35.78023895263672,
+    "mean_length": 2959.73,
+    "loss": 1.6374891996383667,
+    "sps": 2350.737845032927
+  },
+  {
+    "update": 1000,
+    "global_step": 4096000,
+    "num_episodes": 1370,
+    "mean_reward": 35.719558029174806,
+    "mean_length": 2959.73,
+    "loss": 1.2606910467147827,
+    "sps": 2287.2771680470537
+  },
+  {
+    "update": 1005,
+    "global_step": 4116480,
+    "num_episodes": 1381,
+    "mean_reward": 37.67828846931457,
+    "mean_length": 2963.83,
+    "loss": 0.20636317133903503,
+    "sps": 1606.0294791853535
+  },
+  {
+    "update": 1010,
+    "global_step": 4136960,
+    "num_episodes": 1391,
+    "mean_reward": 39.33717452049255,
+    "mean_length": 3249.05,
+    "loss": 3.9951727390289307,
+    "sps": 418.63976772772594
+  },
+  {
+    "update": 1015,
+    "global_step": 4157440,
+    "num_episodes": 1399,
+    "mean_reward": 36.405463542938236,
+    "mean_length": 2962.77,
+    "loss": 1.837266445159912,
+    "sps": 683.5481201646958
+  },
+  {
+    "update": 1020,
+    "global_step": 4177920,
+    "num_episodes": 1401,
+    "mean_reward": 35.86304131507873,
+    "mean_length": 2879.03,
+    "loss": 2.9102792739868164,
+    "sps": 1089.8691629385837
+  },
+  {
+    "update": 1025,
+    "global_step": 4198400,
+    "num_episodes": 1416,
+    "mean_reward": 34.70472603797913,
+    "mean_length": 2801.68,
+    "loss": 0.9925887584686279,
+    "sps": 255.81352094643265
+  },
+  {
+    "update": 1030,
+    "global_step": 4218880,
+    "num_episodes": 1425,
+    "mean_reward": 33.617551488876344,
+    "mean_length": 2725.68,
+    "loss": 1.9214203357696533,
+    "sps": 737.2169737476238
+  },
+  {
+    "update": 1035,
+    "global_step": 4239360,
+    "num_episodes": 1434,
+    "mean_reward": 34.879857649803164,
+    "mean_length": 2533.35,
+    "loss": 0.2052566111087799,
+    "sps": 579.7787814604097
+  },
+  {
+    "update": 1040,
+    "global_step": 4259840,
+    "num_episodes": 1434,
+    "mean_reward": 34.879857649803164,
+    "mean_length": 2533.35,
+    "loss": 1.6612746715545654,
+    "sps": 1248.33251762393
+  },
+  {
+    "update": 1045,
+    "global_step": 4280320,
+    "num_episodes": 1445,
+    "mean_reward": 39.72180326938629,
+    "mean_length": 2610.11,
+    "loss": 4.8466691970825195,
+    "sps": 656.9837380754918
+  },
+  {
+    "update": 1050,
+    "global_step": 4300800,
+    "num_episodes": 1450,
+    "mean_reward": 42.0785810136795,
+    "mean_length": 2811.63,
+    "loss": 2.377190351486206,
+    "sps": 683.0332370956824
+  },
+  {
+    "update": 1055,
+    "global_step": 4321280,
+    "num_episodes": 1461,
+    "mean_reward": 39.593364839553836,
+    "mean_length": 2808.07,
+    "loss": 0.6159077882766724,
+    "sps": 526.5738589213908
+  },
+  {
+    "update": 1060,
+    "global_step": 4341760,
+    "num_episodes": 1461,
+    "mean_reward": 39.593364839553836,
+    "mean_length": 2808.07,
+    "loss": 0.8465878963470459,
+    "sps": 2184.9711988588206
+  },
+  {
+    "update": 1065,
+    "global_step": 4362240,
+    "num_episodes": 1468,
+    "mean_reward": 41.406082754135134,
+    "mean_length": 2993.5,
+    "loss": 2.0758755207061768,
+    "sps": 756.2821057624998
+  },
+  {
+    "update": 1070,
+    "global_step": 4382720,
+    "num_episodes": 1475,
+    "mean_reward": 35.40320841789246,
+    "mean_length": 2801.81,
+    "loss": 9.254124641418457,
+    "sps": 1308.8266389540704
+  },
+  {
+    "update": 1075,
+    "global_step": 4403200,
+    "num_episodes": 1485,
+    "mean_reward": 40.63698256492615,
+    "mean_length": 2897.46,
+    "loss": 0.5495067834854126,
+    "sps": 799.4087672915922
+  },
+  {
+    "update": 1080,
+    "global_step": 4423680,
+    "num_episodes": 1485,
+    "mean_reward": 40.63698256492615,
+    "mean_length": 2897.46,
+    "loss": 0.5127925872802734,
+    "sps": 1340.6154629856217
+  },
+  {
+    "update": 1085,
+    "global_step": 4444160,
+    "num_episodes": 1486,
+    "mean_reward": 41.25436541080475,
+    "mean_length": 2996.57,
+    "loss": 0.2694500684738159,
+    "sps": 1837.4922707085245
+  },
+  {
+    "update": 1090,
+    "global_step": 4464640,
+    "num_episodes": 1495,
+    "mean_reward": 46.52874216079712,
+    "mean_length": 3122.1,
+    "loss": 10.304019927978516,
+    "sps": 1046.1438942302716
+  },
+  {
+    "update": 1095,
+    "global_step": 4485120,
+    "num_episodes": 1509,
+    "mean_reward": 47.41412097454071,
+    "mean_length": 3088.91,
+    "loss": 4.403233051300049,
+    "sps": 705.1397561929946
+  },
+  {
+    "update": 1100,
+    "global_step": 4505600,
+    "num_episodes": 1509,
+    "mean_reward": 47.41412097454071,
+    "mean_length": 3088.91,
+    "loss": 5.204225540161133,
+    "sps": 956.5984516573334
+  },
+  {
+    "update": 1105,
+    "global_step": 4526080,
+    "num_episodes": 1514,
+    "mean_reward": 46.740014991760255,
+    "mean_length": 3048.45,
+    "loss": 1.3485753536224365,
+    "sps": 2885.439969795025
+  },
+  {
+    "update": 1110,
+    "global_step": 4546560,
+    "num_episodes": 1520,
+    "mean_reward": 55.2595415019989,
+    "mean_length": 3313.8,
+    "loss": 2.248853921890259,
+    "sps": 307.51773062645026
+  },
+  {
+    "update": 1115,
+    "global_step": 4567040,
+    "num_episodes": 1535,
+    "mean_reward": 45.75344041824341,
+    "mean_length": 3241.81,
+    "loss": 1.3050477504730225,
+    "sps": 349.05706197896666
+  },
+  {
+    "update": 1120,
+    "global_step": 4587520,
+    "num_episodes": 1538,
+    "mean_reward": 48.896285104751584,
+    "mean_length": 3154.78,
+    "loss": 1.0057706832885742,
+    "sps": 741.8276433683515
+  },
+  {
+    "update": 1125,
+    "global_step": 4608000,
+    "num_episodes": 1543,
+    "mean_reward": 49.45138314723968,
+    "mean_length": 3312.76,
+    "loss": 3.0420098304748535,
+    "sps": 267.5648126417643
+  },
+  {
+    "update": 1130,
+    "global_step": 4628480,
+    "num_episodes": 1554,
+    "mean_reward": 45.64340697288513,
+    "mean_length": 2967.91,
+    "loss": 0.534530758857727,
+    "sps": 595.2909007117904
+  },
+  {
+    "update": 1135,
+    "global_step": 4648960,
+    "num_episodes": 1566,
+    "mean_reward": 46.33692901611328,
+    "mean_length": 3175.93,
+    "loss": 12.374032974243164,
+    "sps": 273.2022330683734
+  },
+  {
+    "update": 1140,
+    "global_step": 4669440,
+    "num_episodes": 1574,
+    "mean_reward": 45.48817714929581,
+    "mean_length": 3007.35,
+    "loss": 9.789308547973633,
+    "sps": 468.42362264683504
+  },
+  {
+    "update": 1145,
+    "global_step": 4689920,
+    "num_episodes": 1579,
+    "mean_reward": 39.85454882383347,
+    "mean_length": 2862.76,
+    "loss": 14.043472290039062,
+    "sps": 685.1921972366682
+  },
+  {
+    "update": 1150,
+    "global_step": 4710400,
+    "num_episodes": 1592,
+    "mean_reward": 33.700903475284576,
+    "mean_length": 2480.53,
+    "loss": 8.177810668945312,
+    "sps": 333.4487379606718
+  },
+  {
+    "update": 1155,
+    "global_step": 4730880,
+    "num_episodes": 1607,
+    "mean_reward": 32.822717969417575,
+    "mean_length": 2485.88,
+    "loss": 58.6836051940918,
+    "sps": 320.2091534142309
+  },
+  {
+    "update": 1160,
+    "global_step": 4751360,
+    "num_episodes": 1613,
+    "mean_reward": 32.850986263751984,
+    "mean_length": 2471.11,
+    "loss": 4.779659748077393,
+    "sps": 886.848895232356
+  },
+  {
+    "update": 1165,
+    "global_step": 4771840,
+    "num_episodes": 1616,
+    "mean_reward": 30.757194340229034,
+    "mean_length": 2379.02,
+    "loss": 0.5000092387199402,
+    "sps": 1201.649782869044
+  },
+  {
+    "update": 1170,
+    "global_step": 4792320,
+    "num_episodes": 1618,
+    "mean_reward": 31.608184831142424,
+    "mean_length": 2481.05,
+    "loss": 0.5737665891647339,
+    "sps": 999.5698665674614
+  },
+  {
+    "update": 1175,
+    "global_step": 4812800,
+    "num_episodes": 1623,
+    "mean_reward": 32.650640833377835,
+    "mean_length": 2658.87,
+    "loss": 3.0942459106445312,
+    "sps": 616.1316245054412
+  },
+  {
+    "update": 1180,
+    "global_step": 4833280,
+    "num_episodes": 1625,
+    "mean_reward": 37.96194513559342,
+    "mean_length": 2682.02,
+    "loss": 9.840081214904785,
+    "sps": 698.9043221988104
+  },
+  {
+    "update": 1185,
+    "global_step": 4853760,
+    "num_episodes": 1630,
+    "mean_reward": 42.92790410280227,
+    "mean_length": 2817.92,
+    "loss": 1.0771808624267578,
+    "sps": 677.5959966787289
+  },
+  {
+    "update": 1190,
+    "global_step": 4874240,
+    "num_episodes": 1631,
+    "mean_reward": 43.29853859186173,
+    "mean_length": 2913.27,
+    "loss": 0.15924929082393646,
+    "sps": 2128.8886043740195
+  },
+  {
+    "update": 1195,
+    "global_step": 4894720,
+    "num_episodes": 1635,
+    "mean_reward": 44.563561956882474,
+    "mean_length": 3101.73,
+    "loss": 24.4128475189209,
+    "sps": 768.3773024267982
+  },
+  {
+    "update": 1200,
+    "global_step": 4915200,
+    "num_episodes": 1642,
+    "mean_reward": 45.34813198566437,
+    "mean_length": 3112.72,
+    "loss": 8.374368667602539,
+    "sps": 976.3571543939305
+  },
+  {
+    "update": 1205,
+    "global_step": 4935680,
+    "num_episodes": 1649,
+    "mean_reward": 46.91295046806336,
+    "mean_length": 3360.16,
+    "loss": 13.855979919433594,
+    "sps": 264.6689562194928
+  },
+  {
+    "update": 1210,
+    "global_step": 4956160,
+    "num_episodes": 1661,
+    "mean_reward": 46.23843379974365,
+    "mean_length": 3281.89,
+    "loss": 95.5245590209961,
+    "sps": 248.04865845009095
+  },
+  {
+    "update": 1215,
+    "global_step": 4976640,
+    "num_episodes": 1669,
+    "mean_reward": 49.53000379562378,
+    "mean_length": 3152.88,
+    "loss": 92.37181091308594,
+    "sps": 241.45699781157901
+  },
+  {
+    "update": 1220,
+    "global_step": 4997120,
+    "num_episodes": 1675,
+    "mean_reward": 49.94117986917496,
+    "mean_length": 3244.89,
+    "loss": 36.136539459228516,
+    "sps": 332.50957910682797
+  },
+  {
+    "update": 1225,
+    "global_step": 5017600,
+    "num_episodes": 1681,
+    "mean_reward": 49.826350367069246,
+    "mean_length": 3322.41,
+    "loss": 4.504380702972412,
+    "sps": 317.38761005771676
+  },
+  {
+    "update": 1230,
+    "global_step": 5038080,
+    "num_episodes": 1686,
+    "mean_reward": 50.16007830858231,
+    "mean_length": 3383.77,
+    "loss": 11.077418327331543,
+    "sps": 614.5178546338719
+  },
+  {
+    "update": 1235,
+    "global_step": 5058560,
+    "num_episodes": 1689,
+    "mean_reward": 50.77651453256607,
+    "mean_length": 3495.66,
+    "loss": 12.558845520019531,
+    "sps": 574.6067562429008
+  },
+  {
+    "update": 1240,
+    "global_step": 5079040,
+    "num_episodes": 1694,
+    "mean_reward": 51.09682610750198,
+    "mean_length": 3719.18,
+    "loss": 21.6842098236084,
+    "sps": 1195.0937330699562
+  },
+  {
+    "update": 1245,
+    "global_step": 5099520,
+    "num_episodes": 1698,
+    "mean_reward": 50.89644577741623,
+    "mean_length": 3737.48,
+    "loss": 65.59449768066406,
+    "sps": 836.6498465651391
+  },
+  {
+    "update": 1250,
+    "global_step": 5120000,
+    "num_episodes": 1701,
+    "mean_reward": 51.103903777599335,
+    "mean_length": 3865.3,
+    "loss": 20.571155548095703,
+    "sps": 835.7550802547468
+  },
+  {
+    "update": 1255,
+    "global_step": 5140480,
+    "num_episodes": 1705,
+    "mean_reward": 56.42267068624496,
+    "mean_length": 4036.09,
+    "loss": 65.08052825927734,
+    "sps": 391.16626869711587
+  },
+  {
+    "update": 1260,
+    "global_step": 5160960,
+    "num_episodes": 1709,
+    "mean_reward": 57.39474864244461,
+    "mean_length": 4054.32,
+    "loss": 9.735504150390625,
+    "sps": 279.94177596903813
+  },
+  {
+    "update": 1265,
+    "global_step": 5181440,
+    "num_episodes": 1715,
+    "mean_reward": 63.1245819735527,
+    "mean_length": 4262.64,
+    "loss": 473.1329040527344,
+    "sps": 195.8880726253604
+  },
+  {
+    "update": 1270,
+    "global_step": 5201920,
+    "num_episodes": 1719,
+    "mean_reward": 56.77260554075241,
+    "mean_length": 4121.23,
+    "loss": 263.35589599609375,
+    "sps": 350.0223876268303
+  },
+  {
+    "update": 1275,
+    "global_step": 5222400,
+    "num_episodes": 1726,
+    "mean_reward": 57.57951702833176,
+    "mean_length": 3960.67,
+    "loss": 59.782310485839844,
+    "sps": 412.90096002367255
+  },
+  {
+    "update": 1280,
+    "global_step": 5242880,
+    "num_episodes": 1736,
+    "mean_reward": 56.08834368228912,
+    "mean_length": 3644.63,
+    "loss": 52.62528991699219,
+    "sps": 343.349341248895
+  },
+  {
+    "update": 1285,
+    "global_step": 5263360,
+    "num_episodes": 1737,
+    "mean_reward": 55.905996737480166,
+    "mean_length": 3640.62,
+    "loss": 104.78556060791016,
+    "sps": 397.8813187415849
+  },
+  {
+    "update": 1290,
+    "global_step": 5283840,
+    "num_episodes": 1746,
+    "mean_reward": 59.82821818828583,
+    "mean_length": 3347.95,
+    "loss": 935.470947265625,
+    "sps": 355.2664513297179
+  },
+  {
+    "update": 1295,
+    "global_step": 5304320,
+    "num_episodes": 1753,
+    "mean_reward": 86.4339063167572,
+    "mean_length": 3459.02,
+    "loss": 795.3289794921875,
+    "sps": 309.51331334453033
+  },
+  {
+    "update": 1300,
+    "global_step": 5324800,
+    "num_episodes": 1762,
+    "mean_reward": 306.1682329654694,
+    "mean_length": 3702.99,
+    "loss": 621.1657104492188,
+    "sps": 312.8076116030221
+  },
+  {
+    "update": 1305,
+    "global_step": 5345280,
+    "num_episodes": 1763,
+    "mean_reward": 306.26519484996794,
+    "mean_length": 3714.28,
+    "loss": 445.75,
+    "sps": 390.84962605893924
+  },
+  {
+    "update": 1310,
+    "global_step": 5365760,
+    "num_episodes": 1772,
+    "mean_reward": 357.3265011548996,
+    "mean_length": 3776.52,
+    "loss": 1374.097412109375,
+    "sps": 385.8458185201923
+  },
+  {
+    "update": 1315,
+    "global_step": 5386240,
+    "num_episodes": 1773,
+    "mean_reward": 357.70143792629244,
+    "mean_length": 3875.1,
+    "loss": 208.59083557128906,
+    "sps": 555.0694538042676
+  },
+  {
+    "update": 1320,
+    "global_step": 5406720,
+    "num_episodes": 1786,
+    "mean_reward": 362.7652148580551,
+    "mean_length": 3706.89,
+    "loss": 1506.414306640625,
+    "sps": 246.7002255816469
+  },
+  {
+    "update": 1325,
+    "global_step": 5427200,
+    "num_episodes": 1786,
+    "mean_reward": 362.7652148580551,
+    "mean_length": 3706.89,
+    "loss": 46.25688934326172,
+    "sps": 409.5117865802824
+  },
+  {
+    "update": 1330,
+    "global_step": 5447680,
+    "num_episodes": 1791,
+    "mean_reward": 501.35476023197174,
+    "mean_length": 3744.98,
+    "loss": 371.16973876953125,
+    "sps": 278.5499936271881
+  },
+  {
+    "update": 1335,
+    "global_step": 5468160,
+    "num_episodes": 1793,
+    "mean_reward": 607.4379647493363,
+    "mean_length": 3921.02,
+    "loss": 73.33538818359375,
+    "sps": 388.619707385681
+  },
+  {
+    "update": 1340,
+    "global_step": 5488640,
+    "num_episodes": 1796,
+    "mean_reward": 756.3367283582687,
+    "mean_length": 4020.31,
+    "loss": 462.8284606933594,
+    "sps": 342.5094025627594
+  },
+  {
+    "update": 1345,
+    "global_step": 5509120,
+    "num_episodes": 1796,
+    "mean_reward": 756.3367283582687,
+    "mean_length": 4020.31,
+    "loss": 67.39126586914062,
+    "sps": 385.48904917796193
+  },
+  {
+    "update": 1350,
+    "global_step": 5529600,
+    "num_episodes": 1802,
+    "mean_reward": 869.4895800352097,
+    "mean_length": 3967.21,
+    "loss": 12.0632905960083,
+    "sps": 668.8793003173347
+  },
+  {
+    "update": 1355,
+    "global_step": 5550080,
+    "num_episodes": 1804,
+    "mean_reward": 989.9085026121139,
+    "mean_length": 4063.23,
+    "loss": 86.83934783935547,
+    "sps": 418.3353031686608
+  },
+  {
+    "update": 1360,
+    "global_step": 5570560,
+    "num_episodes": 1815,
+    "mean_reward": 1078.798788728714,
+    "mean_length": 3886.21,
+    "loss": 666.8345336914062,
+    "sps": 256.51104316818095
+  },
+  {
+    "update": 1365,
+    "global_step": 5591040,
+    "num_episodes": 1815,
+    "mean_reward": 1078.798788728714,
+    "mean_length": 3886.21,
+    "loss": 900.6356811523438,
+    "sps": 495.47242571130397
+  },
+  {
+    "update": 1370,
+    "global_step": 5611520,
+    "num_episodes": 1824,
+    "mean_reward": 1200.6963546466827,
+    "mean_length": 4027.44,
+    "loss": 20.821474075317383,
+    "sps": 506.7840907522114
+  },
+  {
+    "update": 1375,
+    "global_step": 5632000,
+    "num_episodes": 1826,
+    "mean_reward": 1261.5812614822387,
+    "mean_length": 4015.04,
+    "loss": 4563.7763671875,
+    "sps": 675.537157736375
+  },
+  {
+    "update": 1380,
+    "global_step": 5652480,
+    "num_episodes": 1836,
+    "mean_reward": 1445.384720082283,
+    "mean_length": 4027.89,
+    "loss": 459.9952697753906,
+    "sps": 341.6139254031597
+  },
+  {
+    "update": 1385,
+    "global_step": 5672960,
+    "num_episodes": 1839,
+    "mean_reward": 1445.7170259952545,
+    "mean_length": 4039.25,
+    "loss": 1.4308533668518066,
+    "sps": 1488.9461962855103
+  },
+  {
+    "update": 1390,
+    "global_step": 5693440,
+    "num_episodes": 1856,
+    "mean_reward": 1429.1698562908173,
+    "mean_length": 3901.28,
+    "loss": 13.810539245605469,
+    "sps": 583.7912153704766
+  },
+  {
+    "update": 1395,
+    "global_step": 5713920,
+    "num_episodes": 1863,
+    "mean_reward": 1209.2014095544814,
+    "mean_length": 3639.94,
+    "loss": 273.29278564453125,
+    "sps": 645.3489718832685
+  },
+  {
+    "update": 1400,
+    "global_step": 5734400,
+    "num_episodes": 1871,
+    "mean_reward": 1319.8262105464935,
+    "mean_length": 3745.57,
+    "loss": 16.14458656311035,
+    "sps": 324.31654457005965
+  },
+  {
+    "update": 1405,
+    "global_step": 5754880,
+    "num_episodes": 1887,
+    "mean_reward": 1180.7784528923034,
+    "mean_length": 3295.57,
+    "loss": 3.5345799922943115,
+    "sps": 635.0555390978727
+  },
+  {
+    "update": 1410,
+    "global_step": 5775360,
+    "num_episodes": 1897,
+    "mean_reward": 920.7941575908661,
+    "mean_length": 2829.74,
+    "loss": 29.106164932250977,
+    "sps": 283.5497895518039
+  },
+  {
+    "update": 1415,
+    "global_step": 5795840,
+    "num_episodes": 1899,
+    "mean_reward": 1015.2013060569764,
+    "mean_length": 2877.24,
+    "loss": 134.17953491210938,
+    "sps": 514.4558919437959
+  },
+  {
+    "update": 1420,
+    "global_step": 5816320,
+    "num_episodes": 1905,
+    "mean_reward": 759.3930412435532,
+    "mean_length": 2493.3,
+    "loss": 71.0991439819336,
+    "sps": 407.36350916126725
+  },
+  {
+    "update": 1425,
+    "global_step": 5836800,
+    "num_episodes": 1923,
+    "mean_reward": 591.994899687767,
+    "mean_length": 2306.11,
+    "loss": 665.6765747070312,
+    "sps": 292.25754672826326
+  },
+  {
+    "update": 1430,
+    "global_step": 5857280,
+    "num_episodes": 1926,
+    "mean_reward": 491.9243581056595,
+    "mean_length": 2308.91,
+    "loss": 97.95196533203125,
+    "sps": 398.43841528471495
+  },
+  {
+    "update": 1435,
+    "global_step": 5877760,
+    "num_episodes": 1934,
+    "mean_reward": 396.85663065433505,
+    "mean_length": 2257.74,
+    "loss": 1123.9271240234375,
+    "sps": 467.5370774012981
+  },
+  {
+    "update": 1440,
+    "global_step": 5898240,
+    "num_episodes": 1939,
+    "mean_reward": 444.68491759777066,
+    "mean_length": 2252.07,
+    "loss": 149.9423065185547,
+    "sps": 397.96294200167154
+  },
+  {
+    "update": 1445,
+    "global_step": 5918720,
+    "num_episodes": 1952,
+    "mean_reward": 490.6093361234665,
+    "mean_length": 2347.41,
+    "loss": 188.90318298339844,
+    "sps": 433.4020614000179
+  },
+  {
+    "update": 1450,
+    "global_step": 5939200,
+    "num_episodes": 1968,
+    "mean_reward": 341.27103984832763,
+    "mean_length": 2088.38,
+    "loss": 49.23589324951172,
+    "sps": 192.3194868778508
+  },
+  {
+    "update": 1455,
+    "global_step": 5959680,
+    "num_episodes": 1980,
+    "mean_reward": 340.5469175004959,
+    "mean_length": 2102.8,
+    "loss": 53.03819274902344,
+    "sps": 401.45828106312945
+  },
+  {
+    "update": 1460,
+    "global_step": 5980160,
+    "num_episodes": 1987,
+    "mean_reward": 341.23766746520994,
+    "mean_length": 2290.56,
+    "loss": 12.17508316040039,
+    "sps": 321.48619942614465
+  },
+  {
+    "update": 1465,
+    "global_step": 6000640,
+    "num_episodes": 1993,
+    "mean_reward": 340.6004219532013,
+    "mean_length": 2289.4,
+    "loss": 4.770975589752197,
+    "sps": 400.6137968711218
+  },
+  {
+    "update": 1470,
+    "global_step": 6021120,
+    "num_episodes": 2004,
+    "mean_reward": 321.17223945140836,
+    "mean_length": 2148.56,
+    "loss": 29.761117935180664,
+    "sps": 472.58742855602395
+  },
+  {
+    "update": 1475,
+    "global_step": 6041600,
+    "num_episodes": 2026,
+    "mean_reward": 336.3226027822495,
+    "mean_length": 1952.34,
+    "loss": 203.31748962402344,
+    "sps": 357.29518794795473
+  },
+  {
+    "update": 1480,
+    "global_step": 6062080,
+    "num_episodes": 2033,
+    "mean_reward": 313.32783755540845,
+    "mean_length": 1958.32,
+    "loss": 508.1851501464844,
+    "sps": 510.89708007010705
+  },
+  {
+    "update": 1485,
+    "global_step": 6082560,
+    "num_episodes": 2046,
+    "mean_reward": 194.68075824975966,
+    "mean_length": 1639.76,
+    "loss": 7.842682361602783,
+    "sps": 546.9033062498472
+  },
+  {
+    "update": 1490,
+    "global_step": 6103040,
+    "num_episodes": 2050,
+    "mean_reward": 143.90823694944382,
+    "mean_length": 1738.66,
+    "loss": 46.426143646240234,
+    "sps": 306.85539615127885
+  },
+  {
+    "update": 1495,
+    "global_step": 6123520,
+    "num_episodes": 2060,
+    "mean_reward": 334.49683523893356,
+    "mean_length": 1898.53,
+    "loss": 468.38525390625,
+    "sps": 245.82053873712613
+  },
+  {
+    "update": 1500,
+    "global_step": 6144000,
+    "num_episodes": 2069,
+    "mean_reward": 334.39896435022354,
+    "mean_length": 2044.37,
+    "loss": 85.75762939453125,
+    "sps": 334.1080989119973
+  },
+  {
+    "update": 1505,
+    "global_step": 6164480,
+    "num_episodes": 2074,
+    "mean_reward": 333.67030994653703,
+    "mean_length": 1990.3,
+    "loss": 295.0758361816406,
+    "sps": 345.61576592839117
+  },
+  {
+    "update": 1510,
+    "global_step": 6184960,
+    "num_episodes": 2080,
+    "mean_reward": 334.66018194913863,
+    "mean_length": 2150.39,
+    "loss": 10.077195167541504,
+    "sps": 420.2391860174057
+  },
+  {
+    "update": 1515,
+    "global_step": 6205440,
+    "num_episodes": 2097,
+    "mean_reward": 258.6993135094643,
+    "mean_length": 2109.74,
+    "loss": 122.61629486083984,
+    "sps": 229.56375224355847
+  },
+  {
+    "update": 1520,
+    "global_step": 6225920,
+    "num_episodes": 2107,
+    "mean_reward": 258.4274232983589,
+    "mean_length": 2108.42,
+    "loss": 361.1533203125,
+    "sps": 361.8525022786034
+  },
+  {
+    "update": 1525,
+    "global_step": 6246400,
+    "num_episodes": 2125,
+    "mean_reward": 239.45979365587235,
+    "mean_length": 1883.93,
+    "loss": 25.989086151123047,
+    "sps": 275.38397015212564
+  },
+  {
+    "update": 1530,
+    "global_step": 6266880,
+    "num_episodes": 2131,
+    "mean_reward": 239.8594556736946,
+    "mean_length": 1981.69,
+    "loss": 404.8509521484375,
+    "sps": 405.6766540706019
+  },
+  {
+    "update": 1535,
+    "global_step": 6287360,
+    "num_episodes": 2146,
+    "mean_reward": 303.8745777082443,
+    "mean_length": 2254.68,
+    "loss": 1170.5218505859375,
+    "sps": 251.0949486970236
+  },
+  {
+    "update": 1540,
+    "global_step": 6307840,
+    "num_episodes": 2147,
+    "mean_reward": 304.0846936607361,
+    "mean_length": 2341.33,
+    "loss": 8.5966157913208,
+    "sps": 446.2390266117631
+  },
+  {
+    "update": 1545,
+    "global_step": 6328320,
+    "num_episodes": 2154,
+    "mean_reward": 437.3326745700836,
+    "mean_length": 2280.45,
+    "loss": 1299.16552734375,
+    "sps": 259.202559576872
+  },
+  {
+    "update": 1550,
+    "global_step": 6348800,
+    "num_episodes": 2158,
+    "mean_reward": 303.4427977991104,
+    "mean_length": 2211.31,
+    "loss": 1.1450996398925781,
+    "sps": 549.5109518078744
+  },
+  {
+    "update": 1555,
+    "global_step": 6369280,
+    "num_episodes": 2172,
+    "mean_reward": 304.4946458005905,
+    "mean_length": 2210.89,
+    "loss": 59.038814544677734,
+    "sps": 341.25380949315337
+  },
+  {
+    "update": 1560,
+    "global_step": 6389760,
+    "num_episodes": 2175,
+    "mean_reward": 304.3582187414169,
+    "mean_length": 2211.87,
+    "loss": 13.61323070526123,
+    "sps": 642.8393173099237
+  },
+  {
+    "update": 1565,
+    "global_step": 6410240,
+    "num_episodes": 2180,
+    "mean_reward": 305.09624064445495,
+    "mean_length": 2307.71,
+    "loss": 1913.05126953125,
+    "sps": 391.8241680446758
+  },
+  {
+    "update": 1570,
+    "global_step": 6430720,
+    "num_episodes": 2183,
+    "mean_reward": 305.77074447154996,
+    "mean_length": 2405.35,
+    "loss": 136.88858032226562,
+    "sps": 394.7672396028363
+  },
+  {
+    "update": 1575,
+    "global_step": 6451200,
+    "num_episodes": 2189,
+    "mean_reward": 405.41952825069427,
+    "mean_length": 2593.85,
+    "loss": 83.12802124023438,
+    "sps": 299.9621165564932
+  },
+  {
+    "update": 1580,
+    "global_step": 6471680,
+    "num_episodes": 2207,
+    "mean_reward": 405.9634767389297,
+    "mean_length": 2524.02,
+    "loss": 13081.1357421875,
+    "sps": 223.53810669328672
+  },
+  {
+    "update": 1585,
+    "global_step": 6492160,
+    "num_episodes": 2211,
+    "mean_reward": 499.87862377643586,
+    "mean_length": 2495.58,
+    "loss": 46.772918701171875,
+    "sps": 453.36473969791376
+  },
+  {
+    "update": 1590,
+    "global_step": 6512640,
+    "num_episodes": 2221,
+    "mean_reward": 670.4232928800583,
+    "mean_length": 2706.97,
+    "loss": 368.63763427734375,
+    "sps": 429.39001544641116
+  },
+  {
+    "update": 1595,
+    "global_step": 6533120,
+    "num_episodes": 2227,
+    "mean_reward": 803.7308926296234,
+    "mean_length": 2725.72,
+    "loss": 90.26509094238281,
+    "sps": 413.30646156887036
+  },
+  {
+    "update": 1600,
+    "global_step": 6553600,
+    "num_episodes": 2232,
+    "mean_reward": 933.0243373584748,
+    "mean_length": 2915.95,
+    "loss": 68.28546142578125,
+    "sps": 374.56623534825485
+  },
+  {
+    "update": 1605,
+    "global_step": 6574080,
+    "num_episodes": 2235,
+    "mean_reward": 933.269734544754,
+    "mean_length": 2912.74,
+    "loss": 865.8887939453125,
+    "sps": 445.657946827829
+  },
+  {
+    "update": 1610,
+    "global_step": 6594560,
+    "num_episodes": 2237,
+    "mean_reward": 933.7870253133774,
+    "mean_length": 3011.88,
+    "loss": 871.9052734375,
+    "sps": 387.41989720305287
+  },
+  {
+    "update": 1615,
+    "global_step": 6615040,
+    "num_episodes": 2248,
+    "mean_reward": 1077.8849126005173,
+    "mean_length": 3056.53,
+    "loss": 54.994510650634766,
+    "sps": 423.33621401282124
+  },
+  {
+    "update": 1620,
+    "global_step": 6635520,
+    "num_episodes": 2258,
+    "mean_reward": 943.9673701143265,
+    "mean_length": 3023.43,
+    "loss": 137.88092041015625,
+    "sps": 374.57440205325895
+  },
+  {
+    "update": 1625,
+    "global_step": 6656000,
+    "num_episodes": 2263,
+    "mean_reward": 856.1039279556275,
+    "mean_length": 2924.57,
+    "loss": 348.59234619140625,
+    "sps": 377.6081162361363
+  },
+  {
+    "update": 1630,
+    "global_step": 6676480,
+    "num_episodes": 2267,
+    "mean_reward": 931.3061418437958,
+    "mean_length": 2926.62,
+    "loss": 9.557199478149414,
+    "sps": 460.1107258733158
+  },
+  {
+    "update": 1635,
+    "global_step": 6696960,
+    "num_episodes": 2273,
+    "mean_reward": 931.8358981752395,
+    "mean_length": 3120.06,
+    "loss": 514.8411254882812,
+    "sps": 523.5469529045816
+  },
+  {
+    "update": 1640,
+    "global_step": 6717440,
+    "num_episodes": 2276,
+    "mean_reward": 1109.3497059488298,
+    "mean_length": 3219.09,
+    "loss": 31.73206329345703,
+    "sps": 412.7885853043598
+  },
+  {
+    "update": 1645,
+    "global_step": 6737920,
+    "num_episodes": 2282,
+    "mean_reward": 1197.9563966131211,
+    "mean_length": 3220.5,
+    "loss": 35.86986541748047,
+    "sps": 466.28907053794126
+  },
+  {
+    "update": 1650,
+    "global_step": 6758400,
+    "num_episodes": 2285,
+    "mean_reward": 1119.968164858818,
+    "mean_length": 3067.54,
+    "loss": 279.3809509277344,
+    "sps": 415.9298164658184
+  },
+  {
+    "update": 1655,
+    "global_step": 6778880,
+    "num_episodes": 2299,
+    "mean_reward": 1277.0260197210312,
+    "mean_length": 3026.13,
+    "loss": 44.909996032714844,
+    "sps": 289.10855632711406
+  },
+  {
+    "update": 1660,
+    "global_step": 6799360,
+    "num_episodes": 2310,
+    "mean_reward": 1212.9116531801224,
+    "mean_length": 3157.69,
+    "loss": 19.89190673828125,
+    "sps": 442.19954156582503
+  },
+  {
+    "update": 1665,
+    "global_step": 6819840,
+    "num_episodes": 2313,
+    "mean_reward": 1237.1113989305495,
+    "mean_length": 3267.66,
+    "loss": 49.466552734375,
+    "sps": 941.6208281097553
+  },
+  {
+    "update": 1670,
+    "global_step": 6840320,
+    "num_episodes": 2320,
+    "mean_reward": 1185.366361656189,
+    "mean_length": 3170.05,
+    "loss": 226.9459686279297,
+    "sps": 560.8755073464478
+  },
+  {
+    "update": 1675,
+    "global_step": 6860800,
+    "num_episodes": 2327,
+    "mean_reward": 1171.2066772079468,
+    "mean_length": 3355.09,
+    "loss": 10.15625,
+    "sps": 335.8002299447698
+  },
+  {
+    "update": 1680,
+    "global_step": 6881280,
+    "num_episodes": 2333,
+    "mean_reward": 1042.0874049663544,
+    "mean_length": 3211.85,
+    "loss": 82.60301971435547,
+    "sps": 302.2235800221304
+  },
+  {
+    "update": 1685,
+    "global_step": 6901760,
+    "num_episodes": 2339,
+    "mean_reward": 972.5269057083129,
+    "mean_length": 3193.44,
+    "loss": 120.32151794433594,
+    "sps": 664.7036679347333
+  },
+  {
+    "update": 1690,
+    "global_step": 6922240,
+    "num_episodes": 2339,
+    "mean_reward": 972.5269057083129,
+    "mean_length": 3193.44,
+    "loss": 1.7924470901489258,
+    "sps": 1063.7604928109242
+  },
+  {
+    "update": 1695,
+    "global_step": 6942720,
+    "num_episodes": 2342,
+    "mean_reward": 942.9001926231384,
+    "mean_length": 3342.44,
+    "loss": 6.127931594848633,
+    "sps": 592.3562356846871
+  },
+  {
+    "update": 1700,
+    "global_step": 6963200,
+    "num_episodes": 2345,
+    "mean_reward": 943.6058731651306,
+    "mean_length": 3474.03,
+    "loss": 44.08333206176758,
+    "sps": 297.44937442353597
+  },
+  {
+    "update": 1705,
+    "global_step": 6983680,
+    "num_episodes": 2350,
+    "mean_reward": 891.2378608131409,
+    "mean_length": 3613.22,
+    "loss": 139.50340270996094,
+    "sps": 321.276001001162
+  },
+  {
+    "update": 1710,
+    "global_step": 7004160,
+    "num_episodes": 2351,
+    "mean_reward": 891.2994220161438,
+    "mean_length": 3670.58,
+    "loss": 80.7029800415039,
+    "sps": 602.1302177845877
+  },
+  {
+    "update": 1715,
+    "global_step": 7024640,
+    "num_episodes": 2354,
+    "mean_reward": 891.3987969732284,
+    "mean_length": 3707.87,
+    "loss": 43.55419921875,
+    "sps": 609.1286236648131
+  },
+  {
+    "update": 1720,
+    "global_step": 7045120,
+    "num_episodes": 2356,
+    "mean_reward": 908.0675931024551,
+    "mean_length": 3894.63,
+    "loss": 1924.8736572265625,
+    "sps": 521.5708115336275
+  },
+  {
+    "update": 1725,
+    "global_step": 7065600,
+    "num_episodes": 2368,
+    "mean_reward": 877.4067792797089,
+    "mean_length": 4024.38,
+    "loss": 1546.024658203125,
+    "sps": 144.1450251623922
+  },
+  {
+    "update": 1730,
+    "global_step": 7086080,
+    "num_episodes": 2371,
+    "mean_reward": 876.8285467529297,
+    "mean_length": 4001.29,
+    "loss": 140.04344177246094,
+    "sps": 219.3887746950412
+  },
+  {
+    "update": 1735,
+    "global_step": 7106560,
+    "num_episodes": 2379,
+    "mean_reward": 712.0810299873352,
+    "mean_length": 3765.17,
+    "loss": 592.1142578125,
+    "sps": 154.4538739502226
+  },
+  {
+    "update": 1740,
+    "global_step": 7127040,
+    "num_episodes": 2380,
+    "mean_reward": 711.973635725975,
+    "mean_length": 3803.94,
+    "loss": 94.82659149169922,
+    "sps": 312.59798550353105
+  },
+  {
+    "update": 1745,
+    "global_step": 7147520,
+    "num_episodes": 2386,
+    "mean_reward": 721.5674746751786,
+    "mean_length": 4056.06,
+    "loss": 62.10371398925781,
+    "sps": 320.26559324554347
+  },
+  {
+    "update": 1750,
+    "global_step": 7168000,
+    "num_episodes": 2388,
+    "mean_reward": 722.019714179039,
+    "mean_length": 4119.0,
+    "loss": 154.1345977783203,
+    "sps": 490.86442866974096
+  },
+  {
+    "update": 1755,
+    "global_step": 7188480,
+    "num_episodes": 2393,
+    "mean_reward": 641.6925968694687,
+    "mean_length": 4085.19,
+    "loss": 3164.498779296875,
+    "sps": 258.3838207181636
+  },
+  {
+    "update": 1760,
+    "global_step": 7208960,
+    "num_episodes": 2395,
+    "mean_reward": 686.4729778194428,
+    "mean_length": 4253.36,
+    "loss": 425.0414733886719,
+    "sps": 362.7607269459368
+  },
+  {
+    "update": 1765,
+    "global_step": 7229440,
+    "num_episodes": 2398,
+    "mean_reward": 781.3392351341248,
+    "mean_length": 4537.14,
+    "loss": 31.12351417541504,
+    "sps": 385.67594003613885
+  },
+  {
+    "update": 1770,
+    "global_step": 7249920,
+    "num_episodes": 2401,
+    "mean_reward": 780.9093813705445,
+    "mean_length": 4438.34,
+    "loss": 154.23956298828125,
+    "sps": 350.9404403984712
+  },
+  {
+    "update": 1775,
+    "global_step": 7270400,
+    "num_episodes": 2409,
+    "mean_reward": 920.4024870014191,
+    "mean_length": 4671.4,
+    "loss": 68.82001495361328,
+    "sps": 373.0457413366687
+  },
+  {
+    "update": 1780,
+    "global_step": 7290880,
+    "num_episodes": 2431,
+    "mean_reward": 661.6825140190125,
+    "mean_length": 4178.67,
+    "loss": 1634.844482421875,
+    "sps": 334.82655519105265
+  },
+  {
+    "update": 1785,
+    "global_step": 7311360,
+    "num_episodes": 2436,
+    "mean_reward": 703.8663965082169,
+    "mean_length": 4154.8,
+    "loss": 36.222721099853516,
+    "sps": 690.9562224821975
+  },
+  {
+    "update": 1790,
+    "global_step": 7331840,
+    "num_episodes": 2442,
+    "mean_reward": 682.8625202751159,
+    "mean_length": 3792.16,
+    "loss": 7.088906288146973,
+    "sps": 546.4602476565249
+  },
+  {
+    "update": 1795,
+    "global_step": 7352320,
+    "num_episodes": 2452,
+    "mean_reward": 668.2820160484314,
+    "mean_length": 3464.88,
+    "loss": 15.93359375,
+    "sps": 338.3906013362985
+  },
+  {
+    "update": 1800,
+    "global_step": 7372800,
+    "num_episodes": 2479,
+    "mean_reward": 503.18441142082213,
+    "mean_length": 2745.39,
+    "loss": 4.317780494689941,
+    "sps": 247.07750153960802
+  },
+  {
+    "update": 1805,
+    "global_step": 7393280,
+    "num_episodes": 2499,
+    "mean_reward": 254.19922243118287,
+    "mean_length": 1727.29,
+    "loss": 2.4029455184936523,
+    "sps": 343.1557628026188
+  },
+  {
+    "update": 1810,
+    "global_step": 7413760,
+    "num_episodes": 2505,
+    "mean_reward": 156.29735308647156,
+    "mean_length": 1608.82,
+    "loss": 50.26245880126953,
+    "sps": 545.8711155437793
+  },
+  {
+    "update": 1815,
+    "global_step": 7434240,
+    "num_episodes": 2511,
+    "mean_reward": 86.22271653652192,
+    "mean_length": 1490.02,
+    "loss": 0.49680042266845703,
+    "sps": 765.2176779813885
+  },
+  {
+    "update": 1820,
+    "global_step": 7454720,
+    "num_episodes": 2517,
+    "mean_reward": 87.76825302124024,
+    "mean_length": 1667.16,
+    "loss": 0.9337818622589111,
+    "sps": 741.1448982958542
+  },
+  {
+    "update": 1825,
+    "global_step": 7475200,
+    "num_episodes": 2525,
+    "mean_reward": 83.8934311246872,
+    "mean_length": 1771.63,
+    "loss": 1.8736273050308228,
+    "sps": 346.82608481484516
+  },
+  {
+    "update": 1830,
+    "global_step": 7495680,
+    "num_episodes": 2536,
+    "mean_reward": 42.52432149887085,
+    "mean_length": 1779.55,
+    "loss": 43.508872985839844,
+    "sps": 538.257412633311
+  },
+  {
+    "update": 1835,
+    "global_step": 7516160,
+    "num_episodes": 2559,
+    "mean_reward": 22.3750723361969,
+    "mean_length": 1585.34,
+    "loss": 11.682701110839844,
+    "sps": 283.17378891804617
+  },
+  {
+    "update": 1840,
+    "global_step": 7536640,
+    "num_episodes": 2572,
+    "mean_reward": 23.224842309951782,
+    "mean_length": 1714.64,
+    "loss": 0.64460289478302,
+    "sps": 945.2402518055575
+  },
+  {
+    "update": 1845,
+    "global_step": 7557120,
+    "num_episodes": 2575,
+    "mean_reward": 23.735417137145998,
+    "mean_length": 1716.6,
+    "loss": 1.5831042528152466,
+    "sps": 1337.9087240256438
+  },
+  {
+    "update": 1850,
+    "global_step": 7577600,
+    "num_episodes": 2575,
+    "mean_reward": 23.735417137145998,
+    "mean_length": 1716.6,
+    "loss": 0.5824832320213318,
+    "sps": 1306.8006548657663
+  },
+  {
+    "update": 1855,
+    "global_step": 7598080,
+    "num_episodes": 2579,
+    "mean_reward": 25.101176109313965,
+    "mean_length": 2001.32,
+    "loss": 3.1498076915740967,
+    "sps": 383.12937784399907
+  },
+  {
+    "update": 1860,
+    "global_step": 7618560,
+    "num_episodes": 2607,
+    "mean_reward": 27.537090787887575,
+    "mean_length": 2252.34,
+    "loss": 2.7028920650482178,
+    "sps": 276.49082168209895
+  },
+  {
+    "update": 1865,
+    "global_step": 7639040,
+    "num_episodes": 2609,
+    "mean_reward": 27.678814001083374,
+    "mean_length": 2266.75,
+    "loss": 1.0194354057312012,
+    "sps": 829.6019480631423
+  },
+  {
+    "update": 1870,
+    "global_step": 7659520,
+    "num_episodes": 2609,
+    "mean_reward": 27.678814001083374,
+    "mean_length": 2266.75,
+    "loss": 0.69590824842453,
+    "sps": 1339.889941319705
+  },
+  {
+    "update": 1875,
+    "global_step": 7680000,
+    "num_episodes": 2610,
+    "mean_reward": 27.743932600021363,
+    "mean_length": 2363.84,
+    "loss": 4.22328519821167,
+    "sps": 1047.687217978352
+  },
+  {
+    "update": 1880,
+    "global_step": 7700480,
+    "num_episodes": 2623,
+    "mean_reward": 28.715706405639647,
+    "mean_length": 2373.15,
+    "loss": 15.052788734436035,
+    "sps": 180.92434386649956
+  },
+  {
+    "update": 1885,
+    "global_step": 7720960,
+    "num_episodes": 2650,
+    "mean_reward": 28.79868775844574,
+    "mean_length": 2376.77,
+    "loss": 141.97862243652344,
+    "sps": 189.85363801957897
+  },
+  {
+    "update": 1890,
+    "global_step": 7741440,
+    "num_episodes": 2663,
+    "mean_reward": 29.01506271839142,
+    "mean_length": 2187.12,
+    "loss": 2.7120704650878906,
+    "sps": 442.6728082704278
+  },
+  {
+    "update": 1895,
+    "global_step": 7761920,
+    "num_episodes": 2664,
+    "mean_reward": 28.84576090812683,
+    "mean_length": 2212.69,
+    "loss": 0.6469894647598267,
+    "sps": 1603.7569257295258
+  },
+  {
+    "update": 1900,
+    "global_step": 7782400,
+    "num_episodes": 2665,
+    "mean_reward": 29.100953969955444,
+    "mean_length": 2212.69,
+    "loss": 5.927608489990234,
+    "sps": 1560.5345956258918
+  },
+  {
+    "update": 1905,
+    "global_step": 7802880,
+    "num_episodes": 2677,
+    "mean_reward": 29.594479999542237,
+    "mean_length": 2279.92,
+    "loss": 3.0546085834503174,
+    "sps": 239.632742367245
+  },
+  {
+    "update": 1910,
+    "global_step": 7823360,
+    "num_episodes": 2696,
+    "mean_reward": 29.67495318889618,
+    "mean_length": 2013.09,
+    "loss": 0.40485668182373047,
+    "sps": 483.9873532409158
+  },
+  {
+    "update": 1915,
+    "global_step": 7843840,
+    "num_episodes": 2706,
+    "mean_reward": 30.284353771209716,
+    "mean_length": 2114.96,
+    "loss": 1.9687494039535522,
+    "sps": 668.5066115534931
+  },
+  {
+    "update": 1920,
+    "global_step": 7864320,
+    "num_episodes": 2713,
+    "mean_reward": 28.00047863960266,
+    "mean_length": 1718.87,
+    "loss": 0.2566870450973511,
+    "sps": 774.1152448288775
+  },
+  {
+    "update": 1925,
+    "global_step": 7884800,
+    "num_episodes": 2728,
+    "mean_reward": 28.49471785068512,
+    "mean_length": 1709.19,
+    "loss": 0.5462437272071838,
+    "sps": 335.32752289720815
+  },
+  {
+    "update": 1930,
+    "global_step": 7905280,
+    "num_episodes": 2749,
+    "mean_reward": 29.774335384368896,
+    "mean_length": 1793.46,
+    "loss": 1.6195909976959229,
+    "sps": 151.6658643145884
+  },
+  {
+    "update": 1935,
+    "global_step": 7925760,
+    "num_episodes": 2763,
+    "mean_reward": 29.902848501205444,
+    "mean_length": 1903.12,
+    "loss": 3.346653461456299,
+    "sps": 299.9200663991073
+  },
+  {
+    "update": 1940,
+    "global_step": 7946240,
+    "num_episodes": 2766,
+    "mean_reward": 29.951358599662782,
+    "mean_length": 1817.08,
+    "loss": 2.196648120880127,
+    "sps": 612.8349441003236
+  },
+  {
+    "update": 1945,
+    "global_step": 7966720,
+    "num_episodes": 2775,
+    "mean_reward": 27.90867582321167,
+    "mean_length": 1530.25,
+    "loss": 5.727337837219238,
+    "sps": 686.4829120395315
+  },
+  {
+    "update": 1950,
+    "global_step": 7987200,
+    "num_episodes": 2791,
+    "mean_reward": 27.395657448768617,
+    "mean_length": 1515.36,
+    "loss": 0.2642287015914917,
+    "sps": 427.4124315071339
+  },
+  {
+    "update": 1955,
+    "global_step": 8007680,
+    "num_episodes": 2811,
+    "mean_reward": 29.46188481807709,
+    "mean_length": 1804.43,
+    "loss": 1.1504522562026978,
+    "sps": 440.35140268496957
+  },
+  {
+    "update": 1960,
+    "global_step": 8028160,
+    "num_episodes": 2811,
+    "mean_reward": 29.46188481807709,
+    "mean_length": 1804.43,
+    "loss": 0.6443239450454712,
+    "sps": 897.6638295466576
+  },
+  {
+    "update": 1965,
+    "global_step": 8048640,
+    "num_episodes": 2811,
+    "mean_reward": 29.46188481807709,
+    "mean_length": 1804.43,
+    "loss": 0.9464260339736938,
+    "sps": 1231.9540613883512
+  },
+  {
+    "update": 1970,
+    "global_step": 8069120,
+    "num_episodes": 2816,
+    "mean_reward": 30.10758180618286,
+    "mean_length": 1900.22,
+    "loss": 13.55959701538086,
+    "sps": 762.2875965279533
+  },
+  {
+    "update": 1975,
+    "global_step": 8089600,
+    "num_episodes": 2836,
+    "mean_reward": 33.175699801445006,
+    "mean_length": 2303.58,
+    "loss": 7.275441646575928,
+    "sps": 227.75311612847298
+  },
+  {
+    "update": 1980,
+    "global_step": 8110080,
+    "num_episodes": 2856,
+    "mean_reward": 29.203288111686707,
+    "mean_length": 2029.45,
+    "loss": 39.95777130126953,
+    "sps": 325.31625618278383
+  },
+  {
+    "update": 1985,
+    "global_step": 8130560,
+    "num_episodes": 2877,
+    "mean_reward": 26.940120091438292,
+    "mean_length": 1738.19,
+    "loss": 2.9013671875,
+    "sps": 477.60825044232615
+  },
+  {
+    "update": 1990,
+    "global_step": 8151040,
+    "num_episodes": 2881,
+    "mean_reward": 26.892020201683046,
+    "mean_length": 1748.27,
+    "loss": 1.5360685586929321,
+    "sps": 779.3787507434323
+  },
+  {
+    "update": 1995,
+    "global_step": 8171520,
+    "num_episodes": 2887,
+    "mean_reward": 28.383681316375732,
+    "mean_length": 2035.38,
+    "loss": 0.28828829526901245,
+    "sps": 528.7312594886902
+  },
+  {
+    "update": 2000,
+    "global_step": 8192000,
+    "num_episodes": 2894,
+    "mean_reward": 27.269041929244995,
+    "mean_length": 2017.08,
+    "loss": 92.5503158569336,
+    "sps": 423.8490051832835
+  },
+  {
+    "update": 2005,
+    "global_step": 8212480,
+    "num_episodes": 2910,
+    "mean_reward": 29.26204339504242,
+    "mean_length": 2043.4,
+    "loss": 1.737417221069336,
+    "sps": 319.480134990497
+  },
+  {
+    "update": 2010,
+    "global_step": 8232960,
+    "num_episodes": 2914,
+    "mean_reward": 27.918287878036498,
+    "mean_length": 1848.0,
+    "loss": 9.918323516845703,
+    "sps": 520.2825326468419
+  },
+  {
+    "update": 2015,
+    "global_step": 8253440,
+    "num_episodes": 2923,
+    "mean_reward": 25.578041405677794,
+    "mean_length": 1641.01,
+    "loss": 101.00448608398438,
+    "sps": 412.83971034964594
+  },
+  {
+    "update": 2020,
+    "global_step": 8273920,
+    "num_episodes": 2929,
+    "mean_reward": 24.985388655662536,
+    "mean_length": 1728.38,
+    "loss": 33.80594253540039,
+    "sps": 407.32375547775194
+  },
+  {
+    "update": 2025,
+    "global_step": 8294400,
+    "num_episodes": 2941,
+    "mean_reward": 54.48088625907898,
+    "mean_length": 1727.06,
+    "loss": 57.52255630493164,
+    "sps": 251.07848653412563
+  },
+  {
+    "update": 2030,
+    "global_step": 8314880,
+    "num_episodes": 2952,
+    "mean_reward": 62.7720309972763,
+    "mean_length": 1877.51,
+    "loss": 18.3802547454834,
+    "sps": 355.5720033718748
+  },
+  {
+    "update": 2035,
+    "global_step": 8335360,
+    "num_episodes": 2957,
+    "mean_reward": 62.98542010307312,
+    "mean_length": 1979.5,
+    "loss": 1.3044229745864868,
+    "sps": 646.2262220511143
+  },
+  {
+    "update": 2040,
+    "global_step": 8355840,
+    "num_episodes": 2961,
+    "mean_reward": 63.64336392879486,
+    "mean_length": 2180.16,
+    "loss": 92.43401336669922,
+    "sps": 659.5681486510539
+  },
+  {
+    "update": 2045,
+    "global_step": 8376320,
+    "num_episodes": 2962,
+    "mean_reward": 63.40275900363922,
+    "mean_length": 2203.23,
+    "loss": 1.014630675315857,
+    "sps": 1446.1879050125103
+  },
+  {
+    "update": 2050,
+    "global_step": 8396800,
+    "num_episodes": 2986,
+    "mean_reward": 79.26061116695404,
+    "mean_length": 2214.81,
+    "loss": 29.98980712890625,
+    "sps": 205.60997792847448
+  },
+  {
+    "update": 2055,
+    "global_step": 8417280,
+    "num_episodes": 3001,
+    "mean_reward": 82.85091641426087,
+    "mean_length": 2227.86,
+    "loss": 32.844966888427734,
+    "sps": 159.2487235394598
+  },
+  {
+    "update": 2060,
+    "global_step": 8437760,
+    "num_episodes": 3027,
+    "mean_reward": 78.89424407482147,
+    "mean_length": 1849.23,
+    "loss": 103.22246551513672,
+    "sps": 213.8993421623802
+  },
+  {
+    "update": 2065,
+    "global_step": 8458240,
+    "num_episodes": 3033,
+    "mean_reward": 78.97739854335785,
+    "mean_length": 1848.21,
+    "loss": 29.219009399414062,
+    "sps": 303.98893287836097
+  },
+  {
+    "update": 2070,
+    "global_step": 8478720,
+    "num_episodes": 3038,
+    "mean_reward": 78.37423317909241,
+    "mean_length": 1807.15,
+    "loss": 34.15611267089844,
+    "sps": 219.78900023670354
+  },
+  {
+    "update": 2075,
+    "global_step": 8499200,
+    "num_episodes": 3052,
+    "mean_reward": 75.33004099369049,
+    "mean_length": 1865.63,
+    "loss": 26.766494750976562,
+    "sps": 339.7451627672537
+  },
+  {
+    "update": 2080,
+    "global_step": 8519680,
+    "num_episodes": 3056,
+    "mean_reward": 75.64053434848785,
+    "mean_length": 1968.04,
+    "loss": 176.32748413085938,
+    "sps": 334.0194819221175
+  },
+  {
+    "update": 2085,
+    "global_step": 8540160,
+    "num_episodes": 3066,
+    "mean_reward": 74.47905586719513,
+    "mean_length": 1544.14,
+    "loss": 167.32608032226562,
+    "sps": 393.50550599249544
+  },
+  {
+    "update": 2090,
+    "global_step": 8560640,
+    "num_episodes": 3072,
+    "mean_reward": 75.01324489593506,
+    "mean_length": 1620.95,
+    "loss": 40.1760139465332,
+    "sps": 272.5369799883836
+  },
+  {
+    "update": 2095,
+    "global_step": 8581120,
+    "num_episodes": 3084,
+    "mean_reward": 80.59875289440156,
+    "mean_length": 1815.35,
+    "loss": 23.691143035888672,
+    "sps": 460.4555876617225
+  },
+  {
+    "update": 2100,
+    "global_step": 8601600,
+    "num_episodes": 3092,
+    "mean_reward": 151.1377474308014,
+    "mean_length": 2023.52,
+    "loss": 119.31157684326172,
+    "sps": 230.98807371797147
+  },
+  {
+    "update": 2105,
+    "global_step": 8622080,
+    "num_episodes": 3104,
+    "mean_reward": 145.0986036682129,
+    "mean_length": 1964.44,
+    "loss": 17.907983779907227,
+    "sps": 136.7176138415176
+  },
+  {
+    "update": 2110,
+    "global_step": 8642560,
+    "num_episodes": 3113,
+    "mean_reward": 145.05979912757874,
+    "mean_length": 2015.23,
+    "loss": 238.5266876220703,
+    "sps": 191.61578614248398
+  },
+  {
+    "update": 2115,
+    "global_step": 8663040,
+    "num_episodes": 3127,
+    "mean_reward": 145.9841158914566,
+    "mean_length": 2223.16,
+    "loss": 279.5526428222656,
+    "sps": 178.22145258830818
+  },
+  {
+    "update": 2120,
+    "global_step": 8683520,
+    "num_episodes": 3129,
+    "mean_reward": 161.8294042444229,
+    "mean_length": 2317.37,
+    "loss": 4402.13232421875,
+    "sps": 187.96868067815723
+  },
+  {
+    "update": 2125,
+    "global_step": 8704000,
+    "num_episodes": 3131,
+    "mean_reward": 254.4432011270523,
+    "mean_length": 2415.71,
+    "loss": 253.15243530273438,
+    "sps": 389.3006404112279
+  },
+  {
+    "update": 2130,
+    "global_step": 8724480,
+    "num_episodes": 3136,
+    "mean_reward": 308.86108244895934,
+    "mean_length": 2455.38,
+    "loss": 42.87656784057617,
+    "sps": 289.06916804822475
+  },
+  {
+    "update": 2135,
+    "global_step": 8744960,
+    "num_episodes": 3151,
+    "mean_reward": 442.90291794776914,
+    "mean_length": 2544.55,
+    "loss": 52.280975341796875,
+    "sps": 168.18531939110147
+  },
+  {
+    "update": 2140,
+    "global_step": 8765440,
+    "num_episodes": 3153,
+    "mean_reward": 443.3635806703567,
+    "mean_length": 2596.27,
+    "loss": 123.76752471923828,
+    "sps": 250.64368624913917
+  },
+  {
+    "update": 2145,
+    "global_step": 8785920,
+    "num_episodes": 3154,
+    "mean_reward": 443.81920017719267,
+    "mean_length": 2693.35,
+    "loss": 8.35338020324707,
+    "sps": 403.9011407049708
+  },
+  {
+    "update": 2150,
+    "global_step": 8806400,
+    "num_episodes": 3158,
+    "mean_reward": 578.3950145959855,
+    "mean_length": 2690.97,
+    "loss": 117.6234359741211,
+    "sps": 427.3976727795614
+  },
+  {
+    "update": 2155,
+    "global_step": 8826880,
+    "num_episodes": 3168,
+    "mean_reward": 593.7985137224198,
+    "mean_length": 2964.72,
+    "loss": 27.907575607299805,
+    "sps": 408.92434877647025
+  },
+  {
+    "update": 2160,
+    "global_step": 8847360,
+    "num_episodes": 3176,
+    "mean_reward": 696.11108481884,
+    "mean_length": 2789.14,
+    "loss": 37.44743347167969,
+    "sps": 405.43106143344477
+  },
+  {
+    "update": 2165,
+    "global_step": 8867840,
+    "num_episodes": 3181,
+    "mean_reward": 691.3086440181733,
+    "mean_length": 2841.13,
+    "loss": 444.12481689453125,
+    "sps": 410.99575565900966
+  },
+  {
+    "update": 2170,
+    "global_step": 8888320,
+    "num_episodes": 3186,
+    "mean_reward": 696.4550316858291,
+    "mean_length": 2910.25,
+    "loss": 221.63902282714844,
+    "sps": 355.0140974319845
+  },
+  {
+    "update": 2175,
+    "global_step": 8908800,
+    "num_episodes": 3191,
+    "mean_reward": 696.0580731773376,
+    "mean_length": 3004.52,
+    "loss": 762.4512939453125,
+    "sps": 303.24516246589195
+  },
+  {
+    "update": 2180,
+    "global_step": 8929280,
+    "num_episodes": 3197,
+    "mean_reward": 814.6755250024795,
+    "mean_length": 3002.62,
+    "loss": 409.33624267578125,
+    "sps": 427.0124459168339
+  },
+  {
+    "update": 2185,
+    "global_step": 8949760,
+    "num_episodes": 3210,
+    "mean_reward": 934.5317647647857,
+    "mean_length": 3211.49,
+    "loss": 95.55060577392578,
+    "sps": 125.75423605418091
+  },
+  {
+    "update": 2190,
+    "global_step": 8970240,
+    "num_episodes": 3214,
+    "mean_reward": 1002.574015007019,
+    "mean_length": 3132.07,
+    "loss": 68.63960266113281,
+    "sps": 206.72189870239387
+  },
+  {
+    "update": 2195,
+    "global_step": 8990720,
+    "num_episodes": 3225,
+    "mean_reward": 1036.229081878662,
+    "mean_length": 3290.05,
+    "loss": 450.1454772949219,
+    "sps": 308.8153724070321
+  },
+  {
+    "update": 2200,
+    "global_step": 9011200,
+    "num_episodes": 3226,
+    "mean_reward": 1175.8488982009887,
+    "mean_length": 3386.93,
+    "loss": 11.516603469848633,
+    "sps": 505.34211227316496
+  },
+  {
+    "update": 2205,
+    "global_step": 9031680,
+    "num_episodes": 3228,
+    "mean_reward": 1241.2947177028657,
+    "mean_length": 3452.12,
+    "loss": 998.0031127929688,
+    "sps": 380.7435127282772
+  },
+  {
+    "update": 2210,
+    "global_step": 9052160,
+    "num_episodes": 3234,
+    "mean_reward": 1148.464052476883,
+    "mean_length": 3444.2,
+    "loss": 0.2201317399740219,
+    "sps": 447.01108197826755
+  },
+  {
+    "update": 2215,
+    "global_step": 9072640,
+    "num_episodes": 3251,
+    "mean_reward": 983.5756113386154,
+    "mean_length": 3185.62,
+    "loss": 10.140728950500488,
+    "sps": 285.88922550188585
+  },
+  {
+    "update": 2220,
+    "global_step": 9093120,
+    "num_episodes": 3267,
+    "mean_reward": 830.9440940666199,
+    "mean_length": 2711.54,
+    "loss": 23.857481002807617,
+    "sps": 225.0139483384211
+  },
+  {
+    "update": 2225,
+    "global_step": 9113600,
+    "num_episodes": 3285,
+    "mean_reward": 755.7127734375,
+    "mean_length": 2369.97,
+    "loss": 263.3361511230469,
+    "sps": 274.6876121563017
+  },
+  {
+    "update": 2230,
+    "global_step": 9134080,
+    "num_episodes": 3291,
+    "mean_reward": 754.6721585416794,
+    "mean_length": 2306.49,
+    "loss": 4289.046875,
+    "sps": 174.4840112420559
+  },
+  {
+    "update": 2235,
+    "global_step": 9154560,
+    "num_episodes": 3298,
+    "mean_reward": 730.0353905773163,
+    "mean_length": 2354.08,
+    "loss": 84.65528869628906,
+    "sps": 265.5093767251286
+  },
+  {
+    "update": 2240,
+    "global_step": 9175040,
+    "num_episodes": 3300,
+    "mean_reward": 660.2916961956024,
+    "mean_length": 2256.21,
+    "loss": 388.5572814941406,
+    "sps": 407.65937488684284
+  },
+  {
+    "update": 2245,
+    "global_step": 9195520,
+    "num_episodes": 3309,
+    "mean_reward": 687.2124669837951,
+    "mean_length": 2477.43,
+    "loss": 172.86532592773438,
+    "sps": 298.38812190862865
+  },
+  {
+    "update": 2250,
+    "global_step": 9216000,
+    "num_episodes": 3310,
+    "mean_reward": 687.4369230651855,
+    "mean_length": 2481.91,
+    "loss": 14.442253112792969,
+    "sps": 667.1891280431901
+  },
+  {
+    "update": 2255,
+    "global_step": 9236480,
+    "num_episodes": 3320,
+    "mean_reward": 686.6187604236603,
+    "mean_length": 2631.57,
+    "loss": 8.769046783447266,
+    "sps": 248.0906360841076
+  },
+  {
+    "update": 2260,
+    "global_step": 9256960,
+    "num_episodes": 3324,
+    "mean_reward": 714.0761754512787,
+    "mean_length": 2596.79,
+    "loss": 2.7038824558258057,
+    "sps": 611.7833643654915
+  },
+  {
+    "update": 2265,
+    "global_step": 9277440,
+    "num_episodes": 3333,
+    "mean_reward": 514.9936645078659,
+    "mean_length": 2480.39,
+    "loss": 107.10433959960938,
+    "sps": 352.06042337330143
+  },
+  {
+    "update": 2270,
+    "global_step": 9297920,
+    "num_episodes": 3333,
+    "mean_reward": 514.9936645078659,
+    "mean_length": 2480.39,
+    "loss": 205.3878936767578,
+    "sps": 557.4872251219026
+  },
+  {
+    "update": 2275,
+    "global_step": 9318400,
+    "num_episodes": 3341,
+    "mean_reward": 516.5512644720078,
+    "mean_length": 2668.78,
+    "loss": 10.443692207336426,
+    "sps": 381.5541543957095
+  },
+  {
+    "update": 2280,
+    "global_step": 9338880,
+    "num_episodes": 3349,
+    "mean_reward": 462.85379876613615,
+    "mean_length": 2668.78,
+    "loss": 442.84942626953125,
+    "sps": 279.7896158892528
+  },
+  {
+    "update": 2285,
+    "global_step": 9359360,
+    "num_episodes": 3360,
+    "mean_reward": 507.16173345565795,
+    "mean_length": 2757.46,
+    "loss": 65.15730285644531,
+    "sps": 265.8172803900086
+  },
+  {
+    "update": 2290,
+    "global_step": 9379840,
+    "num_episodes": 3363,
+    "mean_reward": 507.4162758731842,
+    "mean_length": 2818.63,
+    "loss": 70.9657211303711,
+    "sps": 353.8588588797036
+  },
+  {
+    "update": 2295,
+    "global_step": 9400320,
+    "num_episodes": 3368,
+    "mean_reward": 551.9985440444947,
+    "mean_length": 2959.32,
+    "loss": 5.74236536026001,
+    "sps": 340.8187159543217
+  },
+  {
+    "update": 2300,
+    "global_step": 9420800,
+    "num_episodes": 3377,
+    "mean_reward": 564.7782349967956,
+    "mean_length": 3130.89,
+    "loss": 5.68098783493042,
+    "sps": 383.2875448626093
+  },
+  {
+    "update": 2305,
+    "global_step": 9441280,
+    "num_episodes": 3390,
+    "mean_reward": 594.4548657274246,
+    "mean_length": 3136.03,
+    "loss": 178.03961181640625,
+    "sps": 278.82986749605175
+  },
+  {
+    "update": 2310,
+    "global_step": 9461760,
+    "num_episodes": 3391,
+    "mean_reward": 595.1103666639328,
+    "mean_length": 3234.65,
+    "loss": 73.16796112060547,
+    "sps": 386.1064879014476
+  },
+  {
+    "update": 2315,
+    "global_step": 9482240,
+    "num_episodes": 3397,
+    "mean_reward": 560.5055163288116,
+    "mean_length": 3213.23,
+    "loss": 1437.6278076171875,
+    "sps": 198.45749967938127
+  },
+  {
+    "update": 2320,
+    "global_step": 9502720,
+    "num_episodes": 3411,
+    "mean_reward": 401.03073542118074,
+    "mean_length": 2986.15,
+    "loss": 3.3738672733306885,
+    "sps": 1083.0313220969133
+  },
+  {
+    "update": 2325,
+    "global_step": 9523200,
+    "num_episodes": 3412,
+    "mean_reward": 413.53732123851773,
+    "mean_length": 3083.93,
+    "loss": 21.815982818603516,
+    "sps": 659.8576311368523
+  },
+  {
+    "update": 2330,
+    "global_step": 9543680,
+    "num_episodes": 3413,
+    "mean_reward": 407.9750119304657,
+    "mean_length": 3002.15,
+    "loss": 0.47333452105522156,
+    "sps": 726.1803705441753
+  },
+  {
+    "update": 2335,
+    "global_step": 9564160,
+    "num_episodes": 3420,
+    "mean_reward": 454.97729597568514,
+    "mean_length": 3001.83,
+    "loss": 0.5463234186172485,
+    "sps": 582.7651222934119
+  },
+  {
+    "update": 2340,
+    "global_step": 9584640,
+    "num_episodes": 3454,
+    "mean_reward": 401.5923653173447,
+    "mean_length": 2474.53,
+    "loss": 0.9732112884521484,
+    "sps": 190.63133488811127
+  },
+  {
+    "update": 2345,
+    "global_step": 9605120,
+    "num_episodes": 3475,
+    "mean_reward": 180.83635659217833,
+    "mean_length": 1855.64,
+    "loss": 0.047050729393959045,
+    "sps": 497.0424644143239
+  },
+  {
+    "update": 2350,
+    "global_step": 9625600,
+    "num_episodes": 3482,
+    "mean_reward": 181.7909957742691,
+    "mean_length": 1925.71,
+    "loss": 1.1374897956848145,
+    "sps": 853.7958489004412
+  },
+  {
+    "update": 2355,
+    "global_step": 9646080,
+    "num_episodes": 3489,
+    "mean_reward": 158.73119886875153,
+    "mean_length": 1914.03,
+    "loss": 1.258785605430603,
+    "sps": 551.9084865795908
+  },
+  {
+    "update": 2360,
+    "global_step": 9666560,
+    "num_episodes": 3502,
+    "mean_reward": 98.6479220199585,
+    "mean_length": 1635.16,
+    "loss": 0.43499165773391724,
+    "sps": 419.3776501549414
+  },
+  {
+    "update": 2365,
+    "global_step": 9687040,
+    "num_episodes": 3515,
+    "mean_reward": 66.50108590602875,
+    "mean_length": 1518.13,
+    "loss": 166.75491333007812,
+    "sps": 281.0402931910567
+  },
+  {
+    "update": 2370,
+    "global_step": 9707520,
+    "num_episodes": 3528,
+    "mean_reward": 33.59190697193146,
+    "mean_length": 1331.87,
+    "loss": 274.23516845703125,
+    "sps": 287.97774743130026
+  },
+  {
+    "update": 2375,
+    "global_step": 9728000,
+    "num_episodes": 3538,
+    "mean_reward": 23.649227323532106,
+    "mean_length": 1328.19,
+    "loss": 24.9585018157959,
+    "sps": 656.234816203577
+  },
+  {
+    "update": 2380,
+    "global_step": 9748480,
+    "num_episodes": 3539,
+    "mean_reward": 24.15320231437683,
+    "mean_length": 1427.05,
+    "loss": 13.851099967956543,
+    "sps": 840.3197888208443
+  },
+  {
+    "update": 2385,
+    "global_step": 9768960,
+    "num_episodes": 3553,
+    "mean_reward": 25.30362526893616,
+    "mean_length": 1552.4,
+    "loss": -0.07959967106580734,
+    "sps": 348.5790453337399
+  },
+  {
+    "update": 2390,
+    "global_step": 9789440,
+    "num_episodes": 3575,
+    "mean_reward": 43.762970843315124,
+    "mean_length": 1834.94,
+    "loss": 256.5601806640625,
+    "sps": 314.0190386184578
+  },
+  {
+    "update": 2395,
+    "global_step": 9809920,
+    "num_episodes": 3584,
+    "mean_reward": 41.22667598724365,
+    "mean_length": 1832.99,
+    "loss": 37.159915924072266,
+    "sps": 261.015177859497
+  },
+  {
+    "update": 2400,
+    "global_step": 9830400,
+    "num_episodes": 3593,
+    "mean_reward": 56.500927686691284,
+    "mean_length": 1809.72,
+    "loss": 949.2464599609375,
+    "sps": 236.54852291533763
+  },
+  {
+    "update": 2405,
+    "global_step": 9850880,
+    "num_episodes": 3599,
+    "mean_reward": 105.25954935073852,
+    "mean_length": 1813.21,
+    "loss": 1035.45751953125,
+    "sps": 654.1456293848414
+  },
+  {
+    "update": 2410,
+    "global_step": 9871360,
+    "num_episodes": 3608,
+    "mean_reward": 174.8251446390152,
+    "mean_length": 1969.27,
+    "loss": 12.137419700622559,
+    "sps": 374.5700328217855
+  },
+  {
+    "update": 2415,
+    "global_step": 9891840,
+    "num_episodes": 3613,
+    "mean_reward": 174.6033950281143,
+    "mean_length": 2093.28,
+    "loss": 470.9022216796875,
+    "sps": 829.3315456709553
+  },
+  {
+    "update": 2420,
+    "global_step": 9912320,
+    "num_episodes": 3614,
+    "mean_reward": 175.19530655384062,
+    "mean_length": 2188.0,
+    "loss": 7.125182628631592,
+    "sps": 1392.3792980312514
+  },
+  {
+    "update": 2425,
+    "global_step": 9932800,
+    "num_episodes": 3616,
+    "mean_reward": 175.76038108348845,
+    "mean_length": 2285.88,
+    "loss": 11.183570861816406,
+    "sps": 303.2125684013846
+  },
+  {
+    "update": 2430,
+    "global_step": 9953280,
+    "num_episodes": 3625,
+    "mean_reward": 245.62015884399415,
+    "mean_length": 2481.25,
+    "loss": 72.24958038330078,
+    "sps": 223.82439359999108
+  },
+  {
+    "update": 2435,
+    "global_step": 9973760,
+    "num_episodes": 3640,
+    "mean_reward": 12.841221110026042,
+    "mean_length": 242.8,
+    "loss": 39.742530822753906,
+    "sps": 444.8952550886448
+  },
+  {
+    "update": 2440,
+    "global_step": 9994240,
+    "num_episodes": 3640,
+    "mean_reward": 12.841221110026042,
+    "mean_length": 242.8,
+    "loss": 0.29037195444107056,
+    "sps": 1069.6427524147307
+  },
+  {
+    "update": 2445,
+    "global_step": 10014720,
+    "num_episodes": 3640,
+    "mean_reward": 12.841221110026042,
+    "mean_length": 242.8,
+    "loss": 3.3830156326293945,
+    "sps": 1318.7931203919134
+  },
+  {
+    "update": 2450,
+    "global_step": 10035200,
+    "num_episodes": 3644,
+    "mean_reward": 91.08290822882401,
+    "mean_length": 2296.9473684210525,
+    "loss": 20.516225814819336,
+    "sps": 481.54488129169596
+  },
+  {
+    "update": 2455,
+    "global_step": 10055680,
+    "num_episodes": 3658,
+    "mean_reward": 124.92099246111783,
+    "mean_length": 2621.4848484848485,
+    "loss": 150.3678436279297,
+    "sps": 253.02333051932328
+  },
+  {
+    "update": 2460,
+    "global_step": 10076160,
+    "num_episodes": 3665,
+    "mean_reward": 104.92156012058258,
+    "mean_length": 2402.575,
+    "loss": 43.09859085083008,
+    "sps": 444.9419436729771
+  },
+  {
+    "update": 2465,
+    "global_step": 10096640,
+    "num_episodes": 3668,
+    "mean_reward": 99.1124037143796,
+    "mean_length": 2358.6744186046512,
+    "loss": 30.914440155029297,
+    "sps": 564.239378309882
+  },
+  {
+    "update": 2470,
+    "global_step": 10117120,
+    "num_episodes": 3674,
+    "mean_reward": 212.01101287530392,
+    "mean_length": 2689.183673469388,
+    "loss": 519.401611328125,
+    "sps": 400.7113960553138
+  },
+  {
+    "update": 2475,
+    "global_step": 10137600,
+    "num_episodes": 3685,
+    "mean_reward": 246.73736356099445,
+    "mean_length": 2406.016666666667,
+    "loss": 18.674598693847656,
+    "sps": 645.3387661404473
+  },
+  {
+    "update": 2480,
+    "global_step": 10158080,
+    "num_episodes": 3690,
+    "mean_reward": 254.607934988462,
+    "mean_length": 2687.8153846153846,
+    "loss": 227.44029235839844,
+    "sps": 409.4558222523011
   }
 ]