diff --git a/.gitattributes b/.gitattributes index ef0da8d342aba2f3cff1bdf74da0098629d4d0fa..7c16e036d39f300a6bff409dc043429e731bb313 100644 --- a/.gitattributes +++ b/.gitattributes @@ -39,3 +39,6 @@ wandb/wandb/run-20251002_155441-70dhy5dq/run-70dhy5dq.wandb filter=lfs diff=lfs wandb/wandb/run-20251002_150921-kqbx0cjv/run-kqbx0cjv.wandb filter=lfs diff=lfs merge=lfs -text cleandesk_l1_regression/wandb/wandb/run-20251008_163831-76mxu43t/run-76mxu43t.wandb filter=lfs diff=lfs merge=lfs -text wipe_l1_regression/wandb/wandb/run-20251005_163743-a1znetn8/run-a1znetn8.wandb filter=lfs diff=lfs merge=lfs -text +cleandesk50_flow_matching/wandb/wandb/run-20251008_163834-quokv8gn/run-quokv8gn.wandb filter=lfs diff=lfs merge=lfs -text +cleandesk50_l1_regression/wandb/wandb/run-20251008_163831-fqdwkc8m/run-fqdwkc8m.wandb filter=lfs diff=lfs merge=lfs -text +glue_flow_matching/wandb/wandb/run-20251002_163728-tmwli25x/run-tmwli25x.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_061930-dnrnwv30/files/output.log b/all_flow_matching/glue_best/wandb/wandb/run-20250924_061930-dnrnwv30/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_061930-dnrnwv30/files/wandb-summary.json b/all_flow_matching/glue_best/wandb/wandb/run-20250924_061930-dnrnwv30/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b0a620d0c1047a4dd8a400939b6da246ed8063a7 --- /dev/null +++ b/all_flow_matching/glue_best/wandb/wandb/run-20250924_061930-dnrnwv30/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":0},"_runtime":0} \ No newline at end of file diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_061930-dnrnwv30/logs/debug-core.log b/all_flow_matching/glue_best/wandb/wandb/run-20250924_061930-dnrnwv30/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..abb415e97c864f63c756660d6daf166a034a1cfa --- /dev/null +++ b/all_flow_matching/glue_best/wandb/wandb/run-20250924_061930-dnrnwv30/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2025-09-24T06:19:30.419454454Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmph06ly_es/port-2188820.txt","pid":2188820,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-09-24T06:19:30.420467727Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2188820} +{"time":"2025-09-24T06:19:30.42153553Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2188820-2188995-771080915/socket","Net":"unix"}} +{"time":"2025-09-24T06:19:30.607757412Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-09-24T06:19:30.624290644Z","level":"INFO","msg":"handleInformInit: received","streamId":"dnrnwv30","id":"1(@)"} +{"time":"2025-09-24T06:19:31.778152452Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"dnrnwv30","id":"1(@)"} +{"time":"2025-09-24T06:19:32.042400183Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2025-09-24T06:19:32.042449263Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-09-24T06:19:32.042441673Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2025-09-24T06:19:32.042538224Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2188820-2188995-771080915/socket","Net":"unix"}} +{"time":"2025-09-24T06:19:32.042560085Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2025-09-24T06:19:32.340314533Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2025-09-24T06:19:32.340328313Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2025-09-24T06:19:32.340337613Z","level":"INFO","msg":"server is closed"} diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_061930-dnrnwv30/logs/debug-internal.log b/all_flow_matching/glue_best/wandb/wandb/run-20250924_061930-dnrnwv30/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..598cdc870947297042c66937dc29a6862b2cb8bc --- /dev/null +++ b/all_flow_matching/glue_best/wandb/wandb/run-20250924_061930-dnrnwv30/logs/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2025-09-24T06:19:30.626115817Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-09-24T06:19:31.778111872Z","level":"INFO","msg":"stream: created new stream","id":"dnrnwv30"} +{"time":"2025-09-24T06:19:31.778147652Z","level":"INFO","msg":"stream: started","id":"dnrnwv30"} +{"time":"2025-09-24T06:19:31.778168212Z","level":"INFO","msg":"writer: started","stream_id":"dnrnwv30"} +{"time":"2025-09-24T06:19:31.778194323Z","level":"INFO","msg":"sender: started","stream_id":"dnrnwv30"} +{"time":"2025-09-24T06:19:31.778192543Z","level":"INFO","msg":"handler: started","stream_id":"dnrnwv30"} +{"time":"2025-09-24T06:19:32.023079005Z","level":"ERROR","msg":"HTTP error","status":403,"method":"POST","url":"https://api.wandb.ai/graphql"} +{"time":"2025-09-24T06:19:32.023158306Z","level":"ERROR","msg":"runupserter: failed to init run","error":"returned error 403: {\"data\":{\"upsertBucket\":null},\"errors\":[{\"message\":\"permission denied\",\"path\":[\"upsertBucket\"],\"extensions\":{\"code\":\"PERMISSION_ERROR\"}}]}"} +{"time":"2025-09-24T06:19:32.042439853Z","level":"INFO","msg":"stream: closing","id":"dnrnwv30"} +{"time":"2025-09-24T06:19:32.048537861Z","level":"ERROR","msg":"sender: uploadConfigFile: stream: no run"} +{"time":"2025-09-24T06:19:32.332555964Z","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"} +{"time":"2025-09-24T06:19:32.332627655Z","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404: {\"data\":{\"createRunFiles\":null},\"errors\":[{\"message\":\"run a1-vla-xiaodan/dnrnwv30 not found during createRunFiles\",\"path\":[\"createRunFiles\"]}]}"} +{"time":"2025-09-24T06:19:32.336908289Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-09-24T06:19:32.33694275Z","level":"INFO","msg":"handler: closed","stream_id":"dnrnwv30"} +{"time":"2025-09-24T06:19:32.338990606Z","level":"INFO","msg":"sender: closed","stream_id":"dnrnwv30"} +{"time":"2025-09-24T06:19:32.338998046Z","level":"INFO","msg":"stream: closed","id":"dnrnwv30"} diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_061930-dnrnwv30/logs/debug.log b/all_flow_matching/glue_best/wandb/wandb/run-20250924_061930-dnrnwv30/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_062357-hmmpns57/files/output.log b/all_flow_matching/glue_best/wandb/wandb/run-20250924_062357-hmmpns57/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_062357-hmmpns57/files/wandb-summary.json b/all_flow_matching/glue_best/wandb/wandb/run-20250924_062357-hmmpns57/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b0a620d0c1047a4dd8a400939b6da246ed8063a7 --- /dev/null +++ b/all_flow_matching/glue_best/wandb/wandb/run-20250924_062357-hmmpns57/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":0},"_runtime":0} \ No newline at end of file diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_062357-hmmpns57/logs/debug-core.log b/all_flow_matching/glue_best/wandb/wandb/run-20250924_062357-hmmpns57/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..a76eb4ade59ad6a8bf9b41caf16c81fd021d06fe --- /dev/null +++ b/all_flow_matching/glue_best/wandb/wandb/run-20250924_062357-hmmpns57/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2025-09-24T06:23:57.33371267Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp7mdgm6nn/port-2190130.txt","pid":2190130,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-09-24T06:23:57.334085295Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2190130} +{"time":"2025-09-24T06:23:57.334062305Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2190130-2190294-2704513851/socket","Net":"unix"}} +{"time":"2025-09-24T06:23:57.511386371Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-09-24T06:23:57.517858166Z","level":"INFO","msg":"handleInformInit: received","streamId":"hmmpns57","id":"1(@)"} +{"time":"2025-09-24T06:23:58.547393843Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"hmmpns57","id":"1(@)"} +{"time":"2025-09-24T06:23:58.805591645Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2025-09-24T06:23:58.805637046Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2025-09-24T06:23:58.805655116Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-09-24T06:23:58.805681387Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2025-09-24T06:23:58.805837029Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2190130-2190294-2704513851/socket","Net":"unix"}} +{"time":"2025-09-24T06:23:59.098817455Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2025-09-24T06:23:59.098836546Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2025-09-24T06:23:59.098846216Z","level":"INFO","msg":"server is closed"} diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_062357-hmmpns57/logs/debug-internal.log b/all_flow_matching/glue_best/wandb/wandb/run-20250924_062357-hmmpns57/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d939e99ed050854fb7e2674200e0228b0a98b417 --- /dev/null +++ b/all_flow_matching/glue_best/wandb/wandb/run-20250924_062357-hmmpns57/logs/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2025-09-24T06:23:57.51970738Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-09-24T06:23:58.547344712Z","level":"INFO","msg":"stream: created new stream","id":"hmmpns57"} +{"time":"2025-09-24T06:23:58.547388123Z","level":"INFO","msg":"stream: started","id":"hmmpns57"} +{"time":"2025-09-24T06:23:58.547412053Z","level":"INFO","msg":"writer: started","stream_id":"hmmpns57"} +{"time":"2025-09-24T06:23:58.547423083Z","level":"INFO","msg":"handler: started","stream_id":"hmmpns57"} +{"time":"2025-09-24T06:23:58.547448154Z","level":"INFO","msg":"sender: started","stream_id":"hmmpns57"} +{"time":"2025-09-24T06:23:58.797771793Z","level":"ERROR","msg":"HTTP error","status":403,"method":"POST","url":"https://api.wandb.ai/graphql"} +{"time":"2025-09-24T06:23:58.797893015Z","level":"ERROR","msg":"runupserter: failed to init run","error":"returned error 403: {\"data\":{\"upsertBucket\":null},\"errors\":[{\"message\":\"permission denied\",\"path\":[\"upsertBucket\"],\"extensions\":{\"code\":\"PERMISSION_ERROR\"}}]}"} +{"time":"2025-09-24T06:23:58.805654996Z","level":"INFO","msg":"stream: closing","id":"hmmpns57"} +{"time":"2025-09-24T06:23:58.809349995Z","level":"ERROR","msg":"sender: uploadConfigFile: stream: no run"} +{"time":"2025-09-24T06:23:59.095800216Z","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"} +{"time":"2025-09-24T06:23:59.095886847Z","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404: {\"data\":{\"createRunFiles\":null},\"errors\":[{\"message\":\"run a1-vla-xiaodan/hmmpns57 not found during createRunFiles\",\"path\":[\"createRunFiles\"]}]}"} +{"time":"2025-09-24T06:23:59.096209701Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-09-24T06:23:59.096244302Z","level":"INFO","msg":"handler: closed","stream_id":"hmmpns57"} +{"time":"2025-09-24T06:23:59.097755731Z","level":"INFO","msg":"sender: closed","stream_id":"hmmpns57"} +{"time":"2025-09-24T06:23:59.097762801Z","level":"INFO","msg":"stream: closed","id":"hmmpns57"} diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_062357-hmmpns57/logs/debug.log b/all_flow_matching/glue_best/wandb/wandb/run-20250924_062357-hmmpns57/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/files/requirements.txt b/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..9cf6722896b8682405e32a4100d835a5be0f3ae2 --- /dev/null +++ b/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/files/requirements.txt @@ -0,0 +1,283 @@ +ai2-molmo==0.0.0 +astunparse==1.6.3 +flatbuffers==25.2.10 +gast==0.6.0 +google-pasta==0.2.0 +h5py==3.14.0 +libclang==18.1.1 +Markdown==3.9 +namex==0.1.0 +opt_einsum==3.4.0 +optree==0.17.0 +tensorboard-data-server==0.7.2 +tensorflow-io-gcs-filesystem==0.37.1 +tensorflow-rocm==2.16.2 +termcolor==3.1.0 +Werkzeug==3.1.3 +Brotli==1.1.0 +Farama-Notifications==0.0.4 +MarkupSafe==2.1.5 +PyYAML==6.0.2 +absl-py==2.3.1 +accelerate==1.10.1 +ai2-molmo==0.0.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +antlr4-python3-runtime==4.9.3 +anyio==4.10.0 +array_record==0.8.1 +async-timeout==5.0.1 +attrs==25.3.0 +av==15.1.0 +backports.tarfile==1.2.0 +beaker-gantry==3.2.0 +beaker-py==2.5.0 +black==23.12.1 +boltons==25.0.0 +boto3==1.40.33 +botocore==1.40.33 +build==1.3.0 +cached_path==1.7.3 +cached-property==2.0.1 +cachetools==5.5.2 +certifi==2025.8.3 +cffi==2.0.0 +charset-normalizer==3.4.3 +click==8.2.1 +click-help-colors==0.9.4 +click-option-group==0.5.7 +cloudpickle==3.1.1 +cmake==4.1.0 +contourpy==1.3.2 +cryptography==46.0.1 +cycler==0.12.1 +dataclass-extensions==0.2.3 +datasets==3.6.0 +decorator==5.2.1 +deepdiff==8.6.1 +diffusers==0.35.1 +dill==0.3.8 +distro==1.9.0 +dlimp==0.0.1 +dm-tree==0.1.9 +docutils==0.22.1 +draccus==0.10.0 +editdistance==0.8.1 +einops==0.8.1 +einops-exts==0.0.4 +etils==1.13.0 +evdev==1.9.2 +exceptiongroup==1.3.0 +face==24.0.0 +fastapi==0.116.2 +ffmpy==0.6.1 +fiddle==0.3.0 +filelock==3.13.1 +fonttools==4.60.0 +frozenlist==1.7.0 +fsspec==2023.9.2 +ftfy==6.3.1 +gcsfs==2023.9.2 +gitdb==4.0.12 +GitPython==3.1.45 +glom==24.11.0 +google-api-core==2.25.1 +google-auth==2.40.3 +google-auth-oauthlib==1.2.2 +google-cloud-core==2.4.3 +google-cloud-storage==2.19.0 +google-crc32c==1.7.1 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +gradio==5.46.0 +gradio_client==1.13.0 +graphviz==0.21 +groovy==0.1.2 +grpcio==1.75.0 +gymnasium==0.29.1 +h11==0.16.0 +hf_transfer==0.1.9 +hf-xet==1.1.10 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==0.35.0 +id==1.5.0 +idna==3.10 +imageio==2.37.0 +imageio-ffmpeg==0.6.0 +importlib_metadata==8.7.0 +importlib_resources==6.5.2 +iniconfig==2.1.0 +inquirerpy==0.3.4 +isort==5.12.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.3.0 +jeepney==0.9.0 +Jinja2==3.1.4 +jiter==0.11.0 +jmespath==1.0.1 +joblib==1.5.2 +jsonlines==4.0.0 +keras==2.15.0 +keyring==25.6.0 +kiwisolver==1.4.9 +latex2sympy2_extended==1.10.2 +lerobot==0.3.4 +Levenshtein==0.27.1 +libcst==1.8.4 +lightning-utilities==0.15.2 +markdown-it-py==4.0.0 +math-verify==0.8.0 +matplotlib==3.10.6 +mdurl==0.1.2 +mergedeep==1.3.4 +ml-dtypes==0.2.0 +ml_dtypes==0.5.3 +more-itertools==10.8.0 +mpmath==1.3.0 +msgspec==0.19.0 +multidict==6.6.4 +multiprocess==0.70.16 +mypy==1.3.0 +mypy_extensions==1.1.0 +necessary==0.4.3 +networkx==3.3 +nh3==0.3.0 +nltk==3.9.1 +numpy==1.26.4 +numpy==2.2.6 +oauthlib==3.3.1 +omegaconf==2.3.0 +openai==1.108.0 +opencv-python-headless==4.12.0.88 +OpenEXR==3.4.0 +orderly-set==5.5.0 +orjson==3.11.3 +packaging==25.0 +pandas==2.3.2 +pathspec==0.12.1 +petname==2.6 +pfzy==0.3.4 +pillow==11.0.0 +pip==25.2 +platformdirs==4.4.0 +pluggy==1.6.0 +promise==2.3 +prompt_toolkit==3.0.52 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==4.21.12 +protobuf==6.32.1 +psutil==7.1.0 +pyarrow==21.0.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.23 +pydantic==2.11.9 +pydantic_core==2.33.2 +pydub==0.25.1 +Pygments==2.19.2 +pynput==1.8.1 +pyparsing==3.2.4 +pyproject_hooks==1.2.0 +pyserial==3.5 +pytest==8.4.2 +pytest-sphinx==0.6.3 +python-dateutil==2.9.0.post0 +python-Levenshtein==0.27.1 +python-multipart==0.0.20 +python-xlib==0.33 +pytorch-triton-rocm==3.4.0 +pytz==2025.2 +pyyaml-include==1.4.1 +RapidFuzz==3.14.1 +readme_renderer==44.0 +regex==2025.9.1 +requests==2.32.5 +requests-oauthlib==2.0.0 +requests-toolbelt==1.0.0 +requirements-parser==0.13.0 +rerun-sdk==0.22.1 +rfc3986==2.0.0 +rich==13.9.4 +rsa==4.9.1 +ruff==0.13.0 +s3transfer==0.14.0 +safehttpx==0.1.6 +safetensors==0.6.2 +scikit-learn==1.7.2 +scipy==1.15.3 +SecretStorage==3.4.0 +semantic-version==2.10.0 +sentencepiece==0.2.1 +sentry-sdk==2.38.0 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.17.0 +smart_open==7.3.1 +smashed==0.21.5 +smmap==5.0.2 +sniffio==1.3.1 +starlette==0.48.0 +sympy==1.13.3 +tensorboard==2.15.2 +tensorboard==2.19.0 +tensorflow==2.15.0 +tensorflow-addons==0.23.0 +tensorflow-datasets==4.9.3 +tensorflow-estimator==2.15.0 +tensorflow-graphics==2021.12.3 +tensorflow-metadata==1.17.2 +threadpoolctl==3.6.0 +timm==1.0.19 +tokenizers==0.22.0 +toml==0.10.2 +tomli==2.2.1 +tomlkit==0.13.3 +torch==2.8.0+rocm6.4 +torchcodec==0.5 +torchmetrics==1.8.2 +torchvision==0.23.0+rocm6.4 +tqdm==4.67.1 +transformers==4.56.1 +trimesh==4.8.2 +trouting==0.3.3 +twine==6.2.0 +typeguard==2.13.3 +typer==0.17.4 +typing_extensions==4.15.0 +typing-inspect==0.9.0 +typing-inspection==0.4.1 +tzdata==2025.2 +urllib3==2.5.0 +uvicorn==0.35.0 +wandb==0.21.4 +wcwidth==0.2.13 +websockets==15.0.1 +wheel==0.45.1 +wrapt==1.14.2 +xxhash==3.5.0 +yarl==1.20.1 +zipp==3.23.0 +lerobot==0.3.4 +minLoRA==0.1.0 +autocommand==2.2.2 +backports.tarfile==1.2.0 +importlib_metadata==8.0.0 +inflect==7.3.1 +jaraco.collections==5.1.0 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +jaraco.text==3.12.1 +more-itertools==10.3.0 +packaging==24.2 +platformdirs==4.2.2 +tomli==2.0.1 +typeguard==4.3.0 +typing_extensions==4.12.2 +wheel==0.45.1 +zipp==3.19.2 diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/files/wandb-metadata.json b/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f4de8982b499ec84be09830d68712bdadb7dad8b --- /dev/null +++ b/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/files/wandb-metadata.json @@ -0,0 +1,204 @@ +{ + "os": "Linux-5.15.0-140-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.18", + "startedAt": "2025-09-24T06:31:28.005264Z", + "args": [ + "qwen2_7b", + "save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt", + "--vision_backbone", + "openai", + "--action_head", + "flow_matching", + "--seq_len", + "768", + "--lora_rank", + "32", + "--lora_llm", + "--checkpoint", + "/vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924", + "--device_train_microbatch_size", + "22", + "--global_batch_size", + "176", + "--dataset", + "vla_dataset_realworld", + "--llm_learning_rate", + "5e-5", + "--wandb_entity", + "henryeap", + "--wandb_project", + "a1-realworld", + "--wandb_run_name", + "glue", + "--save_overwrite" + ], + "program": "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py", + "codePath": "launch_scripts/train_vla.py", + "codePathLocal": "launch_scripts/train_vla.py", + "git": { + "remote": "https://github.com/Spatialtemporal-AI/A1.git", + "commit": "c13f2763af61e0d729a8b5ab4bdefc512205bcc5" + }, + "email": "ihenrykwok@outlook.com", + "root": "/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/wandb", + "host": "auh7-1b-gpu-188", + "executable": "/vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10", + "cpu_count": 64, + "cpu_count_logical": 128, + "gpu": "Instinct MI210", + "gpu_count": 8, + "disk": { + "/": { + "total": "470343073792", + "used": "51147874304" + } + }, + "memory": { + "total": "2434606952448" + }, + "gpu_amd": [ + { + "id": "5", + "uniqueId": "0x137c9ede1bb1518e", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "7", + "uniqueId": "0x21a2e88d06c419dc", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "2", + "uniqueId": "0x399226d2b2bfa544", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "0", + "uniqueId": "0x3558c3014c813fdb", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "3", + "uniqueId": "0xf61ec17df11883bd", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "1", + "uniqueId": "0x9b5c1c302c8129f8", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "6", + "uniqueId": "0xfa8b85a4625b04f", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "4", + "uniqueId": "0xa515afd8ced1d39d", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + } + ], + "slurm": { + "cluster_name": "ai-04r", + "conf": "/etc/slurm/slurm.conf", + "cpus_on_node": "128", + "gpus_on_node": "8", + "gtids": "0", + "job_account": "faculty-acc", + "job_cpus_per_node": "128", + "job_end_time": "1758954648", + "job_gid": "2000", + "job_gpus": "0,1,2,3,4,5,6,7", + "job_id": "1605", + "job_name": "realworld_mh", + "job_nodelist": "auh7-1b-gpu-188", + "job_num_nodes": "1", + "job_partition": "faculty", + "job_qos": "xdqos", + "job_start_time": "1758695448", + "job_uid": "2013", + "job_user": "xiaodan", + "jobid": "1605", + "localid": "0", + "nnodes": "1", + "nodeid": "0", + "nodelist": "auh7-1b-gpu-188", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "oom_kill_step": "0", + "prio_process": "0", + "procid": "0", + "submit_dir": "/vast/users/xiaodan/zhangjian/A1/launch_scripts", + "submit_host": "auh-1b-cpu-login-001", + "task_pid": "2191329", + "tasks_per_node": "1", + "topology_addr": "auh7-1b-gpu-188", + "topology_addr_pattern": "node" + }, + "writerId": "o421nvn5u6ub6ruog26gg83x0g2lmgbt" +} \ No newline at end of file diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/files/wandb-summary.json b/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5031a3fdb5637a89ebf85a0681ed18b6e711fde3 --- /dev/null +++ b/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":2,"_wandb":{"runtime":2}} \ No newline at end of file diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/logs/debug-core.log b/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..5377af8c52f2caba56e67529cbec6c88fb4167e4 --- /dev/null +++ b/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2025-09-24T06:31:28.064738272Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpt5x3_6pq/port-2191415.txt","pid":2191415,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-09-24T06:31:28.066318992Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2191415} +{"time":"2025-09-24T06:31:28.066291612Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2191415-2191580-2229050380/socket","Net":"unix"}} +{"time":"2025-09-24T06:31:28.241348786Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-09-24T06:31:28.248471068Z","level":"INFO","msg":"handleInformInit: received","streamId":"wtatxotn","id":"1(@)"} +{"time":"2025-09-24T06:31:29.271764603Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"wtatxotn","id":"1(@)"} +{"time":"2025-09-24T06:31:32.368379213Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2025-09-24T06:31:32.368730117Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-09-24T06:31:32.368724477Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2025-09-24T06:31:32.368772508Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2025-09-24T06:31:32.368826649Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2191415-2191580-2229050380/socket","Net":"unix"}} +{"time":"2025-09-24T06:31:33.781258776Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2025-09-24T06:31:33.781274636Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2025-09-24T06:31:33.781290036Z","level":"INFO","msg":"server is closed"} diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/logs/debug-internal.log b/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..738ef0a843121cfccec27a8ddea3d926c89a1639 --- /dev/null +++ b/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2025-09-24T06:31:28.251016151Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-09-24T06:31:29.271706933Z","level":"INFO","msg":"stream: created new stream","id":"wtatxotn"} +{"time":"2025-09-24T06:31:29.271758583Z","level":"INFO","msg":"stream: started","id":"wtatxotn"} +{"time":"2025-09-24T06:31:29.271781634Z","level":"INFO","msg":"handler: started","stream_id":"wtatxotn"} +{"time":"2025-09-24T06:31:29.271778354Z","level":"INFO","msg":"writer: started","stream_id":"wtatxotn"} +{"time":"2025-09-24T06:31:29.271793104Z","level":"INFO","msg":"sender: started","stream_id":"wtatxotn"} +{"time":"2025-09-24T06:31:32.368726307Z","level":"INFO","msg":"stream: closing","id":"wtatxotn"} +{"time":"2025-09-24T06:31:33.421915218Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-09-24T06:31:33.778168056Z","level":"INFO","msg":"handler: closed","stream_id":"wtatxotn"} +{"time":"2025-09-24T06:31:33.780128001Z","level":"INFO","msg":"sender: closed","stream_id":"wtatxotn"} +{"time":"2025-09-24T06:31:33.780158132Z","level":"INFO","msg":"stream: closed","id":"wtatxotn"} diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/logs/debug.log b/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..34df0caf6bda80be226f840803ee2d988543ba47 --- /dev/null +++ b/all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/logs/debug.log @@ -0,0 +1 @@ +2025-09-24 06:31:32,368 INFO wandb-AsyncioManager-main:2191415 [service_client.py:_forward_responses():84] Reached EOF. diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_063642-6tj2c8pr/logs/debug-core.log b/all_flow_matching/glue_best/wandb/wandb/run-20250924_063642-6tj2c8pr/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..41c967b3731cb1d3af00228b4d4675471d05c77f --- /dev/null +++ b/all_flow_matching/glue_best/wandb/wandb/run-20250924_063642-6tj2c8pr/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2025-09-24T06:36:42.858452895Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmphjacd0ae/port-2192743.txt","pid":2192743,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-09-24T06:36:42.859464168Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2192743} +{"time":"2025-09-24T06:36:42.859402097Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2192743-2192908-1607882430/socket","Net":"unix"}} +{"time":"2025-09-24T06:36:43.043042785Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-09-24T06:36:43.049816992Z","level":"INFO","msg":"handleInformInit: received","streamId":"6tj2c8pr","id":"1(@)"} +{"time":"2025-09-24T06:36:44.180372147Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"6tj2c8pr","id":"1(@)"} +{"time":"2025-09-24T06:36:47.096248789Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2025-09-24T06:36:47.096289969Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2025-09-24T06:36:47.09631223Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-09-24T06:36:47.09637208Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2025-09-24T06:36:47.096556563Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2192743-2192908-1607882430/socket","Net":"unix"}} +{"time":"2025-09-24T06:36:48.708711296Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2025-09-24T06:36:48.709060661Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2025-09-24T06:36:48.709072391Z","level":"INFO","msg":"server is closed"} diff --git a/all_flow_matching/glue_best/wandb/wandb/run-20250924_063642-6tj2c8pr/logs/debug.log b/all_flow_matching/glue_best/wandb/wandb/run-20250924_063642-6tj2c8pr/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c41ad0c940510218fb8965503de6ea362f29f61e --- /dev/null +++ b/all_flow_matching/glue_best/wandb/wandb/run-20250924_063642-6tj2c8pr/logs/debug.log @@ -0,0 +1 @@ +2025-09-24 06:36:47,096 INFO wandb-AsyncioManager-main:2192743 [service_client.py:_forward_responses():84] Reached EOF. diff --git a/all_l1/step8000-action-head/metadata.pt b/all_l1/step8000-action-head/metadata.pt new file mode 100644 index 0000000000000000000000000000000000000000..55cffed57dceceba7a9dd66cf39e86f1cb5a7883 --- /dev/null +++ b/all_l1/step8000-action-head/metadata.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:883bb1160e5c5ece44d882f29247664bba7a98ff980d816e4b7684797ae2d601 +size 1331 diff --git a/all_l1/step8500-action-head/metadata.pt b/all_l1/step8500-action-head/metadata.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c85de5a66f2e39ba57b51b6d60f3eaf507938f0 --- /dev/null +++ b/all_l1/step8500-action-head/metadata.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80287bb60307f9ac09aacfce4b1df8f509d0f9c0571939e3a38e6f558e2a874c +size 1331 diff --git a/all_l1/step8500-unsharded/config.yaml b/all_l1/step8500-unsharded/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..405080ae17ead9c64a4104eb1a37b6cd91d9c6bd --- /dev/null +++ b/all_l1/step8500-unsharded/config.yaml @@ -0,0 +1,322 @@ +run_name: realworld_20250930_131219 +seed: 6198 +epoch: null +dry_run: false +model: + d_model: 3584 + n_heads: 28 + n_kv_heads: 4 + qkv_bias: true + clip_qkv: null + n_layers: 28 + mlp_ratio: 4 + mlp_hidden_size: 37888 + activation_type: swiglu + block_type: sequential + block_group_size: 1 + rope: true + rope_full_precision: true + rope_theta: 1000000.0 + vision_backbone: + image_model_type: openai + image_default_input_size: + - 336 + - 336 + image_patch_size: 14 + image_pos_patch_size: 14 + image_emb_dim: 1024 + image_num_heads: 16 + image_num_key_value_heads: 16 + image_num_layers: 23 + image_head_dim: 64 + image_mlp_dim: 4096 + image_mlp_activations: quick_gelu + image_dropout_rate: 0.0 + image_num_pos: 577 + image_norm_eps: 1.0e-05 + attention_dropout: 0.0 + residual_dropout: 0.0 + initializer_range: 0.02 + fsdp_wrap: false + resize_mode: default + vit_load_path: /weka/oe-training-default/mm-olmo/pretrained_image_encoders/vit-l-14-336.pt + llm_load_path: /weka/oe-training-default/mm-olmo/pretrained_llms/qwen2-7b.pt + low_cpu_fsdp: true + attention_type: sdpa + float32_attention: true + attention_dropout: 0.0 + attention_layer_norm: false + residual_dropout: 0.1 + response_residual_dropout: 0.0 + embedding_dropout: 0.0 + layer_norm_type: rms + layer_norm_with_affine: true + layer_norm_eps: 1.0e-06 + attention_layer_norm_with_affine: true + max_sequence_length: 4096 + max_position_embeddings: null + include_bias: false + bias_for_layer_norm: null + scale_logits: false + vocab_size: 152064 + embedding_size: 152064 + ff_out_size: 0 + additional_vocab_size: 128 + new_embedding_init_range: 0.02 + weight_tying: false + init_device: null + init_fn: normal + init_std: 0.02 + init_cutoff_factor: null + norm_after: false + precision: amp_bf16 + max_crops: 12 + crop_mode: overlap-and-resize-c2 + use_col_tokens: true + prompt_type: uber_model + system_prompt_kind: demo_or_style + message_formatting: role + always_start_with_space: true + multi_annotation_weighting: root_subsegments + default_inference_len: 65 + overlap_margins: + - 4 + - 4 + pad_value: 0.0 + image_padding_embed: pad_and_partial_pad + fix_image_padding: true + vit_layers: + - -2 + - -9 + image_pooling_h: 2 + image_pooling_w: 2 + image_pooling_2d: attention_meanq + image_projector: mlp + image_feature_dropout: 0.0 + initializer_range: 0.02 + normalize_input_embeds: false + use_position_ids: true + head_dim: null + action_tokenizer: + identifier: physical-intelligence/fast + tokenizer_dir: null + action_dim: 7 + horizon: 8 + tokenizer: + identifier: Qwen/Qwen2-7B + tokenizer_dir: null + pad_tokenizer: true + moe_num_experts: 8 + moe_top_k: 2 + moe_mlp_impl: sparse + moe_log_expert_assignment: false + moe_shared_expert: false + moe_lbl_in_fp32: false + moe_interleave: false + moe_loss_weight: 0.1 + moe_zloss_weight: null + moe_dropless: true + moe_capacity_factor: 1.25 + action_head: l1_regression + num_diffusion_steps: 1000 + num_diffusion_inference_steps: 30 + use_proprio: true + action_head_dit_hidden_size: 1152 + action_head_dit_depth: 28 + action_head_dit_num_heads: 16 + llm_causal_attention: false + action_use_left_eef: true + action_use_mobile_base: false +allow_resume: false +ft_llm: true +ft_vit: false +ft_connector: false +ft_embedding: lm_head +lora: false +use_lora: true +lora_rank: 8 +lora_llm: false +lora_vit: false +lora_connector: false +early_exit: false +train_exit_random_layer: false +optimizer: + name: adamw + learning_rate: 0.0001 + weight_decay: 0.01 + betas: + - 0.9 + - 0.95 + eps: 1.0e-05 + connector_learning_rate: 0.0002 + vit_learning_rate: 6.0e-06 + llm_learning_rate: 5.0e-05 + connector_weight_decay: 0.0 + vit_weight_decay: 0.0 + llm_weight_decay: 0.0 + connector_betas: + - 0.9 + - 0.95 + vit_betas: + - 0.9 + - 0.95 + llm_betas: + - 0.9 + - 0.95 + connector_eps: 1.0e-06 + vit_eps: 1.0e-06 + llm_eps: 1.0e-06 + metrics_log_interval: 20 +scheduler: + name: multimodal + units: steps + t_warmup: 100 + t_max: null + alpha_f: 0.1 + connector_t_warmup: 200 + vit_t_warmup: 2000 + llm_t_warmup: 2000 + grad_clip_warmup_steps: null + grad_clip_warmup_factor: null + warmup_min_lr: 0.0 +data: + dataset: vla_dataset_realworld + mixture: null + root_size_mixture: null + split: train + seed: 95818 + shuffle_messages: false + pad: to_max + sequence_length: 1600 + shuffle: true + for_inference: false + multi_modal: torch + num_workers: 0 + drop_last: true + pin_memory: true + prefetch_factor: null + persistent_workers: false + timeout: 0 + rlds_dataset_name: libero_4_task_suites_no_noops + rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe + use_wrist_image: true + use_proprio: true + rlds_shuffle_buffer_size: 100000 + rlds_traj_threads: 8 + rlds_read_threads: 8 + lerobot_episode_index_start: null + lerobot_episode_index_end: null +restore_dataloader: true +fast_forward_batches: null +evaluators: +- label: val + data: + dataset: vla_dataset_realworld + mixture: null + root_size_mixture: null + split: validation + seed: null + shuffle_messages: false + pad: to_max + sequence_length: 1600 + shuffle: false + for_inference: false + multi_modal: torch + num_workers: 0 + drop_last: true + pin_memory: true + prefetch_factor: null + persistent_workers: true + timeout: 0 + rlds_dataset_name: libero_4_task_suites_no_noops + rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds + use_wrist_image: true + use_proprio: true + rlds_shuffle_buffer_size: 256000 + rlds_traj_threads: 8 + rlds_read_threads: 8 + lerobot_episode_index_start: 353 + lerobot_episode_index_end: 765 + device_eval_batch_size: null + subset_num_batches: 64 + max_examples: null + max_new_tokens: 448 + mm_evaluator: null + save_dir: null + save_to_checkpoint_dir: false + eval_name: null + skip_if_metrics_cached: true +eval_interval: 0 +inf_eval_interval: -1 +inf_evaluators: [] +save_folder: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1 +remote_save_folder: null +canceled_check_interval: 50 +save_interval: 500 +save_interval_unsharded: 500 +save_interval_ephemeral: null +save_interval_action_head: 500 +save_num_checkpoints_to_keep: 1 +save_num_unsharded_checkpoints_to_keep: 1 +save_num_action_head_checkpoints_to_keep: 2 +save_overwrite: true +force_save_unsharded: false +no_pre_train_checkpoint: true +initial_model_checkpoint: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 +load_model_config: null +checkpoint_dir: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 +load_path: null +load_path_sharded_checkpointer: null +reset_optimizer_state: false +reset_trainer_state: false +save_dataloader_state: false +reset_dataloader_state: false +keep_lr_on_load: true +sharded_checkpointer: torch_legacy +max_duration: 500000 +global_train_batch_size: 126 +device_train_batch_size: 15 +device_train_microbatch_size: 16 +device_eval_batch_size: 4 +eval_subset_num_batches: -1 +eval_on_load: false +device_inf_eval_batch_size: 16 +inf_eval_subset_num_batches: -1 +device_train_grad_accum: 0 +max_grad_norm: 1.0 +multi_component_grad_norm: true +batch_divisor: global_batch +max_grad_norm_ratio: null +precision: amp_bf16 +wandb: + project: a1-realworld + entity: henryeap + group: null + name: realworld_20250930_131219 + tags: + - watching + log_artifacts: false + rank_zero_only: true + log_interval: 1 +speed_monitor: + window_size: 20 + gpu_flops_available: null +console_log_interval: 1 +gen1_gc_interval: 1 +compile: null +fsdp: + use_orig_params: true + sharding_strategy: FULL_SHARD + wrapping_strategy: by_block_and_size + precision: float + hybrid_sharding_num_model_replicas: null +softmax_auxiliary_loss: true +softmax_auxiliary_loss_scale: 0.0001 +time_limit: null +extra_steps_after_cancel: 10 +python_profiling: false +torch_profiling: false +stop_at: 500000 +stop_after: null +activation_checkpointing: whole_layer +fused_loss: null diff --git a/all_l1/step8500-unsharded/lora.pt b/all_l1/step8500-unsharded/lora.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3eb8fc48e155d8229340760896a20ba9fc413e2 --- /dev/null +++ b/all_l1/step8500-unsharded/lora.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b09055f15a54dd092b4dd30833406731057005822da0c55c16231cf2e68f7f6 +size 1243 diff --git a/all_l1/step8500-unsharded/train.pt b/all_l1/step8500-unsharded/train.pt new file mode 100644 index 0000000000000000000000000000000000000000..65bd22ead6b70fff307762423fe00219636633e6 --- /dev/null +++ b/all_l1/step8500-unsharded/train.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a978718ae923b1476f2fe06dd89422aa8d20b5aca459e47e40db6b02d009001 +size 15061 diff --git a/all_l1/wandb/wandb/debug-internal.log b/all_l1/wandb/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f830e5e77134044528636ac1e7a7cedc908690b4 --- /dev/null +++ b/all_l1/wandb/wandb/debug-internal.log @@ -0,0 +1,9 @@ +{"time":"2025-09-30T13:12:50.753807221Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-09-30T13:12:51.777951571Z","level":"INFO","msg":"stream: created new stream","id":"ea1k0g3y"} +{"time":"2025-09-30T13:12:51.778008702Z","level":"INFO","msg":"stream: started","id":"ea1k0g3y"} +{"time":"2025-09-30T13:12:51.778019413Z","level":"INFO","msg":"handler: started","stream_id":"ea1k0g3y"} +{"time":"2025-09-30T13:12:51.778022263Z","level":"INFO","msg":"sender: started","stream_id":"ea1k0g3y"} +{"time":"2025-09-30T13:12:51.778055373Z","level":"INFO","msg":"writer: started","stream_id":"ea1k0g3y"} +{"time":"2025-10-01T17:48:53.058127813Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/henryeap/a1-realworld/ea1k0g3y/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2025-10-02T06:50:26.681725611Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/henryeap/a1-realworld/ea1k0g3y/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2025-10-02T07:20:37.728905409Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} diff --git a/all_l1/wandb/wandb/debug.log b/all_l1/wandb/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/config.yaml b/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..377483e43b069a9167502a9feb1eb44cf6772354 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/config.yaml @@ -0,0 +1,621 @@ +_wandb: + value: + cli_version: 0.21.4 + e: + zpxgtt7ztfwstl3gysdjfozjodgqnvkm: + args: + - qwen2_7b + - save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1 + - --vision_backbone + - openai + - --action_head + - l1_regression + - --seq_len + - "1600" + - --ft_llm + - --checkpoint + - /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 + - --device_train_microbatch_size + - "16" + - --global_batch_size + - "126" + - --dataset + - vla_dataset_realworld + - --llm_learning_rate + - "5e-5" + - --wandb_entity + - henryeap + - --wandb_project + - a1-realworld + - --wandb_run_name + - realworld + - --save_overwrite + codePath: launch_scripts/train_vla.py + codePathLocal: launch_scripts/train_vla.py + cpu_count: 64 + cpu_count_logical: 128 + disk: + /: + total: "470343073792" + used: "50842279936" + email: ihenrykwok@outlook.com + executable: /vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10 + git: + commit: 50cf9fd3143e218eb94104381c16c0482ac52f0d + remote: https://github.com/Spatialtemporal-AI/A1.git + gpu: Instinct MI210 + gpu_amd: + - id: "0" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xc6ed7c5159e83b1" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "5" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xa95e252cd7a6e54e" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "1" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x333c966d5f3b3375" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "7" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x5656afec2788d569" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "6" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xad1bf75f50313455" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "3" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xbabaa83bdb6fe877" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "4" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xdee6d87744a36ae" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "2" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x2a954aa975e59d15" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + gpu_count: 8 + host: auh7-1b-gpu-292 + memory: + total: "2434606923776" + os: Linux-5.15.0-140-generic-x86_64-with-glibc2.35 + program: /vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py + python: CPython 3.10.18 + root: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb + slurm: + cluster_name: ai-04r + conf: /etc/slurm/slurm.conf + cpus_on_node: "128" + gpus_on_node: "8" + gtids: "0" + job_account: faculty-acc + job_cpus_per_node: "128" + job_end_time: "1759424668" + job_gid: "2000" + job_gpus: 0,1,2,3,4,5,6,7 + job_id: "1934" + job_name: mh_realworld + job_nodelist: auh7-1b-gpu-292 + job_num_nodes: "1" + job_partition: faculty + job_qos: xdqos + job_start_time: "1759165468" + job_uid: "2013" + job_user: xiaodan + jobid: "1934" + localid: "0" + nnodes: "1" + nodeid: "0" + nodelist: auh7-1b-gpu-292 + nprocs: "1" + ntasks: "1" + ntasks_per_node: "1" + oom_kill_step: "0" + prio_process: "0" + procid: "0" + submit_dir: /vast/users/xiaodan/zhangjian/A1/launch_scripts + submit_host: auh-1b-cpu-login-001 + task_pid: "1357871" + tasks_per_node: "1" + topology_addr: auh7-1b-gpu-292 + topology_addr_pattern: node + startedAt: "2025-09-29T17:05:12.548535Z" + writerId: zpxgtt7ztfwstl3gysdjfozjodgqnvkm + m: [] + python_version: 3.10.18 + t: + "1": + - 1 + - 2 + - 3 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 63 + - 71 + - 83 + - 95 + - 105 + "2": + - 1 + - 2 + - 3 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 63 + - 71 + - 83 + - 95 + - 105 + "3": + - 2 + - 13 + - 15 + - 16 + - 61 + "4": 3.10.18 + "5": 0.21.4 + "6": 4.56.1 + "10": + - 19 + "12": 0.21.4 + "13": linux-x86_64 +activation_checkpointing: + value: whole_layer +allow_resume: + value: false +batch_divisor: + value: global_batch +canceled_check_interval: + value: 50 +checkpoint_dir: + value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 +compile: + value: null +console_log_interval: + value: 1 +data: + value: + dataset: vla_dataset_realworld + drop_last: true + for_inference: false + lerobot_episode_index_end: null + lerobot_episode_index_start: null + mixture: null + multi_modal: torch + num_workers: 0 + pad: to_max + persistent_workers: false + pin_memory: true + prefetch_factor: null + rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds + rlds_dataset_name: libero_4_task_suites_no_noops + rlds_read_threads: 8 + rlds_shuffle_buffer_size: 100000 + rlds_traj_threads: 8 + root_size_mixture: null + seed: 95818 + sequence_length: 1600 + shuffle: true + shuffle_messages: false + split: train + timeout: 0 + use_proprio: true + use_wrist_image: true +device_eval_batch_size: + value: 4 +device_inf_eval_batch_size: + value: 16 +device_train_batch_size: + value: 15 +device_train_grad_accum: + value: 0 +device_train_microbatch_size: + value: 16 +dry_run: + value: false +early_exit: + value: false +epoch: + value: null +eval_interval: + value: 0 +eval_on_load: + value: false +eval_subset_num_batches: + value: -1 +evaluators: + value: + - data: + dataset: vla_dataset_realworld + drop_last: true + for_inference: false + lerobot_episode_index_end: 765 + lerobot_episode_index_start: 353 + mixture: null + multi_modal: torch + num_workers: 0 + pad: to_max + persistent_workers: true + pin_memory: true + prefetch_factor: null + rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds + rlds_dataset_name: libero_4_task_suites_no_noops + rlds_read_threads: 8 + rlds_shuffle_buffer_size: 256000 + rlds_traj_threads: 8 + root_size_mixture: null + seed: null + sequence_length: 1600 + shuffle: false + shuffle_messages: false + split: validation + timeout: 0 + use_proprio: true + use_wrist_image: true + device_eval_batch_size: null + eval_name: null + label: val + max_examples: null + max_new_tokens: 448 + mm_evaluator: null + save_dir: null + save_to_checkpoint_dir: false + skip_if_metrics_cached: true + subset_num_batches: 64 +extra_steps_after_cancel: + value: 10 +fast_forward_batches: + value: null +force_save_unsharded: + value: false +fsdp: + value: + hybrid_sharding_num_model_replicas: null + precision: float + sharding_strategy: FULL_SHARD + use_orig_params: true + wrapping_strategy: by_block_and_size +ft_connector: + value: false +ft_embedding: + value: lm_head +ft_llm: + value: true +ft_vit: + value: false +fused_loss: + value: null +gen1_gc_interval: + value: 1 +global_train_batch_size: + value: 126 +inf_eval_interval: + value: -1 +inf_eval_subset_num_batches: + value: -1 +inf_evaluators: + value: [] +initial_model_checkpoint: + value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 +keep_lr_on_load: + value: true +load_model_config: + value: null +load_path: + value: null +load_path_sharded_checkpointer: + value: null +lora: + value: false +lora_connector: + value: false +lora_llm: + value: false +lora_rank: + value: 8 +lora_vit: + value: false +max_duration: + value: 500000 +max_grad_norm: + value: 1 +max_grad_norm_ratio: + value: null +model: + value: + action_dim: 7 + action_head: l1_regression + action_head_dit_depth: 28 + action_head_dit_hidden_size: 1152 + action_head_dit_num_heads: 16 + action_tokenizer: + identifier: physical-intelligence/fast + tokenizer_dir: null + action_use_left_eef: false + action_use_mobile_base: false + activation_type: swiglu + additional_vocab_size: 128 + always_start_with_space: true + attention_dropout: 0 + attention_layer_norm: false + attention_layer_norm_with_affine: true + attention_type: sdpa + bias_for_layer_norm: null + block_group_size: 1 + block_type: sequential + clip_qkv: null + crop_mode: overlap-and-resize-c2 + d_model: 3584 + default_inference_len: 65 + embedding_dropout: 0 + embedding_size: 152064 + ff_out_size: 0 + fix_image_padding: true + float32_attention: true + head_dim: null + horizon: 8 + image_feature_dropout: 0 + image_padding_embed: pad_and_partial_pad + image_pooling_2d: attention_meanq + image_pooling_h: 2 + image_pooling_w: 2 + image_projector: mlp + include_bias: false + init_cutoff_factor: null + init_device: null + init_fn: normal + init_std: 0.02 + initializer_range: 0.02 + layer_norm_eps: 1e-06 + layer_norm_type: rms + layer_norm_with_affine: true + llm_causal_attention: false + llm_load_path: /weka/oe-training-default/mm-olmo/pretrained_llms/qwen2-7b.pt + low_cpu_fsdp: true + max_crops: 12 + max_position_embeddings: null + max_sequence_length: 4096 + message_formatting: role + mlp_hidden_size: 37888 + mlp_ratio: 4 + moe_capacity_factor: 1.25 + moe_dropless: true + moe_interleave: false + moe_lbl_in_fp32: false + moe_log_expert_assignment: false + moe_loss_weight: 0.1 + moe_mlp_impl: sparse + moe_num_experts: 8 + moe_shared_expert: false + moe_top_k: 2 + moe_zloss_weight: null + multi_annotation_weighting: root_subsegments + n_heads: 28 + n_kv_heads: 4 + n_layers: 28 + new_embedding_init_range: 0.02 + norm_after: false + normalize_input_embeds: false + num_diffusion_inference_steps: 30 + num_diffusion_steps: 1000 + overlap_margins: + - 4 + - 4 + pad_tokenizer: true + pad_value: 0 + precision: amp_bf16 + prompt_type: uber_model + qkv_bias: true + residual_dropout: 0.1 + response_residual_dropout: 0 + rope: true + rope_full_precision: true + rope_theta: 1e+06 + scale_logits: false + system_prompt_kind: demo_or_style + tokenizer: + identifier: Qwen/Qwen2-7B + tokenizer_dir: null + use_col_tokens: true + use_position_ids: true + use_proprio: true + vision_backbone: + attention_dropout: 0 + fsdp_wrap: false + image_default_input_size: + - 336 + - 336 + image_dropout_rate: 0 + image_emb_dim: 1024 + image_head_dim: 64 + image_mlp_activations: quick_gelu + image_mlp_dim: 4096 + image_model_type: openai + image_norm_eps: 1e-05 + image_num_heads: 16 + image_num_key_value_heads: 16 + image_num_layers: 23 + image_num_pos: 577 + image_patch_size: 14 + image_pos_patch_size: 14 + initializer_range: 0.02 + residual_dropout: 0 + resize_mode: default + vit_layers: + - -2 + - -9 + vit_load_path: /weka/oe-training-default/mm-olmo/pretrained_image_encoders/vit-l-14-336.pt + vocab_size: 152064 + weight_tying: false +multi_component_grad_norm: + value: true +no_pre_train_checkpoint: + value: true +optimizer: + value: + betas: + - 0.9 + - 0.95 + connector_betas: + - 0.9 + - 0.95 + connector_eps: 1e-06 + connector_learning_rate: 0.0002 + connector_weight_decay: 0 + eps: 1e-05 + learning_rate: 0.0001 + llm_betas: + - 0.9 + - 0.95 + llm_eps: 1e-06 + llm_learning_rate: 5e-05 + llm_weight_decay: 0 + metrics_log_interval: 20 + name: adamw + vit_betas: + - 0.9 + - 0.95 + vit_eps: 1e-06 + vit_learning_rate: 6e-06 + vit_weight_decay: 0 + weight_decay: 0.01 +precision: + value: amp_bf16 +python_profiling: + value: false +remote_save_folder: + value: null +reset_dataloader_state: + value: false +reset_optimizer_state: + value: false +reset_trainer_state: + value: false +restore_dataloader: + value: true +run_name: + value: realworld_20250929_170441 +save_dataloader_state: + value: false +save_folder: + value: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1 +save_interval: + value: 500 +save_interval_action_head: + value: 500 +save_interval_ephemeral: + value: null +save_interval_unsharded: + value: 500 +save_num_action_head_checkpoints_to_keep: + value: 2 +save_num_checkpoints_to_keep: + value: 1 +save_num_unsharded_checkpoints_to_keep: + value: 1 +save_overwrite: + value: true +scheduler: + value: + alpha_f: 0.1 + connector_t_warmup: 200 + grad_clip_warmup_factor: null + grad_clip_warmup_steps: null + llm_t_warmup: 2000 + name: multimodal + t_max: null + t_warmup: 100 + units: steps + vit_t_warmup: 2000 + warmup_min_lr: 0 +seed: + value: 6198 +sharded_checkpointer: + value: torch_legacy +softmax_auxiliary_loss: + value: true +softmax_auxiliary_loss_scale: + value: 0.0001 +speed_monitor: + value: + gpu_flops_available: null + window_size: 20 +stop_after: + value: null +stop_at: + value: 500000 +time_limit: + value: null +torch_profiling: + value: false +train_exit_random_layer: + value: false +use_lora: + value: true diff --git a/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/output.log b/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4987ad5c51f161a49f973e2c559f3a5b79bb4226 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/output.log @@ -0,0 +1,122 @@ +wandb: Detected [openai] in use. +wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script. +wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/ +09/29 [17:05:14] WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No warnings.py:109 + device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once + +****** vla_cfg: {'datasets': {'rlds': {'name': None, 'path': None, 'weight': 1.0, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk', 0.1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50', 0.1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser', 0.1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue', 0.1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen', 0.1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote', 0.1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue', 0.1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe', 0.1, 'bounds']], 'open-source-real-world': {'rlds': {'name': 'a1_real_world', 'path': '/vast/users/xiaodan/zhangjian/datasets/OXE', 'weight': 0.1, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [], 'agibot': {'path': None, 'weight': 0.1, 'action_proprio_normalization_type': None}}}, 'model': {'action_head': {'action_dim': 16, 'proprio_dim': 16, 'num_actions_chunk': 8, 'action_tokens_mapping': {'left_end_effector': 8, 'right_end_effector': 8}, 'use_left_eef': True, 'use_mobile_base': False}}} +****** Skip RLDS main; path not found: None +****** start build LeRobot main... +build_tokenizer, cache_dir None tokenizer_dir None +09/29 [17:05:28] INFO | >> Padding tokenizer with 418 tokens tokenizer.py:130 +09/29 [17:05:29] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk +****** length of the dataset: 72641 +09/29 [17:05:37] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50 +****** length of the dataset: 27906 +09/29 [17:05:44] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser +****** length of the dataset: 13441 +09/29 [17:05:48] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue +****** length of the dataset: 10316 +09/29 [17:05:49] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen +****** length of the dataset: 17131 +09/29 [17:05:50] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote +****** length of the dataset: 15765 +09/29 [17:05:51] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue +****** length of the dataset: 90 +09/29 [17:05:52] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe +****** length of the dataset: 18397 +****** Skip RLDS open-source-real-world; mixture 'a1_real_world' not found under: /vast/users/xiaodan/zhangjian/datasets/OXE +****** Expect one of: [] +****** path: None +****** Skip AgiBotWorld-Alpha open-source-real-world; path not found: None +****** After build vla train dataset... +****** iterable_sources: [, , , , , , , ] +****** Before build mixed iterable dataset... +****** Build vla train dataloader successfully! +************************* Build train_dataloader successful! +************************* Before build_inf_evaluators +09/29 [17:05:53] WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No warnings.py:109 + device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once + +************************* Build evaluators successful! +************************* Early exit flags: early_exit=False +************************* Initialize model successful! +***** state_dict_path: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924/model.pt +***** Load checkpoint successful! +missing keys: ['action_head.model.layer_norm1.weight', 'action_head.model.layer_norm1.bias', 'action_head.model.fc1.weight', 'action_head.model.fc1.bias', 'action_head.model.mlp_resnet_blocks.0.ffn.0.weight', 'action_head.model.mlp_resnet_blocks.0.ffn.0.bias', 'action_head.model.mlp_resnet_blocks.0.ffn.1.weight', 'action_head.model.mlp_resnet_blocks.0.ffn.1.bias', 'action_head.model.mlp_resnet_blocks.1.ffn.0.weight', 'action_head.model.mlp_resnet_blocks.1.ffn.0.bias', 'action_head.model.mlp_resnet_blocks.1.ffn.1.weight', 'action_head.model.mlp_resnet_blocks.1.ffn.1.bias', 'action_head.model.layer_norm2.weight', 'action_head.model.layer_norm2.bias', 'action_head.model.fc2.weight', 'action_head.model.fc2.bias', 'proprio_projector.fc1.weight', 'proprio_projector.fc1.bias', 'proprio_projector.fc2.weight', 'proprio_projector.fc2.bias'] +unexpected keys: [] +************************* Initialize model successful! +************************* LoRA flags: use_lora=True, lora_llm=False, lora_vit=False, lora_connector=False +************************* Before add lora to model +************************* Before FSDP model wrapping +************************* FSDP model wrapping successful! +************************* Before building optimizer and scheduler +************* Before get lora params +************* After get lora params successfully +09/29 [17:07:04] INFO | >> Constructing optimizer with 2 param groups optim.py:1283 +************************************************** +After building optimizer and scheduler and model, before training, peak GPU memory (MB): 35614 +************************* VLATrainer initialized successfully! +************************* Before trainer.fit() +Pre-train system metrics + System/Peak GPU Memory (MB)=35,614 +09/29 [17:07:05] WARNING | >> /vast/users/xiaodan/zhangjian/A1/olmo/data/collator.py:200: UserWarning: To copy construct from a tensor, it is recommended to use warnings.py:109 + sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + timestep_list = [torch.tensor(ex["timestep"], dtype=torch.int64) for ex in batch] + +!!!Training failed: Given normalized_shape=[57344], expected input with shape [*, 57344], but got input of size[15, 8, 28672] +Traceback (most recent call last): + File "/vast/users/xiaodan/zhangjian/A1/scripts/train_for_action.py", line 593, in main + trainer.fit() + File "/vast/users/xiaodan/zhangjian/A1/olmo/train.py", line 2331, in fit + metrics = self.train_step(batch, reduce_global_loss=should_log_this_step) + File "/vast/users/xiaodan/zhangjian/A1/olmo/train.py", line 2038, in train_step + ce_batch_loss, z_batch_loss, batch_accuracy, lb_batch_loss, moe_z_batch_loss, expert_assignments,action_loss = self.train_batch(batch) + File "/vast/users/xiaodan/zhangjian/A1/olmo/train.py", line 1956, in train_batch + accuracy, ce_loss, z_loss, logits, action_loss = self.model_forward( + File "/vast/users/xiaodan/zhangjian/A1/olmo/train.py", line 1813, in model_forward + outputs = self.fsdp_model.forward( + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py", line 854, in forward + output = self._fsdp_wrapped_module(*args, **kwargs) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl + return forward_call(*args, **kwargs) + File "/vast/users/xiaodan/zhangjian/A1/olmo/vla/affordvla.py", line 1042, in forward + predicted_actions = self.action_head.predict_action(action_hidden_states) + File "/vast/users/xiaodan/zhangjian/A1/olmo/vla/action_heads.py", line 249, in predict_action + action = self.model(rearranged_actions_hidden_states) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl + return forward_call(*args, **kwargs) + File "/vast/users/xiaodan/zhangjian/A1/olmo/vla/action_heads.py", line 210, in forward + x = self.layer_norm1(x) # shape: (batch_size, input_dim) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl + return forward_call(*args, **kwargs) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/normalization.py", line 217, in forward + return F.layer_norm( + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/functional.py", line 2905, in layer_norm + return torch.layer_norm( +RuntimeError: Given normalized_shape=[57344], expected input with shape [*, 57344], but got input of size[15, 8, 28672] +wandb: WARNING The `quiet` argument to `wandb.run.finish()` is deprecated, use `wandb.Settings(quiet=...)` to set this instead. diff --git a/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/requirements.txt b/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..291a344502cb68f8ac9dcf95e2d7e2df5f47b9b6 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/requirements.txt @@ -0,0 +1,284 @@ +ai2-molmo==0.0.0 +astunparse==1.6.3 +flatbuffers==25.2.10 +gast==0.6.0 +google-pasta==0.2.0 +h5py==3.14.0 +libclang==18.1.1 +Markdown==3.9 +namex==0.1.0 +opt_einsum==3.4.0 +optree==0.17.0 +tensorboard-data-server==0.7.2 +tensorflow-io-gcs-filesystem==0.37.1 +termcolor==3.1.0 +Werkzeug==3.1.3 +Brotli==1.1.0 +Farama-Notifications==0.0.4 +MarkupSafe==2.1.5 +PyYAML==6.0.2 +absl-py==2.3.1 +accelerate==1.10.1 +ai2-molmo==0.0.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +antlr4-python3-runtime==4.9.3 +anyio==4.10.0 +array_record==0.8.1 +async-timeout==5.0.1 +attrs==25.3.0 +av==15.1.0 +backports.tarfile==1.2.0 +beaker-gantry==3.2.0 +beaker-py==2.5.0 +black==23.12.1 +blinker==1.9.0 +boltons==25.0.0 +boto3==1.40.33 +botocore==1.40.33 +build==1.3.0 +cached_path==1.7.3 +cached-property==2.0.1 +cachetools==5.5.2 +certifi==2025.8.3 +cffi==2.0.0 +charset-normalizer==3.4.3 +click==8.2.1 +click-help-colors==0.9.4 +click-option-group==0.5.7 +cloudpickle==3.1.1 +cmake==4.1.0 +contourpy==1.3.2 +cryptography==46.0.1 +cycler==0.12.1 +dataclass-extensions==0.2.3 +datasets==3.6.0 +decorator==5.2.1 +deepdiff==8.6.1 +diffusers==0.35.1 +dill==0.3.8 +distro==1.9.0 +dlimp==0.0.1 +dm-tree==0.1.9 +docutils==0.22.1 +draccus==0.10.0 +editdistance==0.8.1 +einops==0.8.1 +einops-exts==0.0.4 +etils==1.13.0 +evdev==1.9.2 +exceptiongroup==1.3.0 +face==24.0.0 +fastapi==0.116.2 +ffmpy==0.6.1 +fiddle==0.3.0 +filelock==3.13.1 +Flask==3.1.2 +fonttools==4.60.0 +frozenlist==1.7.0 +fsspec==2023.9.2 +ftfy==6.3.1 +gcsfs==2023.9.2 +gitdb==4.0.12 +GitPython==3.1.45 +glom==24.11.0 +google-api-core==2.25.1 +google-auth==2.40.3 +google-auth-oauthlib==1.2.2 +google-cloud-core==2.4.3 +google-cloud-storage==2.19.0 +google-crc32c==1.7.1 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +gradio==5.46.0 +gradio_client==1.13.0 +graphviz==0.21 +groovy==0.1.2 +grpcio==1.75.0 +gymnasium==0.29.1 +h11==0.16.0 +hf_transfer==0.1.9 +hf-xet==1.1.10 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==0.35.0 +id==1.5.0 +idna==3.10 +imageio==2.37.0 +imageio-ffmpeg==0.6.0 +importlib_metadata==8.7.0 +importlib_resources==6.5.2 +iniconfig==2.1.0 +inquirerpy==0.3.4 +isort==5.12.0 +itsdangerous==2.2.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.3.0 +jeepney==0.9.0 +Jinja2==3.1.4 +jiter==0.11.0 +jmespath==1.0.1 +joblib==1.5.2 +jsonlines==4.0.0 +keras==2.15.0 +keyring==25.6.0 +kiwisolver==1.4.9 +latex2sympy2_extended==1.10.2 +lerobot==0.3.4 +Levenshtein==0.27.1 +libcst==1.8.4 +lightning-utilities==0.15.2 +markdown-it-py==4.0.0 +math-verify==0.8.0 +matplotlib==3.10.6 +mdurl==0.1.2 +mergedeep==1.3.4 +ml-dtypes==0.2.0 +ml_dtypes==0.5.3 +more-itertools==10.8.0 +mpmath==1.3.0 +msgspec==0.19.0 +multidict==6.6.4 +multiprocess==0.70.16 +mypy==1.3.0 +mypy_extensions==1.1.0 +necessary==0.4.3 +networkx==3.3 +nh3==0.3.0 +nltk==3.9.1 +numpy==1.26.4 +oauthlib==3.3.1 +omegaconf==2.3.0 +openai==1.108.0 +opencv-python-headless==4.12.0.88 +OpenEXR==3.4.0 +orderly-set==5.5.0 +orjson==3.11.3 +packaging==25.0 +pandas==2.3.2 +pathspec==0.12.1 +petname==2.6 +pfzy==0.3.4 +pillow==11.0.0 +pip==25.2 +platformdirs==4.4.0 +pluggy==1.6.0 +promise==2.3 +prompt_toolkit==3.0.52 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==4.21.12 +protobuf==6.32.1 +psutil==7.1.0 +pyarrow==21.0.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.23 +pydantic==2.11.9 +pydantic_core==2.33.2 +pydub==0.25.1 +Pygments==2.19.2 +pynput==1.8.1 +pyparsing==3.2.4 +pyproject_hooks==1.2.0 +pyserial==3.5 +pytest==8.4.2 +pytest-sphinx==0.6.3 +python-dateutil==2.9.0.post0 +python-Levenshtein==0.27.1 +python-multipart==0.0.20 +python-xlib==0.33 +pytorch-triton-rocm==3.4.0 +pytz==2025.2 +pyyaml-include==1.4.1 +RapidFuzz==3.14.1 +readme_renderer==44.0 +regex==2025.9.1 +requests==2.32.5 +requests-oauthlib==2.0.0 +requests-toolbelt==1.0.0 +requirements-parser==0.13.0 +rerun-sdk==0.22.1 +rfc3986==2.0.0 +rich==13.9.4 +rsa==4.9.1 +ruff==0.13.0 +s3transfer==0.14.0 +safehttpx==0.1.6 +safetensors==0.6.2 +scikit-learn==1.7.2 +scipy==1.15.3 +SecretStorage==3.4.0 +semantic-version==2.10.0 +sentencepiece==0.2.1 +sentry-sdk==2.38.0 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.17.0 +smart_open==7.3.1 +smashed==0.21.5 +smmap==5.0.2 +sniffio==1.3.1 +starlette==0.48.0 +sympy==1.13.3 +tensorboard==2.15.2 +tensorboard==2.19.0 +tensorflow==2.15.0 +tensorflow-addons==0.23.0 +tensorflow-datasets==4.9.3 +tensorflow-estimator==2.15.0 +tensorflow-graphics==2021.12.3 +tensorflow-metadata==1.17.2 +threadpoolctl==3.6.0 +timm==1.0.19 +tokenizers==0.22.0 +toml==0.10.2 +tomli==2.2.1 +tomlkit==0.13.3 +torch==2.8.0+rocm6.4 +torchcodec==0.5 +torchmetrics==1.8.2 +torchvision==0.23.0+rocm6.4 +tqdm==4.67.1 +transformers==4.56.1 +trimesh==4.8.2 +trouting==0.3.3 +twine==6.2.0 +typeguard==2.13.3 +typer==0.17.4 +typing_extensions==4.15.0 +typing-inspect==0.9.0 +typing-inspection==0.4.1 +tzdata==2025.2 +urllib3==2.5.0 +uvicorn==0.35.0 +wandb==0.21.4 +wcwidth==0.2.13 +websockets==15.0.1 +wheel==0.45.1 +wrapt==1.14.2 +xxhash==3.5.0 +yarl==1.20.1 +zipp==3.23.0 +lerobot==0.3.4 +minLoRA==0.1.0 +autocommand==2.2.2 +backports.tarfile==1.2.0 +importlib_metadata==8.0.0 +inflect==7.3.1 +jaraco.collections==5.1.0 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +jaraco.text==3.12.1 +more-itertools==10.3.0 +packaging==24.2 +platformdirs==4.2.2 +tomli==2.0.1 +typeguard==4.3.0 +typing_extensions==4.12.2 +wheel==0.45.1 +zipp==3.19.2 diff --git a/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/wandb-metadata.json b/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5556d62976952353e85bafcb6ff4cad43e368ad3 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/wandb-metadata.json @@ -0,0 +1,202 @@ +{ + "os": "Linux-5.15.0-140-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.18", + "startedAt": "2025-09-29T17:05:12.548535Z", + "args": [ + "qwen2_7b", + "save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1", + "--vision_backbone", + "openai", + "--action_head", + "l1_regression", + "--seq_len", + "1600", + "--ft_llm", + "--checkpoint", + "/vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924", + "--device_train_microbatch_size", + "16", + "--global_batch_size", + "126", + "--dataset", + "vla_dataset_realworld", + "--llm_learning_rate", + "5e-5", + "--wandb_entity", + "henryeap", + "--wandb_project", + "a1-realworld", + "--wandb_run_name", + "realworld", + "--save_overwrite" + ], + "program": "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py", + "codePath": "launch_scripts/train_vla.py", + "codePathLocal": "launch_scripts/train_vla.py", + "git": { + "remote": "https://github.com/Spatialtemporal-AI/A1.git", + "commit": "50cf9fd3143e218eb94104381c16c0482ac52f0d" + }, + "email": "ihenrykwok@outlook.com", + "root": "/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb", + "host": "auh7-1b-gpu-292", + "executable": "/vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10", + "cpu_count": 64, + "cpu_count_logical": 128, + "gpu": "Instinct MI210", + "gpu_count": 8, + "disk": { + "/": { + "total": "470343073792", + "used": "50842279936" + } + }, + "memory": { + "total": "2434606923776" + }, + "gpu_amd": [ + { + "id": "0", + "uniqueId": "0xc6ed7c5159e83b1", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "5", + "uniqueId": "0xa95e252cd7a6e54e", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "1", + "uniqueId": "0x333c966d5f3b3375", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "7", + "uniqueId": "0x5656afec2788d569", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "6", + "uniqueId": "0xad1bf75f50313455", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "3", + "uniqueId": "0xbabaa83bdb6fe877", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "4", + "uniqueId": "0xdee6d87744a36ae", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "2", + "uniqueId": "0x2a954aa975e59d15", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + } + ], + "slurm": { + "cluster_name": "ai-04r", + "conf": "/etc/slurm/slurm.conf", + "cpus_on_node": "128", + "gpus_on_node": "8", + "gtids": "0", + "job_account": "faculty-acc", + "job_cpus_per_node": "128", + "job_end_time": "1759424668", + "job_gid": "2000", + "job_gpus": "0,1,2,3,4,5,6,7", + "job_id": "1934", + "job_name": "mh_realworld", + "job_nodelist": "auh7-1b-gpu-292", + "job_num_nodes": "1", + "job_partition": "faculty", + "job_qos": "xdqos", + "job_start_time": "1759165468", + "job_uid": "2013", + "job_user": "xiaodan", + "jobid": "1934", + "localid": "0", + "nnodes": "1", + "nodeid": "0", + "nodelist": "auh7-1b-gpu-292", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "oom_kill_step": "0", + "prio_process": "0", + "procid": "0", + "submit_dir": "/vast/users/xiaodan/zhangjian/A1/launch_scripts", + "submit_host": "auh-1b-cpu-login-001", + "task_pid": "1357871", + "tasks_per_node": "1", + "topology_addr": "auh7-1b-gpu-292", + "topology_addr_pattern": "node" + }, + "writerId": "zpxgtt7ztfwstl3gysdjfozjodgqnvkm" +} \ No newline at end of file diff --git a/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/wandb-summary.json b/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..a79bb9cbafb2005ea305802a28cb776082c6cc20 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/wandb-summary.json @@ -0,0 +1 @@ +{"_step":0,"System/Peak GPU Memory (MB)":35614.78125,"_timestamp":1.759165624842901e+09,"_wandb":{"runtime":118},"_runtime":118.9149238} \ No newline at end of file diff --git a/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/logs/debug-core.log b/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..df1a03c8f4db65d039dc4b3c65f77f97fd647b1e --- /dev/null +++ b/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/logs/debug-core.log @@ -0,0 +1,16 @@ +{"time":"2025-09-29T17:05:12.599767574Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpblcfcrd2/port-1358048.txt","pid":1358048,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-09-29T17:05:12.601129497Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1358048} +{"time":"2025-09-29T17:05:12.601134567Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1358048-1358213-1657513860/socket","Net":"unix"}} +{"time":"2025-09-29T17:05:12.784034321Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-09-29T17:05:12.788802342Z","level":"INFO","msg":"handleInformInit: received","streamId":"2lq20p1f","id":"1(@)"} +{"time":"2025-09-29T17:05:13.92427811Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"2lq20p1f","id":"1(@)"} +{"time":"2025-09-29T17:07:15.469972392Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"2lq20p1f","id":"1(@)"} +{"time":"2025-09-29T17:07:15.473846438Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"2lq20p1f","id":"1(@)"} +{"time":"2025-09-29T17:07:15.473852038Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2025-09-29T17:07:15.473858278Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2025-09-29T17:07:15.473865358Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-09-29T17:07:15.473865688Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2025-09-29T17:07:15.473901259Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2025-09-29T17:07:15.473904939Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2025-09-29T17:07:15.473942719Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1358048-1358213-1657513860/socket","Net":"unix"}} +{"time":"2025-09-29T17:07:15.47396587Z","level":"INFO","msg":"server is closed"} diff --git a/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/logs/debug-internal.log b/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..11d44621118f40b354d734fd97a3801bb26d17fb --- /dev/null +++ b/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/logs/debug-internal.log @@ -0,0 +1,12 @@ +{"time":"2025-09-29T17:05:12.790661794Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-09-29T17:05:13.924227629Z","level":"INFO","msg":"stream: created new stream","id":"2lq20p1f"} +{"time":"2025-09-29T17:05:13.92427238Z","level":"INFO","msg":"stream: started","id":"2lq20p1f"} +{"time":"2025-09-29T17:05:13.92428108Z","level":"INFO","msg":"writer: started","stream_id":"2lq20p1f"} +{"time":"2025-09-29T17:05:13.924301101Z","level":"INFO","msg":"handler: started","stream_id":"2lq20p1f"} +{"time":"2025-09-29T17:05:13.924330541Z","level":"INFO","msg":"sender: started","stream_id":"2lq20p1f"} +{"time":"2025-09-29T17:07:13.196500373Z","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run metadata","runtime_seconds":0.001296762}],"total_operations":1}} +{"time":"2025-09-29T17:07:15.018224116Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-09-29T17:07:15.470258007Z","level":"INFO","msg":"stream: closing","id":"2lq20p1f"} +{"time":"2025-09-29T17:07:15.470272437Z","level":"INFO","msg":"handler: closed","stream_id":"2lq20p1f"} +{"time":"2025-09-29T17:07:15.472123178Z","level":"INFO","msg":"sender: closed","stream_id":"2lq20p1f"} +{"time":"2025-09-29T17:07:15.472129189Z","level":"INFO","msg":"stream: closed","id":"2lq20p1f"} diff --git a/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/logs/debug.log b/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/run-2lq20p1f.wandb b/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/run-2lq20p1f.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c35abd43d66a97ec65d3daea60d9fa5e2f9bbc48 Binary files /dev/null and b/all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/run-2lq20p1f.wandb differ diff --git a/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/config.yaml b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d2fd9fdcb448ffcf4bb3c69f5f62d8eda4e524e9 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/config.yaml @@ -0,0 +1,617 @@ +_wandb: + value: + cli_version: 0.21.4 + e: + xw28krbghcx1p9g77lw10b5hvgwgqmco: + args: + - qwen2_7b + - save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1 + - --vision_backbone + - openai + - --action_head + - l1_regression + - --seq_len + - "1600" + - --ft_llm + - --checkpoint + - /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 + - --device_train_microbatch_size + - "16" + - --global_batch_size + - "126" + - --dataset + - vla_dataset_realworld + - --llm_learning_rate + - "5e-5" + - --wandb_entity + - henryeap + - --wandb_project + - a1-realworld + - --wandb_run_name + - realworld + - --save_overwrite + codePath: launch_scripts/train_vla.py + codePathLocal: launch_scripts/train_vla.py + cpu_count: 64 + cpu_count_logical: 128 + disk: + /: + total: "470343073792" + used: "50128396288" + email: ihenrykwok@outlook.com + executable: /vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10 + git: + commit: f2afcc15e05f491a8e50add64395fc1db0a1188d + remote: https://github.com/Spatialtemporal-AI/A1.git + gpu: Instinct MI210 + gpu_amd: + - id: "5" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x413935505e32b8da" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "3" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x95be8fdc770fcfd7" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "1" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x27087f06439a527d" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "0" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x82728d7f9bd937e4" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "7" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xa0442ab3bdd405c1" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "2" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xaabcddaa244a3d6e" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "6" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x12140cd9e24f12e9" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "4" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x24ee801b7c402006" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + gpu_count: 8 + host: auh7-1b-gpu-306 + memory: + total: "2434611519488" + os: Linux-5.15.0-140-generic-x86_64-with-glibc2.35 + program: /vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py + python: CPython 3.10.18 + root: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb + slurm: + cluster_name: ai-04r + conf: /etc/slurm/slurm.conf + cpus_on_node: "128" + gpus_on_node: "8" + gtids: "0" + job_account: faculty-acc + job_cpus_per_node: "128" + job_end_time: "1759481466" + job_gid: "2000" + job_gpus: 0,1,2,3,4,5,6,7 + job_id: "1970" + job_name: mh_realworld + job_nodelist: auh7-1b-gpu-306 + job_num_nodes: "1" + job_partition: faculty + job_qos: xdqos + job_start_time: "1759222266" + job_uid: "2013" + job_user: xiaodan + jobid: "1970" + localid: "0" + nnodes: "1" + nodeid: "0" + nodelist: auh7-1b-gpu-306 + nprocs: "1" + ntasks: "1" + ntasks_per_node: "1" + oom_kill_step: "0" + prio_process: "0" + procid: "0" + submit_dir: /vast/users/xiaodan/zhangjian/A1/launch_scripts + submit_host: auh-1b-cpu-login-001 + task_pid: "589646" + tasks_per_node: "1" + topology_addr: auh7-1b-gpu-306 + topology_addr_pattern: node + startedAt: "2025-09-30T08:52:06.337927Z" + writerId: xw28krbghcx1p9g77lw10b5hvgwgqmco + m: [] + python_version: 3.10.18 + t: + "1": + - 1 + - 2 + - 3 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 63 + - 71 + - 83 + - 95 + - 105 + "2": + - 1 + - 2 + - 3 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 63 + - 71 + - 83 + - 95 + - 105 + "3": + - 13 + - 15 + - 16 + "4": 3.10.18 + "5": 0.21.4 + "6": 4.56.1 + "12": 0.21.4 + "13": linux-x86_64 +activation_checkpointing: + value: whole_layer +allow_resume: + value: false +batch_divisor: + value: global_batch +canceled_check_interval: + value: 50 +checkpoint_dir: + value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 +compile: + value: null +console_log_interval: + value: 1 +data: + value: + dataset: vla_dataset_realworld + drop_last: true + for_inference: false + lerobot_episode_index_end: null + lerobot_episode_index_start: null + mixture: null + multi_modal: torch + num_workers: 0 + pad: to_max + persistent_workers: false + pin_memory: true + prefetch_factor: null + rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds + rlds_dataset_name: libero_4_task_suites_no_noops + rlds_read_threads: 8 + rlds_shuffle_buffer_size: 100000 + rlds_traj_threads: 8 + root_size_mixture: null + seed: 95818 + sequence_length: 1600 + shuffle: true + shuffle_messages: false + split: train + timeout: 0 + use_proprio: true + use_wrist_image: true +device_eval_batch_size: + value: 4 +device_inf_eval_batch_size: + value: 16 +device_train_batch_size: + value: 15 +device_train_grad_accum: + value: 0 +device_train_microbatch_size: + value: 16 +dry_run: + value: false +early_exit: + value: false +epoch: + value: null +eval_interval: + value: 0 +eval_on_load: + value: false +eval_subset_num_batches: + value: -1 +evaluators: + value: + - data: + dataset: vla_dataset_realworld + drop_last: true + for_inference: false + lerobot_episode_index_end: 765 + lerobot_episode_index_start: 353 + mixture: null + multi_modal: torch + num_workers: 0 + pad: to_max + persistent_workers: true + pin_memory: true + prefetch_factor: null + rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds + rlds_dataset_name: libero_4_task_suites_no_noops + rlds_read_threads: 8 + rlds_shuffle_buffer_size: 256000 + rlds_traj_threads: 8 + root_size_mixture: null + seed: null + sequence_length: 1600 + shuffle: false + shuffle_messages: false + split: validation + timeout: 0 + use_proprio: true + use_wrist_image: true + device_eval_batch_size: null + eval_name: null + label: val + max_examples: null + max_new_tokens: 448 + mm_evaluator: null + save_dir: null + save_to_checkpoint_dir: false + skip_if_metrics_cached: true + subset_num_batches: 64 +extra_steps_after_cancel: + value: 10 +fast_forward_batches: + value: null +force_save_unsharded: + value: false +fsdp: + value: + hybrid_sharding_num_model_replicas: null + precision: float + sharding_strategy: FULL_SHARD + use_orig_params: true + wrapping_strategy: by_block_and_size +ft_connector: + value: false +ft_embedding: + value: lm_head +ft_llm: + value: true +ft_vit: + value: false +fused_loss: + value: null +gen1_gc_interval: + value: 1 +global_train_batch_size: + value: 126 +inf_eval_interval: + value: -1 +inf_eval_subset_num_batches: + value: -1 +inf_evaluators: + value: [] +initial_model_checkpoint: + value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 +keep_lr_on_load: + value: true +load_model_config: + value: null +load_path: + value: null +load_path_sharded_checkpointer: + value: null +lora: + value: false +lora_connector: + value: false +lora_llm: + value: false +lora_rank: + value: 8 +lora_vit: + value: false +max_duration: + value: 500000 +max_grad_norm: + value: 1 +max_grad_norm_ratio: + value: null +model: + value: + action_dim: 7 + action_head: l1_regression + action_head_dit_depth: 28 + action_head_dit_hidden_size: 1152 + action_head_dit_num_heads: 16 + action_tokenizer: + identifier: physical-intelligence/fast + tokenizer_dir: null + action_use_left_eef: false + action_use_mobile_base: false + activation_type: swiglu + additional_vocab_size: 128 + always_start_with_space: true + attention_dropout: 0 + attention_layer_norm: false + attention_layer_norm_with_affine: true + attention_type: sdpa + bias_for_layer_norm: null + block_group_size: 1 + block_type: sequential + clip_qkv: null + crop_mode: overlap-and-resize-c2 + d_model: 3584 + default_inference_len: 65 + embedding_dropout: 0 + embedding_size: 152064 + ff_out_size: 0 + fix_image_padding: true + float32_attention: true + head_dim: null + horizon: 8 + image_feature_dropout: 0 + image_padding_embed: pad_and_partial_pad + image_pooling_2d: attention_meanq + image_pooling_h: 2 + image_pooling_w: 2 + image_projector: mlp + include_bias: false + init_cutoff_factor: null + init_device: null + init_fn: normal + init_std: 0.02 + initializer_range: 0.02 + layer_norm_eps: 1e-06 + layer_norm_type: rms + layer_norm_with_affine: true + llm_causal_attention: false + llm_load_path: /weka/oe-training-default/mm-olmo/pretrained_llms/qwen2-7b.pt + low_cpu_fsdp: true + max_crops: 12 + max_position_embeddings: null + max_sequence_length: 4096 + message_formatting: role + mlp_hidden_size: 37888 + mlp_ratio: 4 + moe_capacity_factor: 1.25 + moe_dropless: true + moe_interleave: false + moe_lbl_in_fp32: false + moe_log_expert_assignment: false + moe_loss_weight: 0.1 + moe_mlp_impl: sparse + moe_num_experts: 8 + moe_shared_expert: false + moe_top_k: 2 + moe_zloss_weight: null + multi_annotation_weighting: root_subsegments + n_heads: 28 + n_kv_heads: 4 + n_layers: 28 + new_embedding_init_range: 0.02 + norm_after: false + normalize_input_embeds: false + num_diffusion_inference_steps: 30 + num_diffusion_steps: 1000 + overlap_margins: + - 4 + - 4 + pad_tokenizer: true + pad_value: 0 + precision: amp_bf16 + prompt_type: uber_model + qkv_bias: true + residual_dropout: 0.1 + response_residual_dropout: 0 + rope: true + rope_full_precision: true + rope_theta: 1e+06 + scale_logits: false + system_prompt_kind: demo_or_style + tokenizer: + identifier: Qwen/Qwen2-7B + tokenizer_dir: null + use_col_tokens: true + use_position_ids: true + use_proprio: true + vision_backbone: + attention_dropout: 0 + fsdp_wrap: false + image_default_input_size: + - 336 + - 336 + image_dropout_rate: 0 + image_emb_dim: 1024 + image_head_dim: 64 + image_mlp_activations: quick_gelu + image_mlp_dim: 4096 + image_model_type: openai + image_norm_eps: 1e-05 + image_num_heads: 16 + image_num_key_value_heads: 16 + image_num_layers: 23 + image_num_pos: 577 + image_patch_size: 14 + image_pos_patch_size: 14 + initializer_range: 0.02 + residual_dropout: 0 + resize_mode: default + vit_layers: + - -2 + - -9 + vit_load_path: /weka/oe-training-default/mm-olmo/pretrained_image_encoders/vit-l-14-336.pt + vocab_size: 152064 + weight_tying: false +multi_component_grad_norm: + value: true +no_pre_train_checkpoint: + value: true +optimizer: + value: + betas: + - 0.9 + - 0.95 + connector_betas: + - 0.9 + - 0.95 + connector_eps: 1e-06 + connector_learning_rate: 0.0002 + connector_weight_decay: 0 + eps: 1e-05 + learning_rate: 0.0001 + llm_betas: + - 0.9 + - 0.95 + llm_eps: 1e-06 + llm_learning_rate: 5e-05 + llm_weight_decay: 0 + metrics_log_interval: 20 + name: adamw + vit_betas: + - 0.9 + - 0.95 + vit_eps: 1e-06 + vit_learning_rate: 6e-06 + vit_weight_decay: 0 + weight_decay: 0.01 +precision: + value: amp_bf16 +python_profiling: + value: false +remote_save_folder: + value: null +reset_dataloader_state: + value: false +reset_optimizer_state: + value: false +reset_trainer_state: + value: false +restore_dataloader: + value: true +run_name: + value: realworld_20250930_085126 +save_dataloader_state: + value: false +save_folder: + value: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1 +save_interval: + value: 500 +save_interval_action_head: + value: 500 +save_interval_ephemeral: + value: null +save_interval_unsharded: + value: 500 +save_num_action_head_checkpoints_to_keep: + value: 2 +save_num_checkpoints_to_keep: + value: 1 +save_num_unsharded_checkpoints_to_keep: + value: 1 +save_overwrite: + value: true +scheduler: + value: + alpha_f: 0.1 + connector_t_warmup: 200 + grad_clip_warmup_factor: null + grad_clip_warmup_steps: null + llm_t_warmup: 2000 + name: multimodal + t_max: null + t_warmup: 100 + units: steps + vit_t_warmup: 2000 + warmup_min_lr: 0 +seed: + value: 6198 +sharded_checkpointer: + value: torch_legacy +softmax_auxiliary_loss: + value: true +softmax_auxiliary_loss_scale: + value: 0.0001 +speed_monitor: + value: + gpu_flops_available: null + window_size: 20 +stop_after: + value: null +stop_at: + value: 500000 +time_limit: + value: null +torch_profiling: + value: false +train_exit_random_layer: + value: false +use_lora: + value: true diff --git a/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/output.log b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..9c4ae3ffc3eb99091b436cc7dbdbe8bc0402d935 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/output.log @@ -0,0 +1,86 @@ +wandb: Detected [openai] in use. +wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script. +wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/ +09/30 [08:52:08] WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No warnings.py:109 + device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once + +****** vla_cfg: {'datasets': {'rlds': {'name': None, 'path': None, 'weight': 1.0, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe', 1, 'bounds']], 'open-source-real-world': {'rlds': {'name': 'a1_real_world', 'path': '/vast/users/xiaodan/zhangjian/datasets/OXE', 'weight': 8, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [], 'agibot': {'path': '/vast/users/xiaodan/zhangjian/datasets/AgiBotWorld-Alpha', 'weight': 8, 'action_proprio_normalization_type': None}}}, 'model': {'action_head': {'action_dim': 16, 'proprio_dim': 16, 'num_actions_chunk': 8, 'action_tokens_mapping': {'left_end_effector': 8, 'right_end_effector': 8}, 'use_left_eef': True, 'use_mobile_base': False}}} +****** Skip RLDS main; path not found: None +****** start build LeRobot main... +build_tokenizer, cache_dir None tokenizer_dir None +09/30 [08:52:19] INFO | >> Padding tokenizer with 418 tokens tokenizer.py:130 +09/30 [08:52:20] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk +****** length of the dataset: 72641 +09/30 [08:52:26] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50 +****** length of the dataset: 27906 +09/30 [08:52:27] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser +****** length of the dataset: 13441 +09/30 [08:52:28] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue +****** length of the dataset: 10316 +09/30 [08:52:29] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen +****** length of the dataset: 17131 +09/30 [08:52:31] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote +****** length of the dataset: 15765 +09/30 [08:52:32] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue +****** length of the dataset: 90 + INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe +****** length of the dataset: 18397 +****** Skip RLDS open-source-real-world; mixture 'a1_real_world' not found under: /vast/users/xiaodan/zhangjian/datasets/OXE +****** Expect one of: [] +****** path: /vast/users/xiaodan/zhangjian/datasets/AgiBotWorld-Alpha +****** before AgiBotWorldAlpha dataset... +09/30 [08:52:33] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:485 +Traceback (most recent call last): + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/compat/_optional.py", line 135, in import_optional_dependency + module = importlib.import_module(name) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/importlib/__init__.py", line 126, in import_module + return _bootstrap._gcd_import(name[level:], package, level) + File "", line 1050, in _gcd_import + File "", line 1027, in _find_and_load + File "", line 1004, in _find_and_load_unlocked +ModuleNotFoundError: No module named 'openpyxl' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py", line 397, in + train(cfg) + File "/vast/users/xiaodan/zhangjian/A1/scripts/train_for_action.py", line 160, in main + train_loader = build_train_dataloader(cfg, device) + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/__init__.py", line 197, in build_train_dataloader + return build_vla_train_dataloader(train_config, device) + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/__init__.py", line 384, in build_vla_train_dataloader + ds = build_agibot_train_dataset(train_config, normalization_type, device) + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/__init__.py", line 487, in build_agibot_train_dataset + dataset = AgiBotWorldAlphaDataset( + File "", line 13, in __init__ + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/vla/agibot_dataset.py", line 145, in __post_init__ + self._frame_ranges_map: Optional[Dict[Tuple[str, int], Tuple[int, int]]] = self._load_frame_ranges_excel() + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/vla/agibot_dataset.py", line 411, in _load_frame_ranges_excel + df = pd.read_excel(excel_path) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/io/excel/_base.py", line 495, in read_excel + io = ExcelFile( + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/io/excel/_base.py", line 1567, in __init__ + self._reader = self._engines[engine]( + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/io/excel/_openpyxl.py", line 552, in __init__ + import_optional_dependency("openpyxl") + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/compat/_optional.py", line 138, in import_optional_dependency + raise ImportError(msg) +ImportError: Missing optional dependency 'openpyxl'. Use pip or conda to install openpyxl. diff --git a/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/requirements.txt b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..291a344502cb68f8ac9dcf95e2d7e2df5f47b9b6 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/requirements.txt @@ -0,0 +1,284 @@ +ai2-molmo==0.0.0 +astunparse==1.6.3 +flatbuffers==25.2.10 +gast==0.6.0 +google-pasta==0.2.0 +h5py==3.14.0 +libclang==18.1.1 +Markdown==3.9 +namex==0.1.0 +opt_einsum==3.4.0 +optree==0.17.0 +tensorboard-data-server==0.7.2 +tensorflow-io-gcs-filesystem==0.37.1 +termcolor==3.1.0 +Werkzeug==3.1.3 +Brotli==1.1.0 +Farama-Notifications==0.0.4 +MarkupSafe==2.1.5 +PyYAML==6.0.2 +absl-py==2.3.1 +accelerate==1.10.1 +ai2-molmo==0.0.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +antlr4-python3-runtime==4.9.3 +anyio==4.10.0 +array_record==0.8.1 +async-timeout==5.0.1 +attrs==25.3.0 +av==15.1.0 +backports.tarfile==1.2.0 +beaker-gantry==3.2.0 +beaker-py==2.5.0 +black==23.12.1 +blinker==1.9.0 +boltons==25.0.0 +boto3==1.40.33 +botocore==1.40.33 +build==1.3.0 +cached_path==1.7.3 +cached-property==2.0.1 +cachetools==5.5.2 +certifi==2025.8.3 +cffi==2.0.0 +charset-normalizer==3.4.3 +click==8.2.1 +click-help-colors==0.9.4 +click-option-group==0.5.7 +cloudpickle==3.1.1 +cmake==4.1.0 +contourpy==1.3.2 +cryptography==46.0.1 +cycler==0.12.1 +dataclass-extensions==0.2.3 +datasets==3.6.0 +decorator==5.2.1 +deepdiff==8.6.1 +diffusers==0.35.1 +dill==0.3.8 +distro==1.9.0 +dlimp==0.0.1 +dm-tree==0.1.9 +docutils==0.22.1 +draccus==0.10.0 +editdistance==0.8.1 +einops==0.8.1 +einops-exts==0.0.4 +etils==1.13.0 +evdev==1.9.2 +exceptiongroup==1.3.0 +face==24.0.0 +fastapi==0.116.2 +ffmpy==0.6.1 +fiddle==0.3.0 +filelock==3.13.1 +Flask==3.1.2 +fonttools==4.60.0 +frozenlist==1.7.0 +fsspec==2023.9.2 +ftfy==6.3.1 +gcsfs==2023.9.2 +gitdb==4.0.12 +GitPython==3.1.45 +glom==24.11.0 +google-api-core==2.25.1 +google-auth==2.40.3 +google-auth-oauthlib==1.2.2 +google-cloud-core==2.4.3 +google-cloud-storage==2.19.0 +google-crc32c==1.7.1 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +gradio==5.46.0 +gradio_client==1.13.0 +graphviz==0.21 +groovy==0.1.2 +grpcio==1.75.0 +gymnasium==0.29.1 +h11==0.16.0 +hf_transfer==0.1.9 +hf-xet==1.1.10 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==0.35.0 +id==1.5.0 +idna==3.10 +imageio==2.37.0 +imageio-ffmpeg==0.6.0 +importlib_metadata==8.7.0 +importlib_resources==6.5.2 +iniconfig==2.1.0 +inquirerpy==0.3.4 +isort==5.12.0 +itsdangerous==2.2.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.3.0 +jeepney==0.9.0 +Jinja2==3.1.4 +jiter==0.11.0 +jmespath==1.0.1 +joblib==1.5.2 +jsonlines==4.0.0 +keras==2.15.0 +keyring==25.6.0 +kiwisolver==1.4.9 +latex2sympy2_extended==1.10.2 +lerobot==0.3.4 +Levenshtein==0.27.1 +libcst==1.8.4 +lightning-utilities==0.15.2 +markdown-it-py==4.0.0 +math-verify==0.8.0 +matplotlib==3.10.6 +mdurl==0.1.2 +mergedeep==1.3.4 +ml-dtypes==0.2.0 +ml_dtypes==0.5.3 +more-itertools==10.8.0 +mpmath==1.3.0 +msgspec==0.19.0 +multidict==6.6.4 +multiprocess==0.70.16 +mypy==1.3.0 +mypy_extensions==1.1.0 +necessary==0.4.3 +networkx==3.3 +nh3==0.3.0 +nltk==3.9.1 +numpy==1.26.4 +oauthlib==3.3.1 +omegaconf==2.3.0 +openai==1.108.0 +opencv-python-headless==4.12.0.88 +OpenEXR==3.4.0 +orderly-set==5.5.0 +orjson==3.11.3 +packaging==25.0 +pandas==2.3.2 +pathspec==0.12.1 +petname==2.6 +pfzy==0.3.4 +pillow==11.0.0 +pip==25.2 +platformdirs==4.4.0 +pluggy==1.6.0 +promise==2.3 +prompt_toolkit==3.0.52 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==4.21.12 +protobuf==6.32.1 +psutil==7.1.0 +pyarrow==21.0.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.23 +pydantic==2.11.9 +pydantic_core==2.33.2 +pydub==0.25.1 +Pygments==2.19.2 +pynput==1.8.1 +pyparsing==3.2.4 +pyproject_hooks==1.2.0 +pyserial==3.5 +pytest==8.4.2 +pytest-sphinx==0.6.3 +python-dateutil==2.9.0.post0 +python-Levenshtein==0.27.1 +python-multipart==0.0.20 +python-xlib==0.33 +pytorch-triton-rocm==3.4.0 +pytz==2025.2 +pyyaml-include==1.4.1 +RapidFuzz==3.14.1 +readme_renderer==44.0 +regex==2025.9.1 +requests==2.32.5 +requests-oauthlib==2.0.0 +requests-toolbelt==1.0.0 +requirements-parser==0.13.0 +rerun-sdk==0.22.1 +rfc3986==2.0.0 +rich==13.9.4 +rsa==4.9.1 +ruff==0.13.0 +s3transfer==0.14.0 +safehttpx==0.1.6 +safetensors==0.6.2 +scikit-learn==1.7.2 +scipy==1.15.3 +SecretStorage==3.4.0 +semantic-version==2.10.0 +sentencepiece==0.2.1 +sentry-sdk==2.38.0 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.17.0 +smart_open==7.3.1 +smashed==0.21.5 +smmap==5.0.2 +sniffio==1.3.1 +starlette==0.48.0 +sympy==1.13.3 +tensorboard==2.15.2 +tensorboard==2.19.0 +tensorflow==2.15.0 +tensorflow-addons==0.23.0 +tensorflow-datasets==4.9.3 +tensorflow-estimator==2.15.0 +tensorflow-graphics==2021.12.3 +tensorflow-metadata==1.17.2 +threadpoolctl==3.6.0 +timm==1.0.19 +tokenizers==0.22.0 +toml==0.10.2 +tomli==2.2.1 +tomlkit==0.13.3 +torch==2.8.0+rocm6.4 +torchcodec==0.5 +torchmetrics==1.8.2 +torchvision==0.23.0+rocm6.4 +tqdm==4.67.1 +transformers==4.56.1 +trimesh==4.8.2 +trouting==0.3.3 +twine==6.2.0 +typeguard==2.13.3 +typer==0.17.4 +typing_extensions==4.15.0 +typing-inspect==0.9.0 +typing-inspection==0.4.1 +tzdata==2025.2 +urllib3==2.5.0 +uvicorn==0.35.0 +wandb==0.21.4 +wcwidth==0.2.13 +websockets==15.0.1 +wheel==0.45.1 +wrapt==1.14.2 +xxhash==3.5.0 +yarl==1.20.1 +zipp==3.23.0 +lerobot==0.3.4 +minLoRA==0.1.0 +autocommand==2.2.2 +backports.tarfile==1.2.0 +importlib_metadata==8.0.0 +inflect==7.3.1 +jaraco.collections==5.1.0 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +jaraco.text==3.12.1 +more-itertools==10.3.0 +packaging==24.2 +platformdirs==4.2.2 +tomli==2.0.1 +typeguard==4.3.0 +typing_extensions==4.12.2 +wheel==0.45.1 +zipp==3.19.2 diff --git a/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/wandb-metadata.json b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..92aad0de7fec10c8e0e8ec90e6a661f7bd5f712b --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/wandb-metadata.json @@ -0,0 +1,202 @@ +{ + "os": "Linux-5.15.0-140-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.18", + "startedAt": "2025-09-30T08:52:06.337927Z", + "args": [ + "qwen2_7b", + "save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1", + "--vision_backbone", + "openai", + "--action_head", + "l1_regression", + "--seq_len", + "1600", + "--ft_llm", + "--checkpoint", + "/vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924", + "--device_train_microbatch_size", + "16", + "--global_batch_size", + "126", + "--dataset", + "vla_dataset_realworld", + "--llm_learning_rate", + "5e-5", + "--wandb_entity", + "henryeap", + "--wandb_project", + "a1-realworld", + "--wandb_run_name", + "realworld", + "--save_overwrite" + ], + "program": "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py", + "codePath": "launch_scripts/train_vla.py", + "codePathLocal": "launch_scripts/train_vla.py", + "git": { + "remote": "https://github.com/Spatialtemporal-AI/A1.git", + "commit": "f2afcc15e05f491a8e50add64395fc1db0a1188d" + }, + "email": "ihenrykwok@outlook.com", + "root": "/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb", + "host": "auh7-1b-gpu-306", + "executable": "/vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10", + "cpu_count": 64, + "cpu_count_logical": 128, + "gpu": "Instinct MI210", + "gpu_count": 8, + "disk": { + "/": { + "total": "470343073792", + "used": "50128396288" + } + }, + "memory": { + "total": "2434611519488" + }, + "gpu_amd": [ + { + "id": "5", + "uniqueId": "0x413935505e32b8da", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "3", + "uniqueId": "0x95be8fdc770fcfd7", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "1", + "uniqueId": "0x27087f06439a527d", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "0", + "uniqueId": "0x82728d7f9bd937e4", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "7", + "uniqueId": "0xa0442ab3bdd405c1", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "2", + "uniqueId": "0xaabcddaa244a3d6e", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "6", + "uniqueId": "0x12140cd9e24f12e9", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "4", + "uniqueId": "0x24ee801b7c402006", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + } + ], + "slurm": { + "cluster_name": "ai-04r", + "conf": "/etc/slurm/slurm.conf", + "cpus_on_node": "128", + "gpus_on_node": "8", + "gtids": "0", + "job_account": "faculty-acc", + "job_cpus_per_node": "128", + "job_end_time": "1759481466", + "job_gid": "2000", + "job_gpus": "0,1,2,3,4,5,6,7", + "job_id": "1970", + "job_name": "mh_realworld", + "job_nodelist": "auh7-1b-gpu-306", + "job_num_nodes": "1", + "job_partition": "faculty", + "job_qos": "xdqos", + "job_start_time": "1759222266", + "job_uid": "2013", + "job_user": "xiaodan", + "jobid": "1970", + "localid": "0", + "nnodes": "1", + "nodeid": "0", + "nodelist": "auh7-1b-gpu-306", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "oom_kill_step": "0", + "prio_process": "0", + "procid": "0", + "submit_dir": "/vast/users/xiaodan/zhangjian/A1/launch_scripts", + "submit_host": "auh-1b-cpu-login-001", + "task_pid": "589646", + "tasks_per_node": "1", + "topology_addr": "auh7-1b-gpu-306", + "topology_addr_pattern": "node" + }, + "writerId": "xw28krbghcx1p9g77lw10b5hvgwgqmco" +} \ No newline at end of file diff --git a/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/wandb-summary.json b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..2d7b7348866aa5fab906f41ff70c8e018458d95f --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":25},"_runtime":25} \ No newline at end of file diff --git a/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/logs/debug-core.log b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..faaa15e20c7c1d1dfdc8501804845084e1bcdfc0 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2025-09-30T08:52:06.638339714Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpyjj93vky/port-589823.txt","pid":589823,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-09-30T08:52:06.641262155Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":589823} +{"time":"2025-09-30T08:52:06.642250983Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-589823-590216-2576664124/socket","Net":"unix"}} +{"time":"2025-09-30T08:52:06.792135331Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-09-30T08:52:06.805602998Z","level":"INFO","msg":"handleInformInit: received","streamId":"50kj35c0","id":"1(@)"} +{"time":"2025-09-30T08:52:08.062408989Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"50kj35c0","id":"1(@)"} +{"time":"2025-09-30T08:52:34.430005899Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2025-09-30T08:52:34.430229963Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-09-30T08:52:34.430227373Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2025-09-30T08:52:34.430267094Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2025-09-30T08:52:34.430322415Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-589823-590216-2576664124/socket","Net":"unix"}} +{"time":"2025-09-30T08:52:36.423142352Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2025-09-30T08:52:36.423156882Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2025-09-30T08:52:36.423167352Z","level":"INFO","msg":"server is closed"} diff --git a/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/logs/debug-internal.log b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1f5398aaef6dbbef94aa18f18575fdc1c4aad1f2 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2025-09-30T08:52:06.807490411Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-09-30T08:52:08.062344498Z","level":"INFO","msg":"stream: created new stream","id":"50kj35c0"} +{"time":"2025-09-30T08:52:08.062403319Z","level":"INFO","msg":"stream: started","id":"50kj35c0"} +{"time":"2025-09-30T08:52:08.062425769Z","level":"INFO","msg":"handler: started","stream_id":"50kj35c0"} +{"time":"2025-09-30T08:52:08.06244382Z","level":"INFO","msg":"sender: started","stream_id":"50kj35c0"} +{"time":"2025-09-30T08:52:08.06243294Z","level":"INFO","msg":"writer: started","stream_id":"50kj35c0"} +{"time":"2025-09-30T08:52:34.430242003Z","level":"INFO","msg":"stream: closing","id":"50kj35c0"} +{"time":"2025-09-30T08:52:36.099290171Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-09-30T08:52:36.421230078Z","level":"INFO","msg":"handler: closed","stream_id":"50kj35c0"} +{"time":"2025-09-30T08:52:36.422272446Z","level":"INFO","msg":"sender: closed","stream_id":"50kj35c0"} +{"time":"2025-09-30T08:52:36.422279596Z","level":"INFO","msg":"stream: closed","id":"50kj35c0"} diff --git a/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/logs/debug.log b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a572bc39e168280fb6c83f9d1eeb52bec35c3516 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/logs/debug.log @@ -0,0 +1 @@ +2025-09-30 08:52:34,430 INFO wandb-AsyncioManager-main:589823 [service_client.py:_forward_responses():84] Reached EOF. diff --git a/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/run-50kj35c0.wandb b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/run-50kj35c0.wandb new file mode 100644 index 0000000000000000000000000000000000000000..3d24940d7b458ba0975dbfc5eca3c9ce112b6b72 Binary files /dev/null and b/all_l1/wandb/wandb/run-20250930_085206-50kj35c0/run-50kj35c0.wandb differ diff --git a/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/config.yaml b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ecb4c5eaa0fa5353e84b8b6c3714ddad5dfdd4f4 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/config.yaml @@ -0,0 +1,617 @@ +_wandb: + value: + cli_version: 0.21.4 + e: + e39r496xjes4qj7ky2l3e9tlyr84a0v3: + args: + - qwen2_7b + - save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1 + - --vision_backbone + - openai + - --action_head + - l1_regression + - --seq_len + - "1600" + - --ft_llm + - --checkpoint + - /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 + - --device_train_microbatch_size + - "16" + - --global_batch_size + - "126" + - --dataset + - vla_dataset_realworld + - --llm_learning_rate + - "5e-5" + - --wandb_entity + - henryeap + - --wandb_project + - a1-realworld + - --wandb_run_name + - realworld + - --save_overwrite + codePath: launch_scripts/train_vla.py + codePathLocal: launch_scripts/train_vla.py + cpu_count: 64 + cpu_count_logical: 128 + disk: + /: + total: "470343073792" + used: "50128465920" + email: ihenrykwok@outlook.com + executable: /vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10 + git: + commit: f2afcc15e05f491a8e50add64395fc1db0a1188d + remote: https://github.com/Spatialtemporal-AI/A1.git + gpu: Instinct MI210 + gpu_amd: + - id: "6" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x12140cd9e24f12e9" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "3" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x95be8fdc770fcfd7" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "7" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xa0442ab3bdd405c1" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "4" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x24ee801b7c402006" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "2" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xaabcddaa244a3d6e" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "0" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x82728d7f9bd937e4" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "5" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x413935505e32b8da" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "1" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x27087f06439a527d" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + gpu_count: 8 + host: auh7-1b-gpu-306 + memory: + total: "2434611519488" + os: Linux-5.15.0-140-generic-x86_64-with-glibc2.35 + program: /vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py + python: CPython 3.10.18 + root: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb + slurm: + cluster_name: ai-04r + conf: /etc/slurm/slurm.conf + cpus_on_node: "128" + gpus_on_node: "8" + gtids: "0" + job_account: faculty-acc + job_cpus_per_node: "128" + job_end_time: "1759482071" + job_gid: "2000" + job_gpus: 0,1,2,3,4,5,6,7 + job_id: "1973" + job_name: mh_realworld + job_nodelist: auh7-1b-gpu-306 + job_num_nodes: "1" + job_partition: faculty + job_qos: xdqos + job_start_time: "1759222871" + job_uid: "2013" + job_user: xiaodan + jobid: "1973" + localid: "0" + nnodes: "1" + nodeid: "0" + nodelist: auh7-1b-gpu-306 + nprocs: "1" + ntasks: "1" + ntasks_per_node: "1" + oom_kill_step: "0" + prio_process: "0" + procid: "0" + submit_dir: /vast/users/xiaodan/zhangjian/A1/launch_scripts + submit_host: auh-1b-cpu-login-001 + task_pid: "594412" + tasks_per_node: "1" + topology_addr: auh7-1b-gpu-306 + topology_addr_pattern: node + startedAt: "2025-09-30T09:01:55.101472Z" + writerId: e39r496xjes4qj7ky2l3e9tlyr84a0v3 + m: [] + python_version: 3.10.18 + t: + "1": + - 1 + - 2 + - 3 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 63 + - 71 + - 83 + - 95 + - 105 + "2": + - 1 + - 2 + - 3 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 63 + - 71 + - 83 + - 95 + - 105 + "3": + - 13 + - 15 + - 16 + "4": 3.10.18 + "5": 0.21.4 + "6": 4.56.1 + "12": 0.21.4 + "13": linux-x86_64 +activation_checkpointing: + value: whole_layer +allow_resume: + value: false +batch_divisor: + value: global_batch +canceled_check_interval: + value: 50 +checkpoint_dir: + value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 +compile: + value: null +console_log_interval: + value: 1 +data: + value: + dataset: vla_dataset_realworld + drop_last: true + for_inference: false + lerobot_episode_index_end: null + lerobot_episode_index_start: null + mixture: null + multi_modal: torch + num_workers: 0 + pad: to_max + persistent_workers: false + pin_memory: true + prefetch_factor: null + rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds + rlds_dataset_name: libero_4_task_suites_no_noops + rlds_read_threads: 8 + rlds_shuffle_buffer_size: 100000 + rlds_traj_threads: 8 + root_size_mixture: null + seed: 95818 + sequence_length: 1600 + shuffle: true + shuffle_messages: false + split: train + timeout: 0 + use_proprio: true + use_wrist_image: true +device_eval_batch_size: + value: 4 +device_inf_eval_batch_size: + value: 16 +device_train_batch_size: + value: 15 +device_train_grad_accum: + value: 0 +device_train_microbatch_size: + value: 16 +dry_run: + value: false +early_exit: + value: false +epoch: + value: null +eval_interval: + value: 0 +eval_on_load: + value: false +eval_subset_num_batches: + value: -1 +evaluators: + value: + - data: + dataset: vla_dataset_realworld + drop_last: true + for_inference: false + lerobot_episode_index_end: 765 + lerobot_episode_index_start: 353 + mixture: null + multi_modal: torch + num_workers: 0 + pad: to_max + persistent_workers: true + pin_memory: true + prefetch_factor: null + rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds + rlds_dataset_name: libero_4_task_suites_no_noops + rlds_read_threads: 8 + rlds_shuffle_buffer_size: 256000 + rlds_traj_threads: 8 + root_size_mixture: null + seed: null + sequence_length: 1600 + shuffle: false + shuffle_messages: false + split: validation + timeout: 0 + use_proprio: true + use_wrist_image: true + device_eval_batch_size: null + eval_name: null + label: val + max_examples: null + max_new_tokens: 448 + mm_evaluator: null + save_dir: null + save_to_checkpoint_dir: false + skip_if_metrics_cached: true + subset_num_batches: 64 +extra_steps_after_cancel: + value: 10 +fast_forward_batches: + value: null +force_save_unsharded: + value: false +fsdp: + value: + hybrid_sharding_num_model_replicas: null + precision: float + sharding_strategy: FULL_SHARD + use_orig_params: true + wrapping_strategy: by_block_and_size +ft_connector: + value: false +ft_embedding: + value: lm_head +ft_llm: + value: true +ft_vit: + value: false +fused_loss: + value: null +gen1_gc_interval: + value: 1 +global_train_batch_size: + value: 126 +inf_eval_interval: + value: -1 +inf_eval_subset_num_batches: + value: -1 +inf_evaluators: + value: [] +initial_model_checkpoint: + value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 +keep_lr_on_load: + value: true +load_model_config: + value: null +load_path: + value: null +load_path_sharded_checkpointer: + value: null +lora: + value: false +lora_connector: + value: false +lora_llm: + value: false +lora_rank: + value: 8 +lora_vit: + value: false +max_duration: + value: 500000 +max_grad_norm: + value: 1 +max_grad_norm_ratio: + value: null +model: + value: + action_dim: 7 + action_head: l1_regression + action_head_dit_depth: 28 + action_head_dit_hidden_size: 1152 + action_head_dit_num_heads: 16 + action_tokenizer: + identifier: physical-intelligence/fast + tokenizer_dir: null + action_use_left_eef: false + action_use_mobile_base: false + activation_type: swiglu + additional_vocab_size: 128 + always_start_with_space: true + attention_dropout: 0 + attention_layer_norm: false + attention_layer_norm_with_affine: true + attention_type: sdpa + bias_for_layer_norm: null + block_group_size: 1 + block_type: sequential + clip_qkv: null + crop_mode: overlap-and-resize-c2 + d_model: 3584 + default_inference_len: 65 + embedding_dropout: 0 + embedding_size: 152064 + ff_out_size: 0 + fix_image_padding: true + float32_attention: true + head_dim: null + horizon: 8 + image_feature_dropout: 0 + image_padding_embed: pad_and_partial_pad + image_pooling_2d: attention_meanq + image_pooling_h: 2 + image_pooling_w: 2 + image_projector: mlp + include_bias: false + init_cutoff_factor: null + init_device: null + init_fn: normal + init_std: 0.02 + initializer_range: 0.02 + layer_norm_eps: 1e-06 + layer_norm_type: rms + layer_norm_with_affine: true + llm_causal_attention: false + llm_load_path: /weka/oe-training-default/mm-olmo/pretrained_llms/qwen2-7b.pt + low_cpu_fsdp: true + max_crops: 12 + max_position_embeddings: null + max_sequence_length: 4096 + message_formatting: role + mlp_hidden_size: 37888 + mlp_ratio: 4 + moe_capacity_factor: 1.25 + moe_dropless: true + moe_interleave: false + moe_lbl_in_fp32: false + moe_log_expert_assignment: false + moe_loss_weight: 0.1 + moe_mlp_impl: sparse + moe_num_experts: 8 + moe_shared_expert: false + moe_top_k: 2 + moe_zloss_weight: null + multi_annotation_weighting: root_subsegments + n_heads: 28 + n_kv_heads: 4 + n_layers: 28 + new_embedding_init_range: 0.02 + norm_after: false + normalize_input_embeds: false + num_diffusion_inference_steps: 30 + num_diffusion_steps: 1000 + overlap_margins: + - 4 + - 4 + pad_tokenizer: true + pad_value: 0 + precision: amp_bf16 + prompt_type: uber_model + qkv_bias: true + residual_dropout: 0.1 + response_residual_dropout: 0 + rope: true + rope_full_precision: true + rope_theta: 1e+06 + scale_logits: false + system_prompt_kind: demo_or_style + tokenizer: + identifier: Qwen/Qwen2-7B + tokenizer_dir: null + use_col_tokens: true + use_position_ids: true + use_proprio: true + vision_backbone: + attention_dropout: 0 + fsdp_wrap: false + image_default_input_size: + - 336 + - 336 + image_dropout_rate: 0 + image_emb_dim: 1024 + image_head_dim: 64 + image_mlp_activations: quick_gelu + image_mlp_dim: 4096 + image_model_type: openai + image_norm_eps: 1e-05 + image_num_heads: 16 + image_num_key_value_heads: 16 + image_num_layers: 23 + image_num_pos: 577 + image_patch_size: 14 + image_pos_patch_size: 14 + initializer_range: 0.02 + residual_dropout: 0 + resize_mode: default + vit_layers: + - -2 + - -9 + vit_load_path: /weka/oe-training-default/mm-olmo/pretrained_image_encoders/vit-l-14-336.pt + vocab_size: 152064 + weight_tying: false +multi_component_grad_norm: + value: true +no_pre_train_checkpoint: + value: true +optimizer: + value: + betas: + - 0.9 + - 0.95 + connector_betas: + - 0.9 + - 0.95 + connector_eps: 1e-06 + connector_learning_rate: 0.0002 + connector_weight_decay: 0 + eps: 1e-05 + learning_rate: 0.0001 + llm_betas: + - 0.9 + - 0.95 + llm_eps: 1e-06 + llm_learning_rate: 5e-05 + llm_weight_decay: 0 + metrics_log_interval: 20 + name: adamw + vit_betas: + - 0.9 + - 0.95 + vit_eps: 1e-06 + vit_learning_rate: 6e-06 + vit_weight_decay: 0 + weight_decay: 0.01 +precision: + value: amp_bf16 +python_profiling: + value: false +remote_save_folder: + value: null +reset_dataloader_state: + value: false +reset_optimizer_state: + value: false +reset_trainer_state: + value: false +restore_dataloader: + value: true +run_name: + value: realworld_20250930_090124 +save_dataloader_state: + value: false +save_folder: + value: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1 +save_interval: + value: 500 +save_interval_action_head: + value: 500 +save_interval_ephemeral: + value: null +save_interval_unsharded: + value: 500 +save_num_action_head_checkpoints_to_keep: + value: 2 +save_num_checkpoints_to_keep: + value: 1 +save_num_unsharded_checkpoints_to_keep: + value: 1 +save_overwrite: + value: true +scheduler: + value: + alpha_f: 0.1 + connector_t_warmup: 200 + grad_clip_warmup_factor: null + grad_clip_warmup_steps: null + llm_t_warmup: 2000 + name: multimodal + t_max: null + t_warmup: 100 + units: steps + vit_t_warmup: 2000 + warmup_min_lr: 0 +seed: + value: 6198 +sharded_checkpointer: + value: torch_legacy +softmax_auxiliary_loss: + value: true +softmax_auxiliary_loss_scale: + value: 0.0001 +speed_monitor: + value: + gpu_flops_available: null + window_size: 20 +stop_after: + value: null +stop_at: + value: 500000 +time_limit: + value: null +torch_profiling: + value: false +train_exit_random_layer: + value: false +use_lora: + value: true diff --git a/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/output.log b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..db1e3d2bf6ab35e4dc4f3055481e93e63d6b2d67 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/output.log @@ -0,0 +1,85 @@ +wandb: Detected [openai] in use. +wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script. +wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/ +09/30 [09:01:56] WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No warnings.py:109 + device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once + +****** vla_cfg: {'datasets': {'rlds': {'name': None, 'path': None, 'weight': 1.0, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe', 1, 'bounds']], 'open-source-real-world': {'rlds': {'name': 'a1_real_world', 'path': None, 'weight': 8, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [], 'agibot': {'path': '/vast/users/xiaodan/zhangjian/datasets/AgiBotWorld-Alpha', 'weight': 8, 'action_proprio_normalization_type': None}}}, 'model': {'action_head': {'action_dim': 16, 'proprio_dim': 16, 'num_actions_chunk': 8, 'action_tokens_mapping': {'left_end_effector': 8, 'right_end_effector': 8}, 'use_left_eef': True, 'use_mobile_base': False}}} +****** Skip RLDS main; path not found: None +****** start build LeRobot main... +build_tokenizer, cache_dir None tokenizer_dir None +09/30 [09:01:58] INFO | >> Padding tokenizer with 418 tokens tokenizer.py:130 + INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk +****** length of the dataset: 72641 +09/30 [09:02:07] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50 +****** length of the dataset: 27906 +09/30 [09:02:09] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser +****** length of the dataset: 13441 +09/30 [09:02:10] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue +****** length of the dataset: 10316 +09/30 [09:02:11] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen +****** length of the dataset: 17131 +09/30 [09:02:12] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote +****** length of the dataset: 15765 +09/30 [09:02:13] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue +****** length of the dataset: 90 +09/30 [09:02:14] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe +****** length of the dataset: 18397 +****** Skip RLDS open-source-real-world; path not found: None +****** path: /vast/users/xiaodan/zhangjian/datasets/AgiBotWorld-Alpha +****** before AgiBotWorldAlpha dataset... +09/30 [09:02:15] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:485 +Traceback (most recent call last): + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/compat/_optional.py", line 135, in import_optional_dependency + module = importlib.import_module(name) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/importlib/__init__.py", line 126, in import_module + return _bootstrap._gcd_import(name[level:], package, level) + File "", line 1050, in _gcd_import + File "", line 1027, in _find_and_load + File "", line 1004, in _find_and_load_unlocked +ModuleNotFoundError: No module named 'openpyxl' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py", line 397, in + train(cfg) + File "/vast/users/xiaodan/zhangjian/A1/scripts/train_for_action.py", line 160, in main + train_loader = build_train_dataloader(cfg, device) + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/__init__.py", line 197, in build_train_dataloader + return build_vla_train_dataloader(train_config, device) + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/__init__.py", line 384, in build_vla_train_dataloader + ds = build_agibot_train_dataset(train_config, normalization_type, device) + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/__init__.py", line 487, in build_agibot_train_dataset + dataset = AgiBotWorldAlphaDataset( + File "", line 13, in __init__ + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/vla/agibot_dataset.py", line 145, in __post_init__ + self._frame_ranges_map: Optional[Dict[Tuple[str, int], Tuple[int, int]]] = self._load_frame_ranges_excel() + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/vla/agibot_dataset.py", line 411, in _load_frame_ranges_excel + df = pd.read_excel(excel_path) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/io/excel/_base.py", line 495, in read_excel + io = ExcelFile( + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/io/excel/_base.py", line 1567, in __init__ + self._reader = self._engines[engine]( + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/io/excel/_openpyxl.py", line 552, in __init__ + import_optional_dependency("openpyxl") + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/compat/_optional.py", line 138, in import_optional_dependency + raise ImportError(msg) +ImportError: Missing optional dependency 'openpyxl'. Use pip or conda to install openpyxl. diff --git a/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/requirements.txt b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..291a344502cb68f8ac9dcf95e2d7e2df5f47b9b6 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/requirements.txt @@ -0,0 +1,284 @@ +ai2-molmo==0.0.0 +astunparse==1.6.3 +flatbuffers==25.2.10 +gast==0.6.0 +google-pasta==0.2.0 +h5py==3.14.0 +libclang==18.1.1 +Markdown==3.9 +namex==0.1.0 +opt_einsum==3.4.0 +optree==0.17.0 +tensorboard-data-server==0.7.2 +tensorflow-io-gcs-filesystem==0.37.1 +termcolor==3.1.0 +Werkzeug==3.1.3 +Brotli==1.1.0 +Farama-Notifications==0.0.4 +MarkupSafe==2.1.5 +PyYAML==6.0.2 +absl-py==2.3.1 +accelerate==1.10.1 +ai2-molmo==0.0.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +antlr4-python3-runtime==4.9.3 +anyio==4.10.0 +array_record==0.8.1 +async-timeout==5.0.1 +attrs==25.3.0 +av==15.1.0 +backports.tarfile==1.2.0 +beaker-gantry==3.2.0 +beaker-py==2.5.0 +black==23.12.1 +blinker==1.9.0 +boltons==25.0.0 +boto3==1.40.33 +botocore==1.40.33 +build==1.3.0 +cached_path==1.7.3 +cached-property==2.0.1 +cachetools==5.5.2 +certifi==2025.8.3 +cffi==2.0.0 +charset-normalizer==3.4.3 +click==8.2.1 +click-help-colors==0.9.4 +click-option-group==0.5.7 +cloudpickle==3.1.1 +cmake==4.1.0 +contourpy==1.3.2 +cryptography==46.0.1 +cycler==0.12.1 +dataclass-extensions==0.2.3 +datasets==3.6.0 +decorator==5.2.1 +deepdiff==8.6.1 +diffusers==0.35.1 +dill==0.3.8 +distro==1.9.0 +dlimp==0.0.1 +dm-tree==0.1.9 +docutils==0.22.1 +draccus==0.10.0 +editdistance==0.8.1 +einops==0.8.1 +einops-exts==0.0.4 +etils==1.13.0 +evdev==1.9.2 +exceptiongroup==1.3.0 +face==24.0.0 +fastapi==0.116.2 +ffmpy==0.6.1 +fiddle==0.3.0 +filelock==3.13.1 +Flask==3.1.2 +fonttools==4.60.0 +frozenlist==1.7.0 +fsspec==2023.9.2 +ftfy==6.3.1 +gcsfs==2023.9.2 +gitdb==4.0.12 +GitPython==3.1.45 +glom==24.11.0 +google-api-core==2.25.1 +google-auth==2.40.3 +google-auth-oauthlib==1.2.2 +google-cloud-core==2.4.3 +google-cloud-storage==2.19.0 +google-crc32c==1.7.1 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +gradio==5.46.0 +gradio_client==1.13.0 +graphviz==0.21 +groovy==0.1.2 +grpcio==1.75.0 +gymnasium==0.29.1 +h11==0.16.0 +hf_transfer==0.1.9 +hf-xet==1.1.10 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==0.35.0 +id==1.5.0 +idna==3.10 +imageio==2.37.0 +imageio-ffmpeg==0.6.0 +importlib_metadata==8.7.0 +importlib_resources==6.5.2 +iniconfig==2.1.0 +inquirerpy==0.3.4 +isort==5.12.0 +itsdangerous==2.2.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.3.0 +jeepney==0.9.0 +Jinja2==3.1.4 +jiter==0.11.0 +jmespath==1.0.1 +joblib==1.5.2 +jsonlines==4.0.0 +keras==2.15.0 +keyring==25.6.0 +kiwisolver==1.4.9 +latex2sympy2_extended==1.10.2 +lerobot==0.3.4 +Levenshtein==0.27.1 +libcst==1.8.4 +lightning-utilities==0.15.2 +markdown-it-py==4.0.0 +math-verify==0.8.0 +matplotlib==3.10.6 +mdurl==0.1.2 +mergedeep==1.3.4 +ml-dtypes==0.2.0 +ml_dtypes==0.5.3 +more-itertools==10.8.0 +mpmath==1.3.0 +msgspec==0.19.0 +multidict==6.6.4 +multiprocess==0.70.16 +mypy==1.3.0 +mypy_extensions==1.1.0 +necessary==0.4.3 +networkx==3.3 +nh3==0.3.0 +nltk==3.9.1 +numpy==1.26.4 +oauthlib==3.3.1 +omegaconf==2.3.0 +openai==1.108.0 +opencv-python-headless==4.12.0.88 +OpenEXR==3.4.0 +orderly-set==5.5.0 +orjson==3.11.3 +packaging==25.0 +pandas==2.3.2 +pathspec==0.12.1 +petname==2.6 +pfzy==0.3.4 +pillow==11.0.0 +pip==25.2 +platformdirs==4.4.0 +pluggy==1.6.0 +promise==2.3 +prompt_toolkit==3.0.52 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==4.21.12 +protobuf==6.32.1 +psutil==7.1.0 +pyarrow==21.0.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.23 +pydantic==2.11.9 +pydantic_core==2.33.2 +pydub==0.25.1 +Pygments==2.19.2 +pynput==1.8.1 +pyparsing==3.2.4 +pyproject_hooks==1.2.0 +pyserial==3.5 +pytest==8.4.2 +pytest-sphinx==0.6.3 +python-dateutil==2.9.0.post0 +python-Levenshtein==0.27.1 +python-multipart==0.0.20 +python-xlib==0.33 +pytorch-triton-rocm==3.4.0 +pytz==2025.2 +pyyaml-include==1.4.1 +RapidFuzz==3.14.1 +readme_renderer==44.0 +regex==2025.9.1 +requests==2.32.5 +requests-oauthlib==2.0.0 +requests-toolbelt==1.0.0 +requirements-parser==0.13.0 +rerun-sdk==0.22.1 +rfc3986==2.0.0 +rich==13.9.4 +rsa==4.9.1 +ruff==0.13.0 +s3transfer==0.14.0 +safehttpx==0.1.6 +safetensors==0.6.2 +scikit-learn==1.7.2 +scipy==1.15.3 +SecretStorage==3.4.0 +semantic-version==2.10.0 +sentencepiece==0.2.1 +sentry-sdk==2.38.0 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.17.0 +smart_open==7.3.1 +smashed==0.21.5 +smmap==5.0.2 +sniffio==1.3.1 +starlette==0.48.0 +sympy==1.13.3 +tensorboard==2.15.2 +tensorboard==2.19.0 +tensorflow==2.15.0 +tensorflow-addons==0.23.0 +tensorflow-datasets==4.9.3 +tensorflow-estimator==2.15.0 +tensorflow-graphics==2021.12.3 +tensorflow-metadata==1.17.2 +threadpoolctl==3.6.0 +timm==1.0.19 +tokenizers==0.22.0 +toml==0.10.2 +tomli==2.2.1 +tomlkit==0.13.3 +torch==2.8.0+rocm6.4 +torchcodec==0.5 +torchmetrics==1.8.2 +torchvision==0.23.0+rocm6.4 +tqdm==4.67.1 +transformers==4.56.1 +trimesh==4.8.2 +trouting==0.3.3 +twine==6.2.0 +typeguard==2.13.3 +typer==0.17.4 +typing_extensions==4.15.0 +typing-inspect==0.9.0 +typing-inspection==0.4.1 +tzdata==2025.2 +urllib3==2.5.0 +uvicorn==0.35.0 +wandb==0.21.4 +wcwidth==0.2.13 +websockets==15.0.1 +wheel==0.45.1 +wrapt==1.14.2 +xxhash==3.5.0 +yarl==1.20.1 +zipp==3.23.0 +lerobot==0.3.4 +minLoRA==0.1.0 +autocommand==2.2.2 +backports.tarfile==1.2.0 +importlib_metadata==8.0.0 +inflect==7.3.1 +jaraco.collections==5.1.0 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +jaraco.text==3.12.1 +more-itertools==10.3.0 +packaging==24.2 +platformdirs==4.2.2 +tomli==2.0.1 +typeguard==4.3.0 +typing_extensions==4.12.2 +wheel==0.45.1 +zipp==3.19.2 diff --git a/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/wandb-metadata.json b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ef09c5afb76a3b20ff17fa87d97ebe18d8dec765 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/wandb-metadata.json @@ -0,0 +1,202 @@ +{ + "os": "Linux-5.15.0-140-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.18", + "startedAt": "2025-09-30T09:01:55.101472Z", + "args": [ + "qwen2_7b", + "save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1", + "--vision_backbone", + "openai", + "--action_head", + "l1_regression", + "--seq_len", + "1600", + "--ft_llm", + "--checkpoint", + "/vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924", + "--device_train_microbatch_size", + "16", + "--global_batch_size", + "126", + "--dataset", + "vla_dataset_realworld", + "--llm_learning_rate", + "5e-5", + "--wandb_entity", + "henryeap", + "--wandb_project", + "a1-realworld", + "--wandb_run_name", + "realworld", + "--save_overwrite" + ], + "program": "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py", + "codePath": "launch_scripts/train_vla.py", + "codePathLocal": "launch_scripts/train_vla.py", + "git": { + "remote": "https://github.com/Spatialtemporal-AI/A1.git", + "commit": "f2afcc15e05f491a8e50add64395fc1db0a1188d" + }, + "email": "ihenrykwok@outlook.com", + "root": "/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb", + "host": "auh7-1b-gpu-306", + "executable": "/vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10", + "cpu_count": 64, + "cpu_count_logical": 128, + "gpu": "Instinct MI210", + "gpu_count": 8, + "disk": { + "/": { + "total": "470343073792", + "used": "50128465920" + } + }, + "memory": { + "total": "2434611519488" + }, + "gpu_amd": [ + { + "id": "6", + "uniqueId": "0x12140cd9e24f12e9", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "3", + "uniqueId": "0x95be8fdc770fcfd7", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "7", + "uniqueId": "0xa0442ab3bdd405c1", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "4", + "uniqueId": "0x24ee801b7c402006", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "2", + "uniqueId": "0xaabcddaa244a3d6e", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "0", + "uniqueId": "0x82728d7f9bd937e4", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "5", + "uniqueId": "0x413935505e32b8da", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "1", + "uniqueId": "0x27087f06439a527d", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + } + ], + "slurm": { + "cluster_name": "ai-04r", + "conf": "/etc/slurm/slurm.conf", + "cpus_on_node": "128", + "gpus_on_node": "8", + "gtids": "0", + "job_account": "faculty-acc", + "job_cpus_per_node": "128", + "job_end_time": "1759482071", + "job_gid": "2000", + "job_gpus": "0,1,2,3,4,5,6,7", + "job_id": "1973", + "job_name": "mh_realworld", + "job_nodelist": "auh7-1b-gpu-306", + "job_num_nodes": "1", + "job_partition": "faculty", + "job_qos": "xdqos", + "job_start_time": "1759222871", + "job_uid": "2013", + "job_user": "xiaodan", + "jobid": "1973", + "localid": "0", + "nnodes": "1", + "nodeid": "0", + "nodelist": "auh7-1b-gpu-306", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "oom_kill_step": "0", + "prio_process": "0", + "procid": "0", + "submit_dir": "/vast/users/xiaodan/zhangjian/A1/launch_scripts", + "submit_host": "auh-1b-cpu-login-001", + "task_pid": "594412", + "tasks_per_node": "1", + "topology_addr": "auh7-1b-gpu-306", + "topology_addr_pattern": "node" + }, + "writerId": "e39r496xjes4qj7ky2l3e9tlyr84a0v3" +} \ No newline at end of file diff --git a/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/wandb-summary.json b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ba532d630a2f6e9086432ef98a4e1304e8ad3f55 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":19,"_wandb":{"runtime":19}} \ No newline at end of file diff --git a/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/logs/debug-core.log b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..eb4ed164e5ec7ad85a29c2d981a6e70aab817ef6 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2025-09-30T09:01:55.162904331Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmprwjho3ja/port-594589.txt","pid":594589,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-09-30T09:01:55.164089452Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":594589} +{"time":"2025-09-30T09:01:55.164075022Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-594589-594975-1877083663/socket","Net":"unix"}} +{"time":"2025-09-30T09:01:55.335963017Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-09-30T09:01:55.342714526Z","level":"INFO","msg":"handleInformInit: received","streamId":"w2yi62pb","id":"1(@)"} +{"time":"2025-09-30T09:01:56.270439585Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"w2yi62pb","id":"1(@)"} +{"time":"2025-09-30T09:02:15.777414547Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2025-09-30T09:02:15.777645561Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2025-09-30T09:02:15.777654891Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-09-30T09:02:15.777680572Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2025-09-30T09:02:15.778959994Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-594589-594975-1877083663/socket","Net":"unix"}} +{"time":"2025-09-30T09:02:17.890949248Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2025-09-30T09:02:17.890967089Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2025-09-30T09:02:17.890978859Z","level":"INFO","msg":"server is closed"} diff --git a/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/logs/debug-internal.log b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..9a15c9d9796c266949732881b3c2ca4c286b8f5c --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2025-09-30T09:01:55.344751712Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-09-30T09:01:56.270392104Z","level":"INFO","msg":"stream: created new stream","id":"w2yi62pb"} +{"time":"2025-09-30T09:01:56.270434675Z","level":"INFO","msg":"stream: started","id":"w2yi62pb"} +{"time":"2025-09-30T09:01:56.270440165Z","level":"INFO","msg":"writer: started","stream_id":"w2yi62pb"} +{"time":"2025-09-30T09:01:56.270445835Z","level":"INFO","msg":"handler: started","stream_id":"w2yi62pb"} +{"time":"2025-09-30T09:01:56.270482246Z","level":"INFO","msg":"sender: started","stream_id":"w2yi62pb"} +{"time":"2025-09-30T09:02:15.777655041Z","level":"INFO","msg":"stream: closing","id":"w2yi62pb"} +{"time":"2025-09-30T09:02:17.562495857Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-09-30T09:02:17.888649918Z","level":"INFO","msg":"handler: closed","stream_id":"w2yi62pb"} +{"time":"2025-09-30T09:02:17.889765777Z","level":"INFO","msg":"sender: closed","stream_id":"w2yi62pb"} +{"time":"2025-09-30T09:02:17.889789108Z","level":"INFO","msg":"stream: closed","id":"w2yi62pb"} diff --git a/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/logs/debug.log b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f3f5468fff145d700a46bd183b6314a3c5c2acbd --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/logs/debug.log @@ -0,0 +1 @@ +2025-09-30 09:02:15,777 INFO wandb-AsyncioManager-main:594589 [service_client.py:_forward_responses():84] Reached EOF. diff --git a/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/run-w2yi62pb.wandb b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/run-w2yi62pb.wandb new file mode 100644 index 0000000000000000000000000000000000000000..aee1b3147d9acd103d95275cd9b078df05a4710e Binary files /dev/null and b/all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/run-w2yi62pb.wandb differ diff --git a/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/files/config.yaml b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b71b52fd7e20a10e4f53101b5646e30ec536c373 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/files/config.yaml @@ -0,0 +1,617 @@ +_wandb: + value: + cli_version: 0.21.4 + e: + esn3ubylhv9g1ubwtuf03ejgtuan58ph: + args: + - qwen2_7b + - save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1 + - --vision_backbone + - openai + - --action_head + - l1_regression + - --seq_len + - "1600" + - --ft_llm + - --checkpoint + - /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 + - --device_train_microbatch_size + - "16" + - --global_batch_size + - "126" + - --dataset + - vla_dataset_realworld + - --llm_learning_rate + - "5e-5" + - --wandb_entity + - henryeap + - --wandb_project + - a1-realworld + - --wandb_run_name + - realworld + - --save_overwrite + codePath: launch_scripts/train_vla.py + codePathLocal: launch_scripts/train_vla.py + cpu_count: 64 + cpu_count_logical: 128 + disk: + /: + total: "470343073792" + used: "50128633856" + email: ihenrykwok@outlook.com + executable: /vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10 + git: + commit: f2afcc15e05f491a8e50add64395fc1db0a1188d + remote: https://github.com/Spatialtemporal-AI/A1.git + gpu: Instinct MI210 + gpu_amd: + - id: "1" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x27087f06439a527d" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "7" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xa0442ab3bdd405c1" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "3" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x95be8fdc770fcfd7" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "4" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x24ee801b7c402006" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "0" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x82728d7f9bd937e4" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "5" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x413935505e32b8da" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "2" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xaabcddaa244a3d6e" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "6" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x12140cd9e24f12e9" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + gpu_count: 8 + host: auh7-1b-gpu-306 + memory: + total: "2434611519488" + os: Linux-5.15.0-140-generic-x86_64-with-glibc2.35 + program: /vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py + python: CPython 3.10.18 + root: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb + slurm: + cluster_name: ai-04r + conf: /etc/slurm/slurm.conf + cpus_on_node: "128" + gpus_on_node: "8" + gtids: "0" + job_account: faculty-acc + job_cpus_per_node: "128" + job_end_time: "1759489779" + job_gid: "2000" + job_gpus: 0,1,2,3,4,5,6,7 + job_id: "1988" + job_name: mh_realworld + job_nodelist: auh7-1b-gpu-306 + job_num_nodes: "1" + job_partition: faculty + job_qos: xdqos + job_start_time: "1759230579" + job_uid: "2013" + job_user: xiaodan + jobid: "1988" + localid: "0" + nnodes: "1" + nodeid: "0" + nodelist: auh7-1b-gpu-306 + nprocs: "1" + ntasks: "1" + ntasks_per_node: "1" + oom_kill_step: "0" + prio_process: "0" + procid: "0" + submit_dir: /vast/users/xiaodan/zhangjian/A1/launch_scripts + submit_host: auh-1b-cpu-login-001 + task_pid: "604425" + tasks_per_node: "1" + topology_addr: auh7-1b-gpu-306 + topology_addr_pattern: node + startedAt: "2025-09-30T11:10:22.173666Z" + writerId: esn3ubylhv9g1ubwtuf03ejgtuan58ph + m: [] + python_version: 3.10.18 + t: + "1": + - 1 + - 2 + - 3 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 63 + - 71 + - 83 + - 95 + - 105 + "2": + - 1 + - 2 + - 3 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 63 + - 71 + - 83 + - 95 + - 105 + "3": + - 13 + - 15 + - 16 + "4": 3.10.18 + "5": 0.21.4 + "6": 4.56.1 + "12": 0.21.4 + "13": linux-x86_64 +activation_checkpointing: + value: whole_layer +allow_resume: + value: false +batch_divisor: + value: global_batch +canceled_check_interval: + value: 50 +checkpoint_dir: + value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 +compile: + value: null +console_log_interval: + value: 1 +data: + value: + dataset: vla_dataset_realworld + drop_last: true + for_inference: false + lerobot_episode_index_end: null + lerobot_episode_index_start: null + mixture: null + multi_modal: torch + num_workers: 0 + pad: to_max + persistent_workers: false + pin_memory: true + prefetch_factor: null + rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds + rlds_dataset_name: libero_4_task_suites_no_noops + rlds_read_threads: 8 + rlds_shuffle_buffer_size: 100000 + rlds_traj_threads: 8 + root_size_mixture: null + seed: 95818 + sequence_length: 1600 + shuffle: true + shuffle_messages: false + split: train + timeout: 0 + use_proprio: true + use_wrist_image: true +device_eval_batch_size: + value: 4 +device_inf_eval_batch_size: + value: 16 +device_train_batch_size: + value: 15 +device_train_grad_accum: + value: 0 +device_train_microbatch_size: + value: 16 +dry_run: + value: false +early_exit: + value: false +epoch: + value: null +eval_interval: + value: 0 +eval_on_load: + value: false +eval_subset_num_batches: + value: -1 +evaluators: + value: + - data: + dataset: vla_dataset_realworld + drop_last: true + for_inference: false + lerobot_episode_index_end: 765 + lerobot_episode_index_start: 353 + mixture: null + multi_modal: torch + num_workers: 0 + pad: to_max + persistent_workers: true + pin_memory: true + prefetch_factor: null + rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds + rlds_dataset_name: libero_4_task_suites_no_noops + rlds_read_threads: 8 + rlds_shuffle_buffer_size: 256000 + rlds_traj_threads: 8 + root_size_mixture: null + seed: null + sequence_length: 1600 + shuffle: false + shuffle_messages: false + split: validation + timeout: 0 + use_proprio: true + use_wrist_image: true + device_eval_batch_size: null + eval_name: null + label: val + max_examples: null + max_new_tokens: 448 + mm_evaluator: null + save_dir: null + save_to_checkpoint_dir: false + skip_if_metrics_cached: true + subset_num_batches: 64 +extra_steps_after_cancel: + value: 10 +fast_forward_batches: + value: null +force_save_unsharded: + value: false +fsdp: + value: + hybrid_sharding_num_model_replicas: null + precision: float + sharding_strategy: FULL_SHARD + use_orig_params: true + wrapping_strategy: by_block_and_size +ft_connector: + value: false +ft_embedding: + value: lm_head +ft_llm: + value: true +ft_vit: + value: false +fused_loss: + value: null +gen1_gc_interval: + value: 1 +global_train_batch_size: + value: 126 +inf_eval_interval: + value: -1 +inf_eval_subset_num_batches: + value: -1 +inf_evaluators: + value: [] +initial_model_checkpoint: + value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 +keep_lr_on_load: + value: true +load_model_config: + value: null +load_path: + value: null +load_path_sharded_checkpointer: + value: null +lora: + value: false +lora_connector: + value: false +lora_llm: + value: false +lora_rank: + value: 8 +lora_vit: + value: false +max_duration: + value: 500000 +max_grad_norm: + value: 1 +max_grad_norm_ratio: + value: null +model: + value: + action_dim: 7 + action_head: l1_regression + action_head_dit_depth: 28 + action_head_dit_hidden_size: 1152 + action_head_dit_num_heads: 16 + action_tokenizer: + identifier: physical-intelligence/fast + tokenizer_dir: null + action_use_left_eef: false + action_use_mobile_base: false + activation_type: swiglu + additional_vocab_size: 128 + always_start_with_space: true + attention_dropout: 0 + attention_layer_norm: false + attention_layer_norm_with_affine: true + attention_type: sdpa + bias_for_layer_norm: null + block_group_size: 1 + block_type: sequential + clip_qkv: null + crop_mode: overlap-and-resize-c2 + d_model: 3584 + default_inference_len: 65 + embedding_dropout: 0 + embedding_size: 152064 + ff_out_size: 0 + fix_image_padding: true + float32_attention: true + head_dim: null + horizon: 8 + image_feature_dropout: 0 + image_padding_embed: pad_and_partial_pad + image_pooling_2d: attention_meanq + image_pooling_h: 2 + image_pooling_w: 2 + image_projector: mlp + include_bias: false + init_cutoff_factor: null + init_device: null + init_fn: normal + init_std: 0.02 + initializer_range: 0.02 + layer_norm_eps: 1e-06 + layer_norm_type: rms + layer_norm_with_affine: true + llm_causal_attention: false + llm_load_path: /weka/oe-training-default/mm-olmo/pretrained_llms/qwen2-7b.pt + low_cpu_fsdp: true + max_crops: 12 + max_position_embeddings: null + max_sequence_length: 4096 + message_formatting: role + mlp_hidden_size: 37888 + mlp_ratio: 4 + moe_capacity_factor: 1.25 + moe_dropless: true + moe_interleave: false + moe_lbl_in_fp32: false + moe_log_expert_assignment: false + moe_loss_weight: 0.1 + moe_mlp_impl: sparse + moe_num_experts: 8 + moe_shared_expert: false + moe_top_k: 2 + moe_zloss_weight: null + multi_annotation_weighting: root_subsegments + n_heads: 28 + n_kv_heads: 4 + n_layers: 28 + new_embedding_init_range: 0.02 + norm_after: false + normalize_input_embeds: false + num_diffusion_inference_steps: 30 + num_diffusion_steps: 1000 + overlap_margins: + - 4 + - 4 + pad_tokenizer: true + pad_value: 0 + precision: amp_bf16 + prompt_type: uber_model + qkv_bias: true + residual_dropout: 0.1 + response_residual_dropout: 0 + rope: true + rope_full_precision: true + rope_theta: 1e+06 + scale_logits: false + system_prompt_kind: demo_or_style + tokenizer: + identifier: Qwen/Qwen2-7B + tokenizer_dir: null + use_col_tokens: true + use_position_ids: true + use_proprio: true + vision_backbone: + attention_dropout: 0 + fsdp_wrap: false + image_default_input_size: + - 336 + - 336 + image_dropout_rate: 0 + image_emb_dim: 1024 + image_head_dim: 64 + image_mlp_activations: quick_gelu + image_mlp_dim: 4096 + image_model_type: openai + image_norm_eps: 1e-05 + image_num_heads: 16 + image_num_key_value_heads: 16 + image_num_layers: 23 + image_num_pos: 577 + image_patch_size: 14 + image_pos_patch_size: 14 + initializer_range: 0.02 + residual_dropout: 0 + resize_mode: default + vit_layers: + - -2 + - -9 + vit_load_path: /weka/oe-training-default/mm-olmo/pretrained_image_encoders/vit-l-14-336.pt + vocab_size: 152064 + weight_tying: false +multi_component_grad_norm: + value: true +no_pre_train_checkpoint: + value: true +optimizer: + value: + betas: + - 0.9 + - 0.95 + connector_betas: + - 0.9 + - 0.95 + connector_eps: 1e-06 + connector_learning_rate: 0.0002 + connector_weight_decay: 0 + eps: 1e-05 + learning_rate: 0.0001 + llm_betas: + - 0.9 + - 0.95 + llm_eps: 1e-06 + llm_learning_rate: 5e-05 + llm_weight_decay: 0 + metrics_log_interval: 20 + name: adamw + vit_betas: + - 0.9 + - 0.95 + vit_eps: 1e-06 + vit_learning_rate: 6e-06 + vit_weight_decay: 0 + weight_decay: 0.01 +precision: + value: amp_bf16 +python_profiling: + value: false +remote_save_folder: + value: null +reset_dataloader_state: + value: false +reset_optimizer_state: + value: false +reset_trainer_state: + value: false +restore_dataloader: + value: true +run_name: + value: realworld_20250930_110952 +save_dataloader_state: + value: false +save_folder: + value: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1 +save_interval: + value: 500 +save_interval_action_head: + value: 500 +save_interval_ephemeral: + value: null +save_interval_unsharded: + value: 500 +save_num_action_head_checkpoints_to_keep: + value: 2 +save_num_checkpoints_to_keep: + value: 1 +save_num_unsharded_checkpoints_to_keep: + value: 1 +save_overwrite: + value: true +scheduler: + value: + alpha_f: 0.1 + connector_t_warmup: 200 + grad_clip_warmup_factor: null + grad_clip_warmup_steps: null + llm_t_warmup: 2000 + name: multimodal + t_max: null + t_warmup: 100 + units: steps + vit_t_warmup: 2000 + warmup_min_lr: 0 +seed: + value: 6198 +sharded_checkpointer: + value: torch_legacy +softmax_auxiliary_loss: + value: true +softmax_auxiliary_loss_scale: + value: 0.0001 +speed_monitor: + value: + gpu_flops_available: null + window_size: 20 +stop_after: + value: null +stop_at: + value: 500000 +time_limit: + value: null +torch_profiling: + value: false +train_exit_random_layer: + value: false +use_lora: + value: true diff --git a/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/files/output.log b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4cffad9d38480bae6f92c39855a7d292d0ab7a21 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/files/output.log @@ -0,0 +1,86 @@ +wandb: Detected [openai] in use. +wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script. +wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/ +09/30 [11:10:24] WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No warnings.py:109 + device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once + +****** vla_cfg: {'datasets': {'rlds': {'name': None, 'path': None, 'weight': 1.0, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe', 1, 'bounds']], 'open-source-real-world': {'rlds': {'name': 'a1_real_world', 'path': '/vast/users/xiaodan/zhangjian/datasets/OXE', 'weight': 8, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [], 'agibot': {'path': '/vast/users/xiaodan/zhangjian/datasets/AgiBotWorld-Alpha', 'weight': 8, 'action_proprio_normalization_type': None}}}, 'model': {'action_head': {'action_dim': 16, 'proprio_dim': 16, 'num_actions_chunk': 8, 'action_tokens_mapping': {'left_end_effector': 8, 'right_end_effector': 8}, 'use_left_eef': True, 'use_mobile_base': False}}} +****** Skip RLDS main; path not found: None +****** start build LeRobot main... +build_tokenizer, cache_dir None tokenizer_dir None +09/30 [11:10:30] INFO | >> Padding tokenizer with 418 tokens tokenizer.py:130 +09/30 [11:10:31] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk +****** length of the dataset: 72641 +09/30 [11:10:39] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50 +****** length of the dataset: 27906 +09/30 [11:10:45] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser +****** length of the dataset: 13441 +09/30 [11:10:47] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue +****** length of the dataset: 10316 +09/30 [11:10:48] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen +****** length of the dataset: 17131 +09/30 [11:10:50] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote +****** length of the dataset: 15765 + INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue +****** length of the dataset: 90 +09/30 [11:10:51] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe +****** length of the dataset: 18397 +****** Skip RLDS open-source-real-world; mixture 'a1_real_world' not found under: /vast/users/xiaodan/zhangjian/datasets/OXE +****** Expect one of: [] +****** path: /vast/users/xiaodan/zhangjian/datasets/AgiBotWorld-Alpha +****** before AgiBotWorldAlpha dataset... +09/30 [11:10:52] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:485 +Traceback (most recent call last): + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/compat/_optional.py", line 135, in import_optional_dependency + module = importlib.import_module(name) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/importlib/__init__.py", line 126, in import_module + return _bootstrap._gcd_import(name[level:], package, level) + File "", line 1050, in _gcd_import + File "", line 1027, in _find_and_load + File "", line 1004, in _find_and_load_unlocked +ModuleNotFoundError: No module named 'openpyxl' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py", line 398, in + train(cfg) + File "/vast/users/xiaodan/zhangjian/A1/scripts/train_for_action.py", line 160, in main + train_loader = build_train_dataloader(cfg, device) + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/__init__.py", line 197, in build_train_dataloader + return build_vla_train_dataloader(train_config, device) + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/__init__.py", line 384, in build_vla_train_dataloader + ds = build_agibot_train_dataset(train_config, normalization_type, device) + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/__init__.py", line 487, in build_agibot_train_dataset + dataset = AgiBotWorldAlphaDataset( + File "", line 13, in __init__ + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/vla/agibot_dataset.py", line 145, in __post_init__ + self._frame_ranges_map: Optional[Dict[Tuple[str, int], Tuple[int, int]]] = self._load_frame_ranges_excel() + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/vla/agibot_dataset.py", line 411, in _load_frame_ranges_excel + df = pd.read_excel(excel_path) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/io/excel/_base.py", line 495, in read_excel + io = ExcelFile( + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/io/excel/_base.py", line 1567, in __init__ + self._reader = self._engines[engine]( + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/io/excel/_openpyxl.py", line 552, in __init__ + import_optional_dependency("openpyxl") + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/compat/_optional.py", line 138, in import_optional_dependency + raise ImportError(msg) +ImportError: Missing optional dependency 'openpyxl'. Use pip or conda to install openpyxl. diff --git a/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/files/requirements.txt b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..291a344502cb68f8ac9dcf95e2d7e2df5f47b9b6 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/files/requirements.txt @@ -0,0 +1,284 @@ +ai2-molmo==0.0.0 +astunparse==1.6.3 +flatbuffers==25.2.10 +gast==0.6.0 +google-pasta==0.2.0 +h5py==3.14.0 +libclang==18.1.1 +Markdown==3.9 +namex==0.1.0 +opt_einsum==3.4.0 +optree==0.17.0 +tensorboard-data-server==0.7.2 +tensorflow-io-gcs-filesystem==0.37.1 +termcolor==3.1.0 +Werkzeug==3.1.3 +Brotli==1.1.0 +Farama-Notifications==0.0.4 +MarkupSafe==2.1.5 +PyYAML==6.0.2 +absl-py==2.3.1 +accelerate==1.10.1 +ai2-molmo==0.0.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +antlr4-python3-runtime==4.9.3 +anyio==4.10.0 +array_record==0.8.1 +async-timeout==5.0.1 +attrs==25.3.0 +av==15.1.0 +backports.tarfile==1.2.0 +beaker-gantry==3.2.0 +beaker-py==2.5.0 +black==23.12.1 +blinker==1.9.0 +boltons==25.0.0 +boto3==1.40.33 +botocore==1.40.33 +build==1.3.0 +cached_path==1.7.3 +cached-property==2.0.1 +cachetools==5.5.2 +certifi==2025.8.3 +cffi==2.0.0 +charset-normalizer==3.4.3 +click==8.2.1 +click-help-colors==0.9.4 +click-option-group==0.5.7 +cloudpickle==3.1.1 +cmake==4.1.0 +contourpy==1.3.2 +cryptography==46.0.1 +cycler==0.12.1 +dataclass-extensions==0.2.3 +datasets==3.6.0 +decorator==5.2.1 +deepdiff==8.6.1 +diffusers==0.35.1 +dill==0.3.8 +distro==1.9.0 +dlimp==0.0.1 +dm-tree==0.1.9 +docutils==0.22.1 +draccus==0.10.0 +editdistance==0.8.1 +einops==0.8.1 +einops-exts==0.0.4 +etils==1.13.0 +evdev==1.9.2 +exceptiongroup==1.3.0 +face==24.0.0 +fastapi==0.116.2 +ffmpy==0.6.1 +fiddle==0.3.0 +filelock==3.13.1 +Flask==3.1.2 +fonttools==4.60.0 +frozenlist==1.7.0 +fsspec==2023.9.2 +ftfy==6.3.1 +gcsfs==2023.9.2 +gitdb==4.0.12 +GitPython==3.1.45 +glom==24.11.0 +google-api-core==2.25.1 +google-auth==2.40.3 +google-auth-oauthlib==1.2.2 +google-cloud-core==2.4.3 +google-cloud-storage==2.19.0 +google-crc32c==1.7.1 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +gradio==5.46.0 +gradio_client==1.13.0 +graphviz==0.21 +groovy==0.1.2 +grpcio==1.75.0 +gymnasium==0.29.1 +h11==0.16.0 +hf_transfer==0.1.9 +hf-xet==1.1.10 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==0.35.0 +id==1.5.0 +idna==3.10 +imageio==2.37.0 +imageio-ffmpeg==0.6.0 +importlib_metadata==8.7.0 +importlib_resources==6.5.2 +iniconfig==2.1.0 +inquirerpy==0.3.4 +isort==5.12.0 +itsdangerous==2.2.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.3.0 +jeepney==0.9.0 +Jinja2==3.1.4 +jiter==0.11.0 +jmespath==1.0.1 +joblib==1.5.2 +jsonlines==4.0.0 +keras==2.15.0 +keyring==25.6.0 +kiwisolver==1.4.9 +latex2sympy2_extended==1.10.2 +lerobot==0.3.4 +Levenshtein==0.27.1 +libcst==1.8.4 +lightning-utilities==0.15.2 +markdown-it-py==4.0.0 +math-verify==0.8.0 +matplotlib==3.10.6 +mdurl==0.1.2 +mergedeep==1.3.4 +ml-dtypes==0.2.0 +ml_dtypes==0.5.3 +more-itertools==10.8.0 +mpmath==1.3.0 +msgspec==0.19.0 +multidict==6.6.4 +multiprocess==0.70.16 +mypy==1.3.0 +mypy_extensions==1.1.0 +necessary==0.4.3 +networkx==3.3 +nh3==0.3.0 +nltk==3.9.1 +numpy==1.26.4 +oauthlib==3.3.1 +omegaconf==2.3.0 +openai==1.108.0 +opencv-python-headless==4.12.0.88 +OpenEXR==3.4.0 +orderly-set==5.5.0 +orjson==3.11.3 +packaging==25.0 +pandas==2.3.2 +pathspec==0.12.1 +petname==2.6 +pfzy==0.3.4 +pillow==11.0.0 +pip==25.2 +platformdirs==4.4.0 +pluggy==1.6.0 +promise==2.3 +prompt_toolkit==3.0.52 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==4.21.12 +protobuf==6.32.1 +psutil==7.1.0 +pyarrow==21.0.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.23 +pydantic==2.11.9 +pydantic_core==2.33.2 +pydub==0.25.1 +Pygments==2.19.2 +pynput==1.8.1 +pyparsing==3.2.4 +pyproject_hooks==1.2.0 +pyserial==3.5 +pytest==8.4.2 +pytest-sphinx==0.6.3 +python-dateutil==2.9.0.post0 +python-Levenshtein==0.27.1 +python-multipart==0.0.20 +python-xlib==0.33 +pytorch-triton-rocm==3.4.0 +pytz==2025.2 +pyyaml-include==1.4.1 +RapidFuzz==3.14.1 +readme_renderer==44.0 +regex==2025.9.1 +requests==2.32.5 +requests-oauthlib==2.0.0 +requests-toolbelt==1.0.0 +requirements-parser==0.13.0 +rerun-sdk==0.22.1 +rfc3986==2.0.0 +rich==13.9.4 +rsa==4.9.1 +ruff==0.13.0 +s3transfer==0.14.0 +safehttpx==0.1.6 +safetensors==0.6.2 +scikit-learn==1.7.2 +scipy==1.15.3 +SecretStorage==3.4.0 +semantic-version==2.10.0 +sentencepiece==0.2.1 +sentry-sdk==2.38.0 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.17.0 +smart_open==7.3.1 +smashed==0.21.5 +smmap==5.0.2 +sniffio==1.3.1 +starlette==0.48.0 +sympy==1.13.3 +tensorboard==2.15.2 +tensorboard==2.19.0 +tensorflow==2.15.0 +tensorflow-addons==0.23.0 +tensorflow-datasets==4.9.3 +tensorflow-estimator==2.15.0 +tensorflow-graphics==2021.12.3 +tensorflow-metadata==1.17.2 +threadpoolctl==3.6.0 +timm==1.0.19 +tokenizers==0.22.0 +toml==0.10.2 +tomli==2.2.1 +tomlkit==0.13.3 +torch==2.8.0+rocm6.4 +torchcodec==0.5 +torchmetrics==1.8.2 +torchvision==0.23.0+rocm6.4 +tqdm==4.67.1 +transformers==4.56.1 +trimesh==4.8.2 +trouting==0.3.3 +twine==6.2.0 +typeguard==2.13.3 +typer==0.17.4 +typing_extensions==4.15.0 +typing-inspect==0.9.0 +typing-inspection==0.4.1 +tzdata==2025.2 +urllib3==2.5.0 +uvicorn==0.35.0 +wandb==0.21.4 +wcwidth==0.2.13 +websockets==15.0.1 +wheel==0.45.1 +wrapt==1.14.2 +xxhash==3.5.0 +yarl==1.20.1 +zipp==3.23.0 +lerobot==0.3.4 +minLoRA==0.1.0 +autocommand==2.2.2 +backports.tarfile==1.2.0 +importlib_metadata==8.0.0 +inflect==7.3.1 +jaraco.collections==5.1.0 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +jaraco.text==3.12.1 +more-itertools==10.3.0 +packaging==24.2 +platformdirs==4.2.2 +tomli==2.0.1 +typeguard==4.3.0 +typing_extensions==4.12.2 +wheel==0.45.1 +zipp==3.19.2 diff --git a/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/files/wandb-metadata.json b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..06b689a7e69499e3a72f95f8a37bb0d06f408597 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/files/wandb-metadata.json @@ -0,0 +1,202 @@ +{ + "os": "Linux-5.15.0-140-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.18", + "startedAt": "2025-09-30T11:10:22.173666Z", + "args": [ + "qwen2_7b", + "save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1", + "--vision_backbone", + "openai", + "--action_head", + "l1_regression", + "--seq_len", + "1600", + "--ft_llm", + "--checkpoint", + "/vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924", + "--device_train_microbatch_size", + "16", + "--global_batch_size", + "126", + "--dataset", + "vla_dataset_realworld", + "--llm_learning_rate", + "5e-5", + "--wandb_entity", + "henryeap", + "--wandb_project", + "a1-realworld", + "--wandb_run_name", + "realworld", + "--save_overwrite" + ], + "program": "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py", + "codePath": "launch_scripts/train_vla.py", + "codePathLocal": "launch_scripts/train_vla.py", + "git": { + "remote": "https://github.com/Spatialtemporal-AI/A1.git", + "commit": "f2afcc15e05f491a8e50add64395fc1db0a1188d" + }, + "email": "ihenrykwok@outlook.com", + "root": "/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb", + "host": "auh7-1b-gpu-306", + "executable": "/vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10", + "cpu_count": 64, + "cpu_count_logical": 128, + "gpu": "Instinct MI210", + "gpu_count": 8, + "disk": { + "/": { + "total": "470343073792", + "used": "50128633856" + } + }, + "memory": { + "total": "2434611519488" + }, + "gpu_amd": [ + { + "id": "1", + "uniqueId": "0x27087f06439a527d", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "7", + "uniqueId": "0xa0442ab3bdd405c1", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "3", + "uniqueId": "0x95be8fdc770fcfd7", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "4", + "uniqueId": "0x24ee801b7c402006", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "0", + "uniqueId": "0x82728d7f9bd937e4", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "5", + "uniqueId": "0x413935505e32b8da", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "2", + "uniqueId": "0xaabcddaa244a3d6e", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "6", + "uniqueId": "0x12140cd9e24f12e9", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + } + ], + "slurm": { + "cluster_name": "ai-04r", + "conf": "/etc/slurm/slurm.conf", + "cpus_on_node": "128", + "gpus_on_node": "8", + "gtids": "0", + "job_account": "faculty-acc", + "job_cpus_per_node": "128", + "job_end_time": "1759489779", + "job_gid": "2000", + "job_gpus": "0,1,2,3,4,5,6,7", + "job_id": "1988", + "job_name": "mh_realworld", + "job_nodelist": "auh7-1b-gpu-306", + "job_num_nodes": "1", + "job_partition": "faculty", + "job_qos": "xdqos", + "job_start_time": "1759230579", + "job_uid": "2013", + "job_user": "xiaodan", + "jobid": "1988", + "localid": "0", + "nnodes": "1", + "nodeid": "0", + "nodelist": "auh7-1b-gpu-306", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "oom_kill_step": "0", + "prio_process": "0", + "procid": "0", + "submit_dir": "/vast/users/xiaodan/zhangjian/A1/launch_scripts", + "submit_host": "auh-1b-cpu-login-001", + "task_pid": "604425", + "tasks_per_node": "1", + "topology_addr": "auh7-1b-gpu-306", + "topology_addr_pattern": "node" + }, + "writerId": "esn3ubylhv9g1ubwtuf03ejgtuan58ph" +} \ No newline at end of file diff --git a/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/files/wandb-summary.json b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..2809f7f6be2cd810753ae1188aee6dbc55b345f9 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/files/wandb-summary.json @@ -0,0 +1 @@ +{"_runtime":29,"_wandb":{"runtime":29}} \ No newline at end of file diff --git a/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/logs/debug-core.log b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..2f7de8e11512cadf2d69c36f1cfa18bc3cb53f9e --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/logs/debug-core.log @@ -0,0 +1,14 @@ +{"time":"2025-09-30T11:10:22.224998099Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpodvish0l/port-604602.txt","pid":604602,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-09-30T11:10:22.22560875Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":604602} +{"time":"2025-09-30T11:10:22.22558572Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-604602-604772-538545887/socket","Net":"unix"}} +{"time":"2025-09-30T11:10:22.408964708Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-09-30T11:10:22.415973911Z","level":"INFO","msg":"handleInformInit: received","streamId":"tyi87rpc","id":"1(@)"} +{"time":"2025-09-30T11:10:23.336017725Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"tyi87rpc","id":"1(@)"} +{"time":"2025-09-30T11:10:53.09639777Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2025-09-30T11:10:53.097801345Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-09-30T11:10:53.096660235Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2025-09-30T11:10:53.097854056Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2025-09-30T11:10:53.098180352Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-604602-604772-538545887/socket","Net":"unix"}} +{"time":"2025-09-30T11:10:55.079824912Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2025-09-30T11:10:55.079840642Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2025-09-30T11:10:55.079850132Z","level":"INFO","msg":"server is closed"} diff --git a/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/logs/debug-internal.log b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..dcaff13accce8758b19c1abb75f378c13336c34c --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/logs/debug-internal.log @@ -0,0 +1,11 @@ +{"time":"2025-09-30T11:10:22.417648611Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-09-30T11:10:23.335963304Z","level":"INFO","msg":"stream: created new stream","id":"tyi87rpc"} +{"time":"2025-09-30T11:10:23.336011325Z","level":"INFO","msg":"stream: started","id":"tyi87rpc"} +{"time":"2025-09-30T11:10:23.336024875Z","level":"INFO","msg":"writer: started","stream_id":"tyi87rpc"} +{"time":"2025-09-30T11:10:23.336032926Z","level":"INFO","msg":"sender: started","stream_id":"tyi87rpc"} +{"time":"2025-09-30T11:10:23.336095647Z","level":"INFO","msg":"handler: started","stream_id":"tyi87rpc"} +{"time":"2025-09-30T11:10:53.096617934Z","level":"INFO","msg":"stream: closing","id":"tyi87rpc"} +{"time":"2025-09-30T11:10:54.757731193Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-09-30T11:10:55.076398492Z","level":"INFO","msg":"handler: closed","stream_id":"tyi87rpc"} +{"time":"2025-09-30T11:10:55.078483378Z","level":"INFO","msg":"sender: closed","stream_id":"tyi87rpc"} +{"time":"2025-09-30T11:10:55.078509419Z","level":"INFO","msg":"stream: closed","id":"tyi87rpc"} diff --git a/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/logs/debug.log b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..16ba2eb808b0855afd61d4277ebbfcb90aeb9cf9 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/logs/debug.log @@ -0,0 +1 @@ +2025-09-30 11:10:53,096 INFO wandb-AsyncioManager-main:604602 [service_client.py:_forward_responses():84] Reached EOF. diff --git a/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/run-tyi87rpc.wandb b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/run-tyi87rpc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..fcec9484fd66434121d446b02b769f62db983c47 Binary files /dev/null and b/all_l1/wandb/wandb/run-20250930_111022-tyi87rpc/run-tyi87rpc.wandb differ diff --git a/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/files/config.yaml b/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c8d562f37e05268c721ceadaf57c54657a09e54 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/files/config.yaml @@ -0,0 +1,621 @@ +_wandb: + value: + cli_version: 0.21.4 + e: + lwz9u7135qzjvugxcx5lynr52j566znm: + args: + - qwen2_7b + - save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1 + - --vision_backbone + - openai + - --action_head + - l1_regression + - --seq_len + - "1600" + - --ft_llm + - --checkpoint + - /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 + - --device_train_microbatch_size + - "16" + - --global_batch_size + - "126" + - --dataset + - vla_dataset_realworld + - --llm_learning_rate + - "5e-5" + - --wandb_entity + - henryeap + - --wandb_project + - a1-realworld + - --wandb_run_name + - realworld + - --save_overwrite + codePath: launch_scripts/train_vla.py + codePathLocal: launch_scripts/train_vla.py + cpu_count: 64 + cpu_count_logical: 128 + disk: + /: + total: "470343073792" + used: "50128723968" + email: ihenrykwok@outlook.com + executable: /vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10 + git: + commit: f2afcc15e05f491a8e50add64395fc1db0a1188d + remote: https://github.com/Spatialtemporal-AI/A1.git + gpu: Instinct MI210 + gpu_amd: + - id: "2" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xaabcddaa244a3d6e" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "0" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x82728d7f9bd937e4" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "1" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x27087f06439a527d" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "4" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x24ee801b7c402006" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "5" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x413935505e32b8da" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "3" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x95be8fdc770fcfd7" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "6" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x12140cd9e24f12e9" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "7" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xa0442ab3bdd405c1" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + gpu_count: 8 + host: auh7-1b-gpu-306 + memory: + total: "2434611519488" + os: Linux-5.15.0-140-generic-x86_64-with-glibc2.35 + program: /vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py + python: CPython 3.10.18 + root: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb + slurm: + cluster_name: ai-04r + conf: /etc/slurm/slurm.conf + cpus_on_node: "128" + gpus_on_node: "8" + gtids: "0" + job_account: faculty-acc + job_cpus_per_node: "128" + job_end_time: "1759492424" + job_gid: "2000" + job_gpus: 0,1,2,3,4,5,6,7 + job_id: "1989" + job_name: mh_realworld + job_nodelist: auh7-1b-gpu-306 + job_num_nodes: "1" + job_partition: faculty + job_qos: xdqos + job_start_time: "1759233224" + job_uid: "2013" + job_user: xiaodan + jobid: "1989" + localid: "0" + nnodes: "1" + nodeid: "0" + nodelist: auh7-1b-gpu-306 + nprocs: "1" + ntasks: "1" + ntasks_per_node: "1" + oom_kill_step: "0" + prio_process: "0" + procid: "0" + submit_dir: /vast/users/xiaodan/zhangjian/A1/launch_scripts + submit_host: auh-1b-cpu-login-001 + task_pid: "610629" + tasks_per_node: "1" + topology_addr: auh7-1b-gpu-306 + topology_addr_pattern: node + startedAt: "2025-09-30T11:54:26.454498Z" + writerId: lwz9u7135qzjvugxcx5lynr52j566znm + m: [] + python_version: 3.10.18 + t: + "1": + - 1 + - 2 + - 3 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 63 + - 71 + - 83 + - 95 + - 105 + "2": + - 1 + - 2 + - 3 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 63 + - 71 + - 83 + - 95 + - 105 + "3": + - 2 + - 13 + - 15 + - 16 + - 61 + "4": 3.10.18 + "5": 0.21.4 + "6": 4.56.1 + "10": + - 19 + "12": 0.21.4 + "13": linux-x86_64 +activation_checkpointing: + value: whole_layer +allow_resume: + value: false +batch_divisor: + value: global_batch +canceled_check_interval: + value: 50 +checkpoint_dir: + value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 +compile: + value: null +console_log_interval: + value: 1 +data: + value: + dataset: vla_dataset_realworld + drop_last: true + for_inference: false + lerobot_episode_index_end: null + lerobot_episode_index_start: null + mixture: null + multi_modal: torch + num_workers: 0 + pad: to_max + persistent_workers: false + pin_memory: true + prefetch_factor: null + rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds + rlds_dataset_name: libero_4_task_suites_no_noops + rlds_read_threads: 8 + rlds_shuffle_buffer_size: 100000 + rlds_traj_threads: 8 + root_size_mixture: null + seed: 95818 + sequence_length: 1600 + shuffle: true + shuffle_messages: false + split: train + timeout: 0 + use_proprio: true + use_wrist_image: true +device_eval_batch_size: + value: 4 +device_inf_eval_batch_size: + value: 16 +device_train_batch_size: + value: 15 +device_train_grad_accum: + value: 0 +device_train_microbatch_size: + value: 16 +dry_run: + value: false +early_exit: + value: false +epoch: + value: null +eval_interval: + value: 0 +eval_on_load: + value: false +eval_subset_num_batches: + value: -1 +evaluators: + value: + - data: + dataset: vla_dataset_realworld + drop_last: true + for_inference: false + lerobot_episode_index_end: 765 + lerobot_episode_index_start: 353 + mixture: null + multi_modal: torch + num_workers: 0 + pad: to_max + persistent_workers: true + pin_memory: true + prefetch_factor: null + rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds + rlds_dataset_name: libero_4_task_suites_no_noops + rlds_read_threads: 8 + rlds_shuffle_buffer_size: 256000 + rlds_traj_threads: 8 + root_size_mixture: null + seed: null + sequence_length: 1600 + shuffle: false + shuffle_messages: false + split: validation + timeout: 0 + use_proprio: true + use_wrist_image: true + device_eval_batch_size: null + eval_name: null + label: val + max_examples: null + max_new_tokens: 448 + mm_evaluator: null + save_dir: null + save_to_checkpoint_dir: false + skip_if_metrics_cached: true + subset_num_batches: 64 +extra_steps_after_cancel: + value: 10 +fast_forward_batches: + value: null +force_save_unsharded: + value: false +fsdp: + value: + hybrid_sharding_num_model_replicas: null + precision: float + sharding_strategy: FULL_SHARD + use_orig_params: true + wrapping_strategy: by_block_and_size +ft_connector: + value: false +ft_embedding: + value: lm_head +ft_llm: + value: true +ft_vit: + value: false +fused_loss: + value: null +gen1_gc_interval: + value: 1 +global_train_batch_size: + value: 126 +inf_eval_interval: + value: -1 +inf_eval_subset_num_batches: + value: -1 +inf_evaluators: + value: [] +initial_model_checkpoint: + value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 +keep_lr_on_load: + value: true +load_model_config: + value: null +load_path: + value: null +load_path_sharded_checkpointer: + value: null +lora: + value: false +lora_connector: + value: false +lora_llm: + value: false +lora_rank: + value: 8 +lora_vit: + value: false +max_duration: + value: 500000 +max_grad_norm: + value: 1 +max_grad_norm_ratio: + value: null +model: + value: + action_dim: 7 + action_head: l1_regression + action_head_dit_depth: 28 + action_head_dit_hidden_size: 1152 + action_head_dit_num_heads: 16 + action_tokenizer: + identifier: physical-intelligence/fast + tokenizer_dir: null + action_use_left_eef: false + action_use_mobile_base: false + activation_type: swiglu + additional_vocab_size: 128 + always_start_with_space: true + attention_dropout: 0 + attention_layer_norm: false + attention_layer_norm_with_affine: true + attention_type: sdpa + bias_for_layer_norm: null + block_group_size: 1 + block_type: sequential + clip_qkv: null + crop_mode: overlap-and-resize-c2 + d_model: 3584 + default_inference_len: 65 + embedding_dropout: 0 + embedding_size: 152064 + ff_out_size: 0 + fix_image_padding: true + float32_attention: true + head_dim: null + horizon: 8 + image_feature_dropout: 0 + image_padding_embed: pad_and_partial_pad + image_pooling_2d: attention_meanq + image_pooling_h: 2 + image_pooling_w: 2 + image_projector: mlp + include_bias: false + init_cutoff_factor: null + init_device: null + init_fn: normal + init_std: 0.02 + initializer_range: 0.02 + layer_norm_eps: 1e-06 + layer_norm_type: rms + layer_norm_with_affine: true + llm_causal_attention: false + llm_load_path: /weka/oe-training-default/mm-olmo/pretrained_llms/qwen2-7b.pt + low_cpu_fsdp: true + max_crops: 12 + max_position_embeddings: null + max_sequence_length: 4096 + message_formatting: role + mlp_hidden_size: 37888 + mlp_ratio: 4 + moe_capacity_factor: 1.25 + moe_dropless: true + moe_interleave: false + moe_lbl_in_fp32: false + moe_log_expert_assignment: false + moe_loss_weight: 0.1 + moe_mlp_impl: sparse + moe_num_experts: 8 + moe_shared_expert: false + moe_top_k: 2 + moe_zloss_weight: null + multi_annotation_weighting: root_subsegments + n_heads: 28 + n_kv_heads: 4 + n_layers: 28 + new_embedding_init_range: 0.02 + norm_after: false + normalize_input_embeds: false + num_diffusion_inference_steps: 30 + num_diffusion_steps: 1000 + overlap_margins: + - 4 + - 4 + pad_tokenizer: true + pad_value: 0 + precision: amp_bf16 + prompt_type: uber_model + qkv_bias: true + residual_dropout: 0.1 + response_residual_dropout: 0 + rope: true + rope_full_precision: true + rope_theta: 1e+06 + scale_logits: false + system_prompt_kind: demo_or_style + tokenizer: + identifier: Qwen/Qwen2-7B + tokenizer_dir: null + use_col_tokens: true + use_position_ids: true + use_proprio: true + vision_backbone: + attention_dropout: 0 + fsdp_wrap: false + image_default_input_size: + - 336 + - 336 + image_dropout_rate: 0 + image_emb_dim: 1024 + image_head_dim: 64 + image_mlp_activations: quick_gelu + image_mlp_dim: 4096 + image_model_type: openai + image_norm_eps: 1e-05 + image_num_heads: 16 + image_num_key_value_heads: 16 + image_num_layers: 23 + image_num_pos: 577 + image_patch_size: 14 + image_pos_patch_size: 14 + initializer_range: 0.02 + residual_dropout: 0 + resize_mode: default + vit_layers: + - -2 + - -9 + vit_load_path: /weka/oe-training-default/mm-olmo/pretrained_image_encoders/vit-l-14-336.pt + vocab_size: 152064 + weight_tying: false +multi_component_grad_norm: + value: true +no_pre_train_checkpoint: + value: true +optimizer: + value: + betas: + - 0.9 + - 0.95 + connector_betas: + - 0.9 + - 0.95 + connector_eps: 1e-06 + connector_learning_rate: 0.0002 + connector_weight_decay: 0 + eps: 1e-05 + learning_rate: 0.0001 + llm_betas: + - 0.9 + - 0.95 + llm_eps: 1e-06 + llm_learning_rate: 5e-05 + llm_weight_decay: 0 + metrics_log_interval: 20 + name: adamw + vit_betas: + - 0.9 + - 0.95 + vit_eps: 1e-06 + vit_learning_rate: 6e-06 + vit_weight_decay: 0 + weight_decay: 0.01 +precision: + value: amp_bf16 +python_profiling: + value: false +remote_save_folder: + value: null +reset_dataloader_state: + value: false +reset_optimizer_state: + value: false +reset_trainer_state: + value: false +restore_dataloader: + value: true +run_name: + value: realworld_20250930_115356 +save_dataloader_state: + value: false +save_folder: + value: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1 +save_interval: + value: 500 +save_interval_action_head: + value: 500 +save_interval_ephemeral: + value: null +save_interval_unsharded: + value: 500 +save_num_action_head_checkpoints_to_keep: + value: 2 +save_num_checkpoints_to_keep: + value: 1 +save_num_unsharded_checkpoints_to_keep: + value: 1 +save_overwrite: + value: true +scheduler: + value: + alpha_f: 0.1 + connector_t_warmup: 200 + grad_clip_warmup_factor: null + grad_clip_warmup_steps: null + llm_t_warmup: 2000 + name: multimodal + t_max: null + t_warmup: 100 + units: steps + vit_t_warmup: 2000 + warmup_min_lr: 0 +seed: + value: 6198 +sharded_checkpointer: + value: torch_legacy +softmax_auxiliary_loss: + value: true +softmax_auxiliary_loss_scale: + value: 0.0001 +speed_monitor: + value: + gpu_flops_available: null + window_size: 20 +stop_after: + value: null +stop_at: + value: 500000 +time_limit: + value: null +torch_profiling: + value: false +train_exit_random_layer: + value: false +use_lora: + value: true diff --git a/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/files/output.log b/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4c6498c5464eaa9218f7990906e0a0ab83f7ee2c --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/files/output.log @@ -0,0 +1,103 @@ +wandb: Detected [openai] in use. +wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script. +wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/ +09/30 [11:54:28] WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No warnings.py:109 + device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once + +****** vla_cfg: {'datasets': {'rlds': {'name': None, 'path': None, 'weight': 1.0, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe', 1, 'bounds']], 'open-source-real-world': {'rlds': {'name': 'a1_real_world', 'path': '/vast/users/xiaodan/zhangjian/datasets/OXE', 'weight': 8, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [], 'agibot': {'path': '/vast/users/xiaodan/zhangjian/datasets/AgiBotWorld-Alpha', 'weight': 8, 'action_proprio_normalization_type': None}}}, 'model': {'action_head': {'action_dim': 16, 'proprio_dim': 16, 'num_actions_chunk': 8, 'action_tokens_mapping': {'left_end_effector': 8, 'right_end_effector': 8}, 'use_left_eef': True, 'use_mobile_base': False}}} +****** Skip RLDS main; path not found: None +****** start build LeRobot main... +build_tokenizer, cache_dir None tokenizer_dir None +09/30 [11:54:34] INFO | >> Padding tokenizer with 418 tokens tokenizer.py:130 + INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk +****** length of the dataset: 72641 +09/30 [11:54:37] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50 +****** length of the dataset: 27906 +09/30 [11:54:38] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser +****** length of the dataset: 13441 +09/30 [11:54:39] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue +****** length of the dataset: 10316 +09/30 [11:54:40] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen +****** length of the dataset: 17131 +09/30 [11:54:41] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote +****** length of the dataset: 15765 +09/30 [11:54:42] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue +****** length of the dataset: 90 +09/30 [11:54:43] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe +****** length of the dataset: 18397 +****** Skip RLDS open-source-real-world; mixture 'a1_real_world' not found under: /vast/users/xiaodan/zhangjian/datasets/OXE +****** Expect one of: [] +****** path: /vast/users/xiaodan/zhangjian/datasets/AgiBotWorld-Alpha +****** before AgiBotWorldAlpha dataset... +09/30 [11:54:44] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:485 +****** length of the AgiBotWorldAlphaDataset: 1395683 +****** length of the AgiBotWorldAlphaDataset: 1395683 +****** After build vla train dataset... +****** iterable_sources: [, , , , , , , , ] +****** Before build mixed iterable dataset... +****** Build vla train dataloader successfully! +************************* Build train_dataloader successful! +************************* Before build_inf_evaluators +09/30 [11:54:45] WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No warnings.py:109 + device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once + +************************* Build evaluators successful! +************************* Early exit flags: early_exit=False +************************* Initialize model successful! +***** state_dict_path: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924/model.pt +***** Load checkpoint successful! +missing keys: ['action_head.model.layer_norm1.weight', 'action_head.model.layer_norm1.bias', 'action_head.model.fc1.weight', 'action_head.model.fc1.bias', 'action_head.model.mlp_resnet_blocks.0.ffn.0.weight', 'action_head.model.mlp_resnet_blocks.0.ffn.0.bias', 'action_head.model.mlp_resnet_blocks.0.ffn.1.weight', 'action_head.model.mlp_resnet_blocks.0.ffn.1.bias', 'action_head.model.mlp_resnet_blocks.1.ffn.0.weight', 'action_head.model.mlp_resnet_blocks.1.ffn.0.bias', 'action_head.model.mlp_resnet_blocks.1.ffn.1.weight', 'action_head.model.mlp_resnet_blocks.1.ffn.1.bias', 'action_head.model.layer_norm2.weight', 'action_head.model.layer_norm2.bias', 'action_head.model.fc2.weight', 'action_head.model.fc2.bias', 'proprio_projector.fc1.weight', 'proprio_projector.fc1.bias', 'proprio_projector.fc2.weight', 'proprio_projector.fc2.bias'] +unexpected keys: [] +************************* Initialize model successful! +************************* LoRA flags: use_lora=True, lora_llm=False, lora_vit=False, lora_connector=False +************************* Before add lora to model +************************* Before FSDP model wrapping +************************* FSDP model wrapping successful! +************************* Before building optimizer and scheduler +************* Before get lora params +************* After get lora params successfully +09/30 [11:56:10] INFO | >> Constructing optimizer with 2 param groups optim.py:1283 +************************************************** +After building optimizer and scheduler and model, before training, peak GPU memory (MB): 35614 +************************* VLATrainer initialized successfully! +************************* Before trainer.fit() +Pre-train system metrics + System/Peak GPU Memory (MB)=35,614 +!!!Training failed: __enter__ +Traceback (most recent call last): + File "/vast/users/xiaodan/zhangjian/A1/scripts/train_for_action.py", line 593, in main + trainer.fit() + File "/vast/users/xiaodan/zhangjian/A1/olmo/train.py", line 2284, in fit + for batch in self.train_loader: + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 734, in __next__ + data = self._next_data() + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 790, in _next_data + data = self._dataset_fetcher.fetch(index) # may raise StopIteration + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 33, in fetch + data.append(next(self.dataset_iter)) + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/iterable_dataset_mixture.py", line 156, in __iter__ + yield next(source_iters[src_idx]) + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/dataset.py", line 111, in __iter__ + item = self.dataset.get(int(idx), rng) + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/vla/agibot_dataset.py", line 350, in get + with self._open_h5(set_id, episode_id) as h5: +AttributeError: __enter__ +wandb: WARNING The `quiet` argument to `wandb.run.finish()` is deprecated, use `wandb.Settings(quiet=...)` to set this instead. diff --git a/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/files/requirements.txt b/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..5a5e971a5d9f3b6f6a75248e203682c5fbcd69e8 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/files/requirements.txt @@ -0,0 +1,286 @@ +ai2-molmo==0.0.0 +astunparse==1.6.3 +flatbuffers==25.2.10 +gast==0.6.0 +google-pasta==0.2.0 +h5py==3.14.0 +libclang==18.1.1 +Markdown==3.9 +namex==0.1.0 +opt_einsum==3.4.0 +optree==0.17.0 +tensorboard-data-server==0.7.2 +tensorflow-io-gcs-filesystem==0.37.1 +termcolor==3.1.0 +Werkzeug==3.1.3 +Brotli==1.1.0 +Farama-Notifications==0.0.4 +MarkupSafe==2.1.5 +PyYAML==6.0.2 +absl-py==2.3.1 +accelerate==1.10.1 +ai2-molmo==0.0.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +antlr4-python3-runtime==4.9.3 +anyio==4.10.0 +array_record==0.8.1 +async-timeout==5.0.1 +attrs==25.3.0 +av==15.1.0 +backports.tarfile==1.2.0 +beaker-gantry==3.2.0 +beaker-py==2.5.0 +black==23.12.1 +blinker==1.9.0 +boltons==25.0.0 +boto3==1.40.33 +botocore==1.40.33 +build==1.3.0 +cached_path==1.7.3 +cached-property==2.0.1 +cachetools==5.5.2 +certifi==2025.8.3 +cffi==2.0.0 +charset-normalizer==3.4.3 +click==8.2.1 +click-help-colors==0.9.4 +click-option-group==0.5.7 +cloudpickle==3.1.1 +cmake==4.1.0 +contourpy==1.3.2 +cryptography==46.0.1 +cycler==0.12.1 +dataclass-extensions==0.2.3 +datasets==3.6.0 +decorator==5.2.1 +deepdiff==8.6.1 +diffusers==0.35.1 +dill==0.3.8 +distro==1.9.0 +dlimp==0.0.1 +dm-tree==0.1.9 +docutils==0.22.1 +draccus==0.10.0 +editdistance==0.8.1 +einops==0.8.1 +einops-exts==0.0.4 +et_xmlfile==2.0.0 +etils==1.13.0 +evdev==1.9.2 +exceptiongroup==1.3.0 +face==24.0.0 +fastapi==0.116.2 +ffmpy==0.6.1 +fiddle==0.3.0 +filelock==3.13.1 +Flask==3.1.2 +fonttools==4.60.0 +frozenlist==1.7.0 +fsspec==2023.9.2 +ftfy==6.3.1 +gcsfs==2023.9.2 +gitdb==4.0.12 +GitPython==3.1.45 +glom==24.11.0 +google-api-core==2.25.1 +google-auth==2.40.3 +google-auth-oauthlib==1.2.2 +google-cloud-core==2.4.3 +google-cloud-storage==2.19.0 +google-crc32c==1.7.1 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +gradio==5.46.0 +gradio_client==1.13.0 +graphviz==0.21 +groovy==0.1.2 +grpcio==1.75.0 +gymnasium==0.29.1 +h11==0.16.0 +hf_transfer==0.1.9 +hf-xet==1.1.10 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==0.35.0 +id==1.5.0 +idna==3.10 +imageio==2.37.0 +imageio-ffmpeg==0.6.0 +importlib_metadata==8.7.0 +importlib_resources==6.5.2 +iniconfig==2.1.0 +inquirerpy==0.3.4 +isort==5.12.0 +itsdangerous==2.2.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.3.0 +jeepney==0.9.0 +Jinja2==3.1.4 +jiter==0.11.0 +jmespath==1.0.1 +joblib==1.5.2 +jsonlines==4.0.0 +keras==2.15.0 +keyring==25.6.0 +kiwisolver==1.4.9 +latex2sympy2_extended==1.10.2 +lerobot==0.3.4 +Levenshtein==0.27.1 +libcst==1.8.4 +lightning-utilities==0.15.2 +markdown-it-py==4.0.0 +math-verify==0.8.0 +matplotlib==3.10.6 +mdurl==0.1.2 +mergedeep==1.3.4 +ml-dtypes==0.2.0 +ml_dtypes==0.5.3 +more-itertools==10.8.0 +mpmath==1.3.0 +msgspec==0.19.0 +multidict==6.6.4 +multiprocess==0.70.16 +mypy==1.3.0 +mypy_extensions==1.1.0 +necessary==0.4.3 +networkx==3.3 +nh3==0.3.0 +nltk==3.9.1 +numpy==1.26.4 +oauthlib==3.3.1 +omegaconf==2.3.0 +openai==1.108.0 +opencv-python-headless==4.12.0.88 +OpenEXR==3.4.0 +openpyxl==3.1.5 +orderly-set==5.5.0 +orjson==3.11.3 +packaging==25.0 +pandas==2.3.2 +pathspec==0.12.1 +petname==2.6 +pfzy==0.3.4 +pillow==11.0.0 +pip==25.2 +platformdirs==4.4.0 +pluggy==1.6.0 +promise==2.3 +prompt_toolkit==3.0.52 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==4.21.12 +protobuf==6.32.1 +psutil==7.1.0 +pyarrow==21.0.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.23 +pydantic==2.11.9 +pydantic_core==2.33.2 +pydub==0.25.1 +Pygments==2.19.2 +pynput==1.8.1 +pyparsing==3.2.4 +pyproject_hooks==1.2.0 +pyserial==3.5 +pytest==8.4.2 +pytest-sphinx==0.6.3 +python-dateutil==2.9.0.post0 +python-Levenshtein==0.27.1 +python-multipart==0.0.20 +python-xlib==0.33 +pytorch-triton-rocm==3.4.0 +pytz==2025.2 +pyyaml-include==1.4.1 +RapidFuzz==3.14.1 +readme_renderer==44.0 +regex==2025.9.1 +requests==2.32.5 +requests-oauthlib==2.0.0 +requests-toolbelt==1.0.0 +requirements-parser==0.13.0 +rerun-sdk==0.22.1 +rfc3986==2.0.0 +rich==13.9.4 +rsa==4.9.1 +ruff==0.13.0 +s3transfer==0.14.0 +safehttpx==0.1.6 +safetensors==0.6.2 +scikit-learn==1.7.2 +scipy==1.15.3 +SecretStorage==3.4.0 +semantic-version==2.10.0 +sentencepiece==0.2.1 +sentry-sdk==2.38.0 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.17.0 +smart_open==7.3.1 +smashed==0.21.5 +smmap==5.0.2 +sniffio==1.3.1 +starlette==0.48.0 +sympy==1.13.3 +tensorboard==2.15.2 +tensorboard==2.19.0 +tensorflow==2.15.0 +tensorflow-addons==0.23.0 +tensorflow-datasets==4.9.3 +tensorflow-estimator==2.15.0 +tensorflow-graphics==2021.12.3 +tensorflow-metadata==1.17.2 +threadpoolctl==3.6.0 +timm==1.0.19 +tokenizers==0.22.0 +toml==0.10.2 +tomli==2.2.1 +tomlkit==0.13.3 +torch==2.8.0+rocm6.4 +torchcodec==0.5 +torchmetrics==1.8.2 +torchvision==0.23.0+rocm6.4 +tqdm==4.67.1 +transformers==4.56.1 +trimesh==4.8.2 +trouting==0.3.3 +twine==6.2.0 +typeguard==2.13.3 +typer==0.17.4 +typing_extensions==4.15.0 +typing-inspect==0.9.0 +typing-inspection==0.4.1 +tzdata==2025.2 +urllib3==2.5.0 +uvicorn==0.35.0 +wandb==0.21.4 +wcwidth==0.2.13 +websockets==15.0.1 +wheel==0.45.1 +wrapt==1.14.2 +xxhash==3.5.0 +yarl==1.20.1 +zipp==3.23.0 +lerobot==0.3.4 +minLoRA==0.1.0 +autocommand==2.2.2 +backports.tarfile==1.2.0 +importlib_metadata==8.0.0 +inflect==7.3.1 +jaraco.collections==5.1.0 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +jaraco.text==3.12.1 +more-itertools==10.3.0 +packaging==24.2 +platformdirs==4.2.2 +tomli==2.0.1 +typeguard==4.3.0 +typing_extensions==4.12.2 +wheel==0.45.1 +zipp==3.19.2 diff --git a/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/files/wandb-metadata.json b/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b3f30af8129228073f6da90798d7edb75ad90c66 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/files/wandb-metadata.json @@ -0,0 +1,202 @@ +{ + "os": "Linux-5.15.0-140-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.18", + "startedAt": "2025-09-30T11:54:26.454498Z", + "args": [ + "qwen2_7b", + "save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1", + "--vision_backbone", + "openai", + "--action_head", + "l1_regression", + "--seq_len", + "1600", + "--ft_llm", + "--checkpoint", + "/vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924", + "--device_train_microbatch_size", + "16", + "--global_batch_size", + "126", + "--dataset", + "vla_dataset_realworld", + "--llm_learning_rate", + "5e-5", + "--wandb_entity", + "henryeap", + "--wandb_project", + "a1-realworld", + "--wandb_run_name", + "realworld", + "--save_overwrite" + ], + "program": "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py", + "codePath": "launch_scripts/train_vla.py", + "codePathLocal": "launch_scripts/train_vla.py", + "git": { + "remote": "https://github.com/Spatialtemporal-AI/A1.git", + "commit": "f2afcc15e05f491a8e50add64395fc1db0a1188d" + }, + "email": "ihenrykwok@outlook.com", + "root": "/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb", + "host": "auh7-1b-gpu-306", + "executable": "/vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10", + "cpu_count": 64, + "cpu_count_logical": 128, + "gpu": "Instinct MI210", + "gpu_count": 8, + "disk": { + "/": { + "total": "470343073792", + "used": "50128723968" + } + }, + "memory": { + "total": "2434611519488" + }, + "gpu_amd": [ + { + "id": "2", + "uniqueId": "0xaabcddaa244a3d6e", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "0", + "uniqueId": "0x82728d7f9bd937e4", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "1", + "uniqueId": "0x27087f06439a527d", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "4", + "uniqueId": "0x24ee801b7c402006", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "5", + "uniqueId": "0x413935505e32b8da", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "3", + "uniqueId": "0x95be8fdc770fcfd7", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "6", + "uniqueId": "0x12140cd9e24f12e9", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "7", + "uniqueId": "0xa0442ab3bdd405c1", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + } + ], + "slurm": { + "cluster_name": "ai-04r", + "conf": "/etc/slurm/slurm.conf", + "cpus_on_node": "128", + "gpus_on_node": "8", + "gtids": "0", + "job_account": "faculty-acc", + "job_cpus_per_node": "128", + "job_end_time": "1759492424", + "job_gid": "2000", + "job_gpus": "0,1,2,3,4,5,6,7", + "job_id": "1989", + "job_name": "mh_realworld", + "job_nodelist": "auh7-1b-gpu-306", + "job_num_nodes": "1", + "job_partition": "faculty", + "job_qos": "xdqos", + "job_start_time": "1759233224", + "job_uid": "2013", + "job_user": "xiaodan", + "jobid": "1989", + "localid": "0", + "nnodes": "1", + "nodeid": "0", + "nodelist": "auh7-1b-gpu-306", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "oom_kill_step": "0", + "prio_process": "0", + "procid": "0", + "submit_dir": "/vast/users/xiaodan/zhangjian/A1/launch_scripts", + "submit_host": "auh-1b-cpu-login-001", + "task_pid": "610629", + "tasks_per_node": "1", + "topology_addr": "auh7-1b-gpu-306", + "topology_addr_pattern": "node" + }, + "writerId": "lwz9u7135qzjvugxcx5lynr52j566znm" +} \ No newline at end of file diff --git a/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/files/wandb-summary.json b/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7080d373195ac5cbd74b89b0b4a01a70bcc29b8d --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":104},"_runtime":104.60638138,"_step":0,"System/Peak GPU Memory (MB)":35614.78125,"_timestamp":1.7592333707721887e+09} \ No newline at end of file diff --git a/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/logs/debug-core.log b/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..c55fdf4dcebf7604e1d3780f20dc28f2baca6c30 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/logs/debug-core.log @@ -0,0 +1,16 @@ +{"time":"2025-09-30T11:54:26.521817705Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmprzszz_9w/port-610806.txt","pid":610806,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-09-30T11:54:26.522336125Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":610806} +{"time":"2025-09-30T11:54:26.522321354Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-610806-610974-3557492925/socket","Net":"unix"}} +{"time":"2025-09-30T11:54:26.705925546Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-09-30T11:54:26.712556793Z","level":"INFO","msg":"handleInformInit: received","streamId":"4y0fn6ka","id":"1(@)"} +{"time":"2025-09-30T11:54:27.766430612Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"4y0fn6ka","id":"1(@)"} +{"time":"2025-09-30T11:56:14.452974036Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"4y0fn6ka","id":"1(@)"} +{"time":"2025-09-30T11:56:14.456105611Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"4y0fn6ka","id":"1(@)"} +{"time":"2025-09-30T11:56:14.483739548Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2025-09-30T11:56:14.483758558Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2025-09-30T11:56:14.483763869Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-09-30T11:56:14.483789359Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2025-09-30T11:56:14.48383279Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-610806-610974-3557492925/socket","Net":"unix"}} +{"time":"2025-09-30T11:56:14.483901891Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2025-09-30T11:56:14.483907841Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2025-09-30T11:56:14.483922101Z","level":"INFO","msg":"server is closed"} diff --git a/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/logs/debug-internal.log b/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..89122e33061292032942ab7695935c6170d04920 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/logs/debug-internal.log @@ -0,0 +1,12 @@ +{"time":"2025-09-30T11:54:26.714354905Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-09-30T11:54:27.766379841Z","level":"INFO","msg":"stream: created new stream","id":"4y0fn6ka"} +{"time":"2025-09-30T11:54:27.766424922Z","level":"INFO","msg":"stream: started","id":"4y0fn6ka"} +{"time":"2025-09-30T11:54:27.766455023Z","level":"INFO","msg":"sender: started","stream_id":"4y0fn6ka"} +{"time":"2025-09-30T11:54:27.766455013Z","level":"INFO","msg":"handler: started","stream_id":"4y0fn6ka"} +{"time":"2025-09-30T11:54:27.766467603Z","level":"INFO","msg":"writer: started","stream_id":"4y0fn6ka"} +{"time":"2025-09-30T11:56:12.736999283Z","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run metadata","runtime_seconds":1.9632518270000001}],"total_operations":1}} +{"time":"2025-09-30T11:56:13.795981592Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-09-30T11:56:14.453298712Z","level":"INFO","msg":"stream: closing","id":"4y0fn6ka"} +{"time":"2025-09-30T11:56:14.453319212Z","level":"INFO","msg":"handler: closed","stream_id":"4y0fn6ka"} +{"time":"2025-09-30T11:56:14.454730867Z","level":"INFO","msg":"sender: closed","stream_id":"4y0fn6ka"} +{"time":"2025-09-30T11:56:14.454741967Z","level":"INFO","msg":"stream: closed","id":"4y0fn6ka"} diff --git a/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/logs/debug.log b/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/run-4y0fn6ka.wandb b/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/run-4y0fn6ka.wandb new file mode 100644 index 0000000000000000000000000000000000000000..6a97d89b06d298e670806f72900c4be302ebf45e Binary files /dev/null and b/all_l1/wandb/wandb/run-20250930_115426-4y0fn6ka/run-4y0fn6ka.wandb differ diff --git a/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/files/config.yaml b/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5900e54610df9e31d8e1a4cd62cef55a64fc7a50 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/files/config.yaml @@ -0,0 +1,621 @@ +_wandb: + value: + cli_version: 0.21.4 + e: + np3futv3ypcdn5zstcz9ip385cmxztpr: + args: + - qwen2_7b + - save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1 + - --vision_backbone + - openai + - --action_head + - l1_regression + - --seq_len + - "1600" + - --ft_llm + - --checkpoint + - /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 + - --device_train_microbatch_size + - "16" + - --global_batch_size + - "126" + - --dataset + - vla_dataset_realworld + - --llm_learning_rate + - "5e-5" + - --wandb_entity + - henryeap + - --wandb_project + - a1-realworld + - --wandb_run_name + - realworld + - --save_overwrite + codePath: launch_scripts/train_vla.py + codePathLocal: launch_scripts/train_vla.py + cpu_count: 64 + cpu_count_logical: 128 + disk: + /: + total: "470343073792" + used: "50128797696" + email: ihenrykwok@outlook.com + executable: /vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10 + git: + commit: f2afcc15e05f491a8e50add64395fc1db0a1188d + remote: https://github.com/Spatialtemporal-AI/A1.git + gpu: Instinct MI210 + gpu_amd: + - id: "1" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x27087f06439a527d" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "4" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x24ee801b7c402006" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "2" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xaabcddaa244a3d6e" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "0" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x82728d7f9bd937e4" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "3" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x95be8fdc770fcfd7" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "7" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xa0442ab3bdd405c1" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "6" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x12140cd9e24f12e9" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "5" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x413935505e32b8da" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + gpu_count: 8 + host: auh7-1b-gpu-306 + memory: + total: "2434611519488" + os: Linux-5.15.0-140-generic-x86_64-with-glibc2.35 + program: /vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py + python: CPython 3.10.18 + root: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb + slurm: + cluster_name: ai-04r + conf: /etc/slurm/slurm.conf + cpus_on_node: "128" + gpus_on_node: "8" + gtids: "0" + job_account: faculty-acc + job_cpus_per_node: "128" + job_end_time: "1759493653" + job_gid: "2000" + job_gpus: 0,1,2,3,4,5,6,7 + job_id: "1990" + job_name: mh_realworld + job_nodelist: auh7-1b-gpu-306 + job_num_nodes: "1" + job_partition: faculty + job_qos: xdqos + job_start_time: "1759234453" + job_uid: "2013" + job_user: xiaodan + jobid: "1990" + localid: "0" + nnodes: "1" + nodeid: "0" + nodelist: auh7-1b-gpu-306 + nprocs: "1" + ntasks: "1" + ntasks_per_node: "1" + oom_kill_step: "0" + prio_process: "0" + procid: "0" + submit_dir: /vast/users/xiaodan/zhangjian/A1/launch_scripts + submit_host: auh-1b-cpu-login-001 + task_pid: "627093" + tasks_per_node: "1" + topology_addr: auh7-1b-gpu-306 + topology_addr_pattern: node + startedAt: "2025-09-30T12:14:56.412445Z" + writerId: np3futv3ypcdn5zstcz9ip385cmxztpr + m: [] + python_version: 3.10.18 + t: + "1": + - 1 + - 2 + - 3 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 63 + - 71 + - 83 + - 95 + - 105 + "2": + - 1 + - 2 + - 3 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 63 + - 71 + - 83 + - 95 + - 105 + "3": + - 2 + - 13 + - 15 + - 16 + - 61 + "4": 3.10.18 + "5": 0.21.4 + "6": 4.56.1 + "10": + - 19 + "12": 0.21.4 + "13": linux-x86_64 +activation_checkpointing: + value: whole_layer +allow_resume: + value: false +batch_divisor: + value: global_batch +canceled_check_interval: + value: 50 +checkpoint_dir: + value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 +compile: + value: null +console_log_interval: + value: 1 +data: + value: + dataset: vla_dataset_realworld + drop_last: true + for_inference: false + lerobot_episode_index_end: null + lerobot_episode_index_start: null + mixture: null + multi_modal: torch + num_workers: 0 + pad: to_max + persistent_workers: false + pin_memory: true + prefetch_factor: null + rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds + rlds_dataset_name: libero_4_task_suites_no_noops + rlds_read_threads: 8 + rlds_shuffle_buffer_size: 100000 + rlds_traj_threads: 8 + root_size_mixture: null + seed: 95818 + sequence_length: 1600 + shuffle: true + shuffle_messages: false + split: train + timeout: 0 + use_proprio: true + use_wrist_image: true +device_eval_batch_size: + value: 4 +device_inf_eval_batch_size: + value: 16 +device_train_batch_size: + value: 15 +device_train_grad_accum: + value: 0 +device_train_microbatch_size: + value: 16 +dry_run: + value: false +early_exit: + value: false +epoch: + value: null +eval_interval: + value: 0 +eval_on_load: + value: false +eval_subset_num_batches: + value: -1 +evaluators: + value: + - data: + dataset: vla_dataset_realworld + drop_last: true + for_inference: false + lerobot_episode_index_end: 765 + lerobot_episode_index_start: 353 + mixture: null + multi_modal: torch + num_workers: 0 + pad: to_max + persistent_workers: true + pin_memory: true + prefetch_factor: null + rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds + rlds_dataset_name: libero_4_task_suites_no_noops + rlds_read_threads: 8 + rlds_shuffle_buffer_size: 256000 + rlds_traj_threads: 8 + root_size_mixture: null + seed: null + sequence_length: 1600 + shuffle: false + shuffle_messages: false + split: validation + timeout: 0 + use_proprio: true + use_wrist_image: true + device_eval_batch_size: null + eval_name: null + label: val + max_examples: null + max_new_tokens: 448 + mm_evaluator: null + save_dir: null + save_to_checkpoint_dir: false + skip_if_metrics_cached: true + subset_num_batches: 64 +extra_steps_after_cancel: + value: 10 +fast_forward_batches: + value: null +force_save_unsharded: + value: false +fsdp: + value: + hybrid_sharding_num_model_replicas: null + precision: float + sharding_strategy: FULL_SHARD + use_orig_params: true + wrapping_strategy: by_block_and_size +ft_connector: + value: false +ft_embedding: + value: lm_head +ft_llm: + value: true +ft_vit: + value: false +fused_loss: + value: null +gen1_gc_interval: + value: 1 +global_train_batch_size: + value: 126 +inf_eval_interval: + value: -1 +inf_eval_subset_num_batches: + value: -1 +inf_evaluators: + value: [] +initial_model_checkpoint: + value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 +keep_lr_on_load: + value: true +load_model_config: + value: null +load_path: + value: null +load_path_sharded_checkpointer: + value: null +lora: + value: false +lora_connector: + value: false +lora_llm: + value: false +lora_rank: + value: 8 +lora_vit: + value: false +max_duration: + value: 500000 +max_grad_norm: + value: 1 +max_grad_norm_ratio: + value: null +model: + value: + action_dim: 7 + action_head: l1_regression + action_head_dit_depth: 28 + action_head_dit_hidden_size: 1152 + action_head_dit_num_heads: 16 + action_tokenizer: + identifier: physical-intelligence/fast + tokenizer_dir: null + action_use_left_eef: false + action_use_mobile_base: false + activation_type: swiglu + additional_vocab_size: 128 + always_start_with_space: true + attention_dropout: 0 + attention_layer_norm: false + attention_layer_norm_with_affine: true + attention_type: sdpa + bias_for_layer_norm: null + block_group_size: 1 + block_type: sequential + clip_qkv: null + crop_mode: overlap-and-resize-c2 + d_model: 3584 + default_inference_len: 65 + embedding_dropout: 0 + embedding_size: 152064 + ff_out_size: 0 + fix_image_padding: true + float32_attention: true + head_dim: null + horizon: 8 + image_feature_dropout: 0 + image_padding_embed: pad_and_partial_pad + image_pooling_2d: attention_meanq + image_pooling_h: 2 + image_pooling_w: 2 + image_projector: mlp + include_bias: false + init_cutoff_factor: null + init_device: null + init_fn: normal + init_std: 0.02 + initializer_range: 0.02 + layer_norm_eps: 1e-06 + layer_norm_type: rms + layer_norm_with_affine: true + llm_causal_attention: false + llm_load_path: /weka/oe-training-default/mm-olmo/pretrained_llms/qwen2-7b.pt + low_cpu_fsdp: true + max_crops: 12 + max_position_embeddings: null + max_sequence_length: 4096 + message_formatting: role + mlp_hidden_size: 37888 + mlp_ratio: 4 + moe_capacity_factor: 1.25 + moe_dropless: true + moe_interleave: false + moe_lbl_in_fp32: false + moe_log_expert_assignment: false + moe_loss_weight: 0.1 + moe_mlp_impl: sparse + moe_num_experts: 8 + moe_shared_expert: false + moe_top_k: 2 + moe_zloss_weight: null + multi_annotation_weighting: root_subsegments + n_heads: 28 + n_kv_heads: 4 + n_layers: 28 + new_embedding_init_range: 0.02 + norm_after: false + normalize_input_embeds: false + num_diffusion_inference_steps: 30 + num_diffusion_steps: 1000 + overlap_margins: + - 4 + - 4 + pad_tokenizer: true + pad_value: 0 + precision: amp_bf16 + prompt_type: uber_model + qkv_bias: true + residual_dropout: 0.1 + response_residual_dropout: 0 + rope: true + rope_full_precision: true + rope_theta: 1e+06 + scale_logits: false + system_prompt_kind: demo_or_style + tokenizer: + identifier: Qwen/Qwen2-7B + tokenizer_dir: null + use_col_tokens: true + use_position_ids: true + use_proprio: true + vision_backbone: + attention_dropout: 0 + fsdp_wrap: false + image_default_input_size: + - 336 + - 336 + image_dropout_rate: 0 + image_emb_dim: 1024 + image_head_dim: 64 + image_mlp_activations: quick_gelu + image_mlp_dim: 4096 + image_model_type: openai + image_norm_eps: 1e-05 + image_num_heads: 16 + image_num_key_value_heads: 16 + image_num_layers: 23 + image_num_pos: 577 + image_patch_size: 14 + image_pos_patch_size: 14 + initializer_range: 0.02 + residual_dropout: 0 + resize_mode: default + vit_layers: + - -2 + - -9 + vit_load_path: /weka/oe-training-default/mm-olmo/pretrained_image_encoders/vit-l-14-336.pt + vocab_size: 152064 + weight_tying: false +multi_component_grad_norm: + value: true +no_pre_train_checkpoint: + value: true +optimizer: + value: + betas: + - 0.9 + - 0.95 + connector_betas: + - 0.9 + - 0.95 + connector_eps: 1e-06 + connector_learning_rate: 0.0002 + connector_weight_decay: 0 + eps: 1e-05 + learning_rate: 0.0001 + llm_betas: + - 0.9 + - 0.95 + llm_eps: 1e-06 + llm_learning_rate: 5e-05 + llm_weight_decay: 0 + metrics_log_interval: 20 + name: adamw + vit_betas: + - 0.9 + - 0.95 + vit_eps: 1e-06 + vit_learning_rate: 6e-06 + vit_weight_decay: 0 + weight_decay: 0.01 +precision: + value: amp_bf16 +python_profiling: + value: false +remote_save_folder: + value: null +reset_dataloader_state: + value: false +reset_optimizer_state: + value: false +reset_trainer_state: + value: false +restore_dataloader: + value: true +run_name: + value: realworld_20250930_121426 +save_dataloader_state: + value: false +save_folder: + value: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1 +save_interval: + value: 500 +save_interval_action_head: + value: 500 +save_interval_ephemeral: + value: null +save_interval_unsharded: + value: 500 +save_num_action_head_checkpoints_to_keep: + value: 2 +save_num_checkpoints_to_keep: + value: 1 +save_num_unsharded_checkpoints_to_keep: + value: 1 +save_overwrite: + value: true +scheduler: + value: + alpha_f: 0.1 + connector_t_warmup: 200 + grad_clip_warmup_factor: null + grad_clip_warmup_steps: null + llm_t_warmup: 2000 + name: multimodal + t_max: null + t_warmup: 100 + units: steps + vit_t_warmup: 2000 + warmup_min_lr: 0 +seed: + value: 6198 +sharded_checkpointer: + value: torch_legacy +softmax_auxiliary_loss: + value: true +softmax_auxiliary_loss_scale: + value: 0.0001 +speed_monitor: + value: + gpu_flops_available: null + window_size: 20 +stop_after: + value: null +stop_at: + value: 500000 +time_limit: + value: null +torch_profiling: + value: false +train_exit_random_layer: + value: false +use_lora: + value: true diff --git a/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/files/output.log b/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..506b7ae0552729d61b64efd7e0c0ce8cd8548c70 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/files/output.log @@ -0,0 +1,103 @@ +wandb: Detected [openai] in use. +wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script. +wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/ +09/30 [12:14:58] WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No warnings.py:109 + device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once + +****** vla_cfg: {'datasets': {'rlds': {'name': None, 'path': None, 'weight': 1.0, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe', 1, 'bounds']], 'open-source-real-world': {'rlds': {'name': 'a1_real_world', 'path': '/vast/users/xiaodan/zhangjian/datasets/OXE', 'weight': 8, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [], 'agibot': {'path': '/vast/users/xiaodan/zhangjian/datasets/AgiBotWorld-Alpha', 'weight': 8, 'action_proprio_normalization_type': None}}}, 'model': {'action_head': {'action_dim': 16, 'proprio_dim': 16, 'num_actions_chunk': 8, 'action_tokens_mapping': {'left_end_effector': 8, 'right_end_effector': 8}, 'use_left_eef': True, 'use_mobile_base': False}}} +****** Skip RLDS main; path not found: None +****** start build LeRobot main... +build_tokenizer, cache_dir None tokenizer_dir None +09/30 [12:15:04] INFO | >> Padding tokenizer with 418 tokens tokenizer.py:130 +09/30 [12:15:05] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk +****** length of the dataset: 72641 +09/30 [12:15:13] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50 +****** length of the dataset: 27906 +09/30 [12:15:14] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser +****** length of the dataset: 13441 +09/30 [12:15:15] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue +****** length of the dataset: 10316 +09/30 [12:15:16] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen +****** length of the dataset: 17131 +09/30 [12:15:17] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote +****** length of the dataset: 15765 +09/30 [12:15:18] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue +****** length of the dataset: 90 +09/30 [12:15:19] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe +****** length of the dataset: 18397 +****** Skip RLDS open-source-real-world; mixture 'a1_real_world' not found under: /vast/users/xiaodan/zhangjian/datasets/OXE +****** Expect one of: [] +****** path: /vast/users/xiaodan/zhangjian/datasets/AgiBotWorld-Alpha +****** before AgiBotWorldAlpha dataset... +09/30 [12:15:20] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:485 +****** length of the AgiBotWorldAlphaDataset: 1395683 +****** length of the AgiBotWorldAlphaDataset: 1395683 +****** After build vla train dataset... +****** iterable_sources: [, , , , , , , , ] +****** Before build mixed iterable dataset... +****** Build vla train dataloader successfully! +************************* Build train_dataloader successful! +************************* Before build_inf_evaluators + WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No warnings.py:109 + device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once + +************************* Build evaluators successful! +************************* Early exit flags: early_exit=False +************************* Initialize model successful! +***** state_dict_path: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924/model.pt +***** Load checkpoint successful! +missing keys: ['action_head.model.layer_norm1.weight', 'action_head.model.layer_norm1.bias', 'action_head.model.fc1.weight', 'action_head.model.fc1.bias', 'action_head.model.mlp_resnet_blocks.0.ffn.0.weight', 'action_head.model.mlp_resnet_blocks.0.ffn.0.bias', 'action_head.model.mlp_resnet_blocks.0.ffn.1.weight', 'action_head.model.mlp_resnet_blocks.0.ffn.1.bias', 'action_head.model.mlp_resnet_blocks.1.ffn.0.weight', 'action_head.model.mlp_resnet_blocks.1.ffn.0.bias', 'action_head.model.mlp_resnet_blocks.1.ffn.1.weight', 'action_head.model.mlp_resnet_blocks.1.ffn.1.bias', 'action_head.model.layer_norm2.weight', 'action_head.model.layer_norm2.bias', 'action_head.model.fc2.weight', 'action_head.model.fc2.bias', 'proprio_projector.fc1.weight', 'proprio_projector.fc1.bias', 'proprio_projector.fc2.weight', 'proprio_projector.fc2.bias'] +unexpected keys: [] +************************* Initialize model successful! +************************* LoRA flags: use_lora=True, lora_llm=False, lora_vit=False, lora_connector=False +************************* Before add lora to model +************************* Before FSDP model wrapping +************************* FSDP model wrapping successful! +************************* Before building optimizer and scheduler +************* Before get lora params +************* After get lora params successfully +09/30 [12:16:34] INFO | >> Constructing optimizer with 2 param groups optim.py:1283 +************************************************** +After building optimizer and scheduler and model, before training, peak GPU memory (MB): 35614 +************************* VLATrainer initialized successfully! +************************* Before trainer.fit() +Pre-train system metrics + System/Peak GPU Memory (MB)=35,614 +!!!Training failed: __enter__ +Traceback (most recent call last): + File "/vast/users/xiaodan/zhangjian/A1/scripts/train_for_action.py", line 593, in main + trainer.fit() + File "/vast/users/xiaodan/zhangjian/A1/olmo/train.py", line 2284, in fit + for batch in self.train_loader: + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 734, in __next__ + data = self._next_data() + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 790, in _next_data + data = self._dataset_fetcher.fetch(index) # may raise StopIteration + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 33, in fetch + data.append(next(self.dataset_iter)) + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/iterable_dataset_mixture.py", line 156, in __iter__ + yield next(source_iters[src_idx]) + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/dataset.py", line 111, in __iter__ + item = self.dataset.get(int(idx), rng) + File "/vast/users/xiaodan/zhangjian/A1/olmo/data/vla/agibot_dataset.py", line 350, in get + with self._open_h5(set_id, episode_id) as h5: +AttributeError: __enter__ +wandb: WARNING The `quiet` argument to `wandb.run.finish()` is deprecated, use `wandb.Settings(quiet=...)` to set this instead. diff --git a/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/files/requirements.txt b/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..5a5e971a5d9f3b6f6a75248e203682c5fbcd69e8 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/files/requirements.txt @@ -0,0 +1,286 @@ +ai2-molmo==0.0.0 +astunparse==1.6.3 +flatbuffers==25.2.10 +gast==0.6.0 +google-pasta==0.2.0 +h5py==3.14.0 +libclang==18.1.1 +Markdown==3.9 +namex==0.1.0 +opt_einsum==3.4.0 +optree==0.17.0 +tensorboard-data-server==0.7.2 +tensorflow-io-gcs-filesystem==0.37.1 +termcolor==3.1.0 +Werkzeug==3.1.3 +Brotli==1.1.0 +Farama-Notifications==0.0.4 +MarkupSafe==2.1.5 +PyYAML==6.0.2 +absl-py==2.3.1 +accelerate==1.10.1 +ai2-molmo==0.0.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +antlr4-python3-runtime==4.9.3 +anyio==4.10.0 +array_record==0.8.1 +async-timeout==5.0.1 +attrs==25.3.0 +av==15.1.0 +backports.tarfile==1.2.0 +beaker-gantry==3.2.0 +beaker-py==2.5.0 +black==23.12.1 +blinker==1.9.0 +boltons==25.0.0 +boto3==1.40.33 +botocore==1.40.33 +build==1.3.0 +cached_path==1.7.3 +cached-property==2.0.1 +cachetools==5.5.2 +certifi==2025.8.3 +cffi==2.0.0 +charset-normalizer==3.4.3 +click==8.2.1 +click-help-colors==0.9.4 +click-option-group==0.5.7 +cloudpickle==3.1.1 +cmake==4.1.0 +contourpy==1.3.2 +cryptography==46.0.1 +cycler==0.12.1 +dataclass-extensions==0.2.3 +datasets==3.6.0 +decorator==5.2.1 +deepdiff==8.6.1 +diffusers==0.35.1 +dill==0.3.8 +distro==1.9.0 +dlimp==0.0.1 +dm-tree==0.1.9 +docutils==0.22.1 +draccus==0.10.0 +editdistance==0.8.1 +einops==0.8.1 +einops-exts==0.0.4 +et_xmlfile==2.0.0 +etils==1.13.0 +evdev==1.9.2 +exceptiongroup==1.3.0 +face==24.0.0 +fastapi==0.116.2 +ffmpy==0.6.1 +fiddle==0.3.0 +filelock==3.13.1 +Flask==3.1.2 +fonttools==4.60.0 +frozenlist==1.7.0 +fsspec==2023.9.2 +ftfy==6.3.1 +gcsfs==2023.9.2 +gitdb==4.0.12 +GitPython==3.1.45 +glom==24.11.0 +google-api-core==2.25.1 +google-auth==2.40.3 +google-auth-oauthlib==1.2.2 +google-cloud-core==2.4.3 +google-cloud-storage==2.19.0 +google-crc32c==1.7.1 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +gradio==5.46.0 +gradio_client==1.13.0 +graphviz==0.21 +groovy==0.1.2 +grpcio==1.75.0 +gymnasium==0.29.1 +h11==0.16.0 +hf_transfer==0.1.9 +hf-xet==1.1.10 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==0.35.0 +id==1.5.0 +idna==3.10 +imageio==2.37.0 +imageio-ffmpeg==0.6.0 +importlib_metadata==8.7.0 +importlib_resources==6.5.2 +iniconfig==2.1.0 +inquirerpy==0.3.4 +isort==5.12.0 +itsdangerous==2.2.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.3.0 +jeepney==0.9.0 +Jinja2==3.1.4 +jiter==0.11.0 +jmespath==1.0.1 +joblib==1.5.2 +jsonlines==4.0.0 +keras==2.15.0 +keyring==25.6.0 +kiwisolver==1.4.9 +latex2sympy2_extended==1.10.2 +lerobot==0.3.4 +Levenshtein==0.27.1 +libcst==1.8.4 +lightning-utilities==0.15.2 +markdown-it-py==4.0.0 +math-verify==0.8.0 +matplotlib==3.10.6 +mdurl==0.1.2 +mergedeep==1.3.4 +ml-dtypes==0.2.0 +ml_dtypes==0.5.3 +more-itertools==10.8.0 +mpmath==1.3.0 +msgspec==0.19.0 +multidict==6.6.4 +multiprocess==0.70.16 +mypy==1.3.0 +mypy_extensions==1.1.0 +necessary==0.4.3 +networkx==3.3 +nh3==0.3.0 +nltk==3.9.1 +numpy==1.26.4 +oauthlib==3.3.1 +omegaconf==2.3.0 +openai==1.108.0 +opencv-python-headless==4.12.0.88 +OpenEXR==3.4.0 +openpyxl==3.1.5 +orderly-set==5.5.0 +orjson==3.11.3 +packaging==25.0 +pandas==2.3.2 +pathspec==0.12.1 +petname==2.6 +pfzy==0.3.4 +pillow==11.0.0 +pip==25.2 +platformdirs==4.4.0 +pluggy==1.6.0 +promise==2.3 +prompt_toolkit==3.0.52 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==4.21.12 +protobuf==6.32.1 +psutil==7.1.0 +pyarrow==21.0.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.23 +pydantic==2.11.9 +pydantic_core==2.33.2 +pydub==0.25.1 +Pygments==2.19.2 +pynput==1.8.1 +pyparsing==3.2.4 +pyproject_hooks==1.2.0 +pyserial==3.5 +pytest==8.4.2 +pytest-sphinx==0.6.3 +python-dateutil==2.9.0.post0 +python-Levenshtein==0.27.1 +python-multipart==0.0.20 +python-xlib==0.33 +pytorch-triton-rocm==3.4.0 +pytz==2025.2 +pyyaml-include==1.4.1 +RapidFuzz==3.14.1 +readme_renderer==44.0 +regex==2025.9.1 +requests==2.32.5 +requests-oauthlib==2.0.0 +requests-toolbelt==1.0.0 +requirements-parser==0.13.0 +rerun-sdk==0.22.1 +rfc3986==2.0.0 +rich==13.9.4 +rsa==4.9.1 +ruff==0.13.0 +s3transfer==0.14.0 +safehttpx==0.1.6 +safetensors==0.6.2 +scikit-learn==1.7.2 +scipy==1.15.3 +SecretStorage==3.4.0 +semantic-version==2.10.0 +sentencepiece==0.2.1 +sentry-sdk==2.38.0 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.17.0 +smart_open==7.3.1 +smashed==0.21.5 +smmap==5.0.2 +sniffio==1.3.1 +starlette==0.48.0 +sympy==1.13.3 +tensorboard==2.15.2 +tensorboard==2.19.0 +tensorflow==2.15.0 +tensorflow-addons==0.23.0 +tensorflow-datasets==4.9.3 +tensorflow-estimator==2.15.0 +tensorflow-graphics==2021.12.3 +tensorflow-metadata==1.17.2 +threadpoolctl==3.6.0 +timm==1.0.19 +tokenizers==0.22.0 +toml==0.10.2 +tomli==2.2.1 +tomlkit==0.13.3 +torch==2.8.0+rocm6.4 +torchcodec==0.5 +torchmetrics==1.8.2 +torchvision==0.23.0+rocm6.4 +tqdm==4.67.1 +transformers==4.56.1 +trimesh==4.8.2 +trouting==0.3.3 +twine==6.2.0 +typeguard==2.13.3 +typer==0.17.4 +typing_extensions==4.15.0 +typing-inspect==0.9.0 +typing-inspection==0.4.1 +tzdata==2025.2 +urllib3==2.5.0 +uvicorn==0.35.0 +wandb==0.21.4 +wcwidth==0.2.13 +websockets==15.0.1 +wheel==0.45.1 +wrapt==1.14.2 +xxhash==3.5.0 +yarl==1.20.1 +zipp==3.23.0 +lerobot==0.3.4 +minLoRA==0.1.0 +autocommand==2.2.2 +backports.tarfile==1.2.0 +importlib_metadata==8.0.0 +inflect==7.3.1 +jaraco.collections==5.1.0 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +jaraco.text==3.12.1 +more-itertools==10.3.0 +packaging==24.2 +platformdirs==4.2.2 +tomli==2.0.1 +typeguard==4.3.0 +typing_extensions==4.12.2 +wheel==0.45.1 +zipp==3.19.2 diff --git a/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/files/wandb-metadata.json b/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b1cf7c08c3085cc327b55a5a47ae80e71639e12f --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/files/wandb-metadata.json @@ -0,0 +1,202 @@ +{ + "os": "Linux-5.15.0-140-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.18", + "startedAt": "2025-09-30T12:14:56.412445Z", + "args": [ + "qwen2_7b", + "save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1", + "--vision_backbone", + "openai", + "--action_head", + "l1_regression", + "--seq_len", + "1600", + "--ft_llm", + "--checkpoint", + "/vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924", + "--device_train_microbatch_size", + "16", + "--global_batch_size", + "126", + "--dataset", + "vla_dataset_realworld", + "--llm_learning_rate", + "5e-5", + "--wandb_entity", + "henryeap", + "--wandb_project", + "a1-realworld", + "--wandb_run_name", + "realworld", + "--save_overwrite" + ], + "program": "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py", + "codePath": "launch_scripts/train_vla.py", + "codePathLocal": "launch_scripts/train_vla.py", + "git": { + "remote": "https://github.com/Spatialtemporal-AI/A1.git", + "commit": "f2afcc15e05f491a8e50add64395fc1db0a1188d" + }, + "email": "ihenrykwok@outlook.com", + "root": "/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb", + "host": "auh7-1b-gpu-306", + "executable": "/vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10", + "cpu_count": 64, + "cpu_count_logical": 128, + "gpu": "Instinct MI210", + "gpu_count": 8, + "disk": { + "/": { + "total": "470343073792", + "used": "50128797696" + } + }, + "memory": { + "total": "2434611519488" + }, + "gpu_amd": [ + { + "id": "1", + "uniqueId": "0x27087f06439a527d", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "4", + "uniqueId": "0x24ee801b7c402006", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "2", + "uniqueId": "0xaabcddaa244a3d6e", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "0", + "uniqueId": "0x82728d7f9bd937e4", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "3", + "uniqueId": "0x95be8fdc770fcfd7", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "7", + "uniqueId": "0xa0442ab3bdd405c1", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "6", + "uniqueId": "0x12140cd9e24f12e9", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "5", + "uniqueId": "0x413935505e32b8da", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + } + ], + "slurm": { + "cluster_name": "ai-04r", + "conf": "/etc/slurm/slurm.conf", + "cpus_on_node": "128", + "gpus_on_node": "8", + "gtids": "0", + "job_account": "faculty-acc", + "job_cpus_per_node": "128", + "job_end_time": "1759493653", + "job_gid": "2000", + "job_gpus": "0,1,2,3,4,5,6,7", + "job_id": "1990", + "job_name": "mh_realworld", + "job_nodelist": "auh7-1b-gpu-306", + "job_num_nodes": "1", + "job_partition": "faculty", + "job_qos": "xdqos", + "job_start_time": "1759234453", + "job_uid": "2013", + "job_user": "xiaodan", + "jobid": "1990", + "localid": "0", + "nnodes": "1", + "nodeid": "0", + "nodelist": "auh7-1b-gpu-306", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "oom_kill_step": "0", + "prio_process": "0", + "procid": "0", + "submit_dir": "/vast/users/xiaodan/zhangjian/A1/launch_scripts", + "submit_host": "auh-1b-cpu-login-001", + "task_pid": "627093", + "tasks_per_node": "1", + "topology_addr": "auh7-1b-gpu-306", + "topology_addr_pattern": "node" + }, + "writerId": "np3futv3ypcdn5zstcz9ip385cmxztpr" +} \ No newline at end of file diff --git a/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/files/wandb-summary.json b/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0b21b78871c4f850d046142ad98d6a73df4542da --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":98},"_runtime":98.460616905,"_timestamp":1.7592345947656755e+09,"_step":0,"System/Peak GPU Memory (MB)":35614.78125} \ No newline at end of file diff --git a/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/logs/debug-core.log b/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..29809d1714c841fc7a1e1aef9cc5ba3bc5e0c5dd --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/logs/debug-core.log @@ -0,0 +1,16 @@ +{"time":"2025-09-30T12:14:56.463776788Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpak81ny0c/port-627270.txt","pid":627270,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-09-30T12:14:56.464292447Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":627270} +{"time":"2025-09-30T12:14:56.464288217Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-627270-627439-1346290193/socket","Net":"unix"}} +{"time":"2025-09-30T12:14:56.647255937Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-09-30T12:14:56.65366928Z","level":"INFO","msg":"handleInformInit: received","streamId":"q6s75me3","id":"1(@)"} +{"time":"2025-09-30T12:14:57.578834615Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"q6s75me3","id":"1(@)"} +{"time":"2025-09-30T12:16:37.901143997Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"q6s75me3","id":"1(@)"} +{"time":"2025-09-30T12:16:37.904637269Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"q6s75me3","id":"1(@)"} +{"time":"2025-09-30T12:16:37.904643079Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2025-09-30T12:16:37.904651619Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2025-09-30T12:16:37.904661199Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-09-30T12:16:37.904660139Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2025-09-30T12:16:37.90470073Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2025-09-30T12:16:37.90470506Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2025-09-30T12:16:37.904758371Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-627270-627439-1346290193/socket","Net":"unix"}} +{"time":"2025-09-30T12:16:37.904779601Z","level":"INFO","msg":"server is closed"} diff --git a/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/logs/debug-internal.log b/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d86c456286adc102e15c8133d669b7ccfe327259 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/logs/debug-internal.log @@ -0,0 +1,12 @@ +{"time":"2025-09-30T12:14:56.655582724Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-09-30T12:14:57.578785874Z","level":"INFO","msg":"stream: created new stream","id":"q6s75me3"} +{"time":"2025-09-30T12:14:57.578828744Z","level":"INFO","msg":"stream: started","id":"q6s75me3"} +{"time":"2025-09-30T12:14:57.578853335Z","level":"INFO","msg":"writer: started","stream_id":"q6s75me3"} +{"time":"2025-09-30T12:14:57.578853335Z","level":"INFO","msg":"handler: started","stream_id":"q6s75me3"} +{"time":"2025-09-30T12:14:57.578883645Z","level":"INFO","msg":"sender: started","stream_id":"q6s75me3"} +{"time":"2025-09-30T12:16:36.433285201Z","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run metadata","runtime_seconds":1.666228789}],"total_operations":1}} +{"time":"2025-09-30T12:16:37.447855539Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-09-30T12:16:37.901604365Z","level":"INFO","msg":"stream: closing","id":"q6s75me3"} +{"time":"2025-09-30T12:16:37.901626916Z","level":"INFO","msg":"handler: closed","stream_id":"q6s75me3"} +{"time":"2025-09-30T12:16:37.903184523Z","level":"INFO","msg":"sender: closed","stream_id":"q6s75me3"} +{"time":"2025-09-30T12:16:37.903192913Z","level":"INFO","msg":"stream: closed","id":"q6s75me3"} diff --git a/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/logs/debug.log b/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/run-q6s75me3.wandb b/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/run-q6s75me3.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b3edfb0a38469ef79c48ea54a2f7fb48f39262ac Binary files /dev/null and b/all_l1/wandb/wandb/run-20250930_121456-q6s75me3/run-q6s75me3.wandb differ diff --git a/all_l1/wandb/wandb/run-20250930_123935-on919ofh/files/config.yaml b/all_l1/wandb/wandb/run-20250930_123935-on919ofh/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..567aa0b7dc6c0873ec5703d649c56dac7ebc5b37 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_123935-on919ofh/files/config.yaml @@ -0,0 +1,621 @@ +_wandb: + value: + cli_version: 0.21.4 + e: + ddpjl58x45oxalea5he84xtx2t0hmwvi: + args: + - qwen2_7b + - save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1 + - --vision_backbone + - openai + - --action_head + - l1_regression + - --seq_len + - "1600" + - --ft_llm + - --checkpoint + - /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 + - --device_train_microbatch_size + - "16" + - --global_batch_size + - "126" + - --dataset + - vla_dataset_realworld + - --llm_learning_rate + - "5e-5" + - --wandb_entity + - henryeap + - --wandb_project + - a1-realworld + - --wandb_run_name + - realworld + - --save_overwrite + codePath: launch_scripts/train_vla.py + codePathLocal: launch_scripts/train_vla.py + cpu_count: 64 + cpu_count_logical: 128 + disk: + /: + total: "470343073792" + used: "50365325312" + email: ihenrykwok@outlook.com + executable: /vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10 + git: + commit: f2afcc15e05f491a8e50add64395fc1db0a1188d + remote: https://github.com/Spatialtemporal-AI/A1.git + gpu: Instinct MI210 + gpu_amd: + - id: "5" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x55366988951d07ad" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "2" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x433fd00772eb3fb7" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "7" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xec87d8b9ba4d4aa9" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "3" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xdaf8107e479aecf0" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "4" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xe7ce40a1ea1cbac2" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "1" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x1633d5497b099b7f" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "0" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0xef820a5c24b0c740" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + - id: "6" + maxPower: "300.0" + mclkRange: 400Mhz - 1600Mhz + model: "0x740f" + performanceLevel: auto + sclkRange: 500Mhz - 1700Mhz + series: Instinct MI210 + sku: D67301V + uniqueId: "0x38b61e086b79539" + vbiosVersion: 113-D67301V-073 + vendor: Advanced Micro Devices, Inc. [AMD/ATI] + gpu_count: 8 + host: auh7-1b-gpu-266 + memory: + total: "2434606915584" + os: Linux-5.15.0-140-generic-x86_64-with-glibc2.35 + program: /vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py + python: CPython 3.10.18 + root: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb + slurm: + cluster_name: ai-04r + conf: /etc/slurm/slurm.conf + cpus_on_node: "128" + gpus_on_node: "8" + gtids: "0" + job_account: faculty-acc + job_cpus_per_node: "128" + job_end_time: "1759495098" + job_gid: "2000" + job_gpus: 0,1,2,3,4,5,6,7 + job_id: "1992" + job_name: mh_realworld + job_nodelist: auh7-1b-gpu-266 + job_num_nodes: "1" + job_partition: faculty + job_qos: xdqos + job_start_time: "1759235898" + job_uid: "2013" + job_user: xiaodan + jobid: "1992" + localid: "0" + nnodes: "1" + nodeid: "0" + nodelist: auh7-1b-gpu-266 + nprocs: "1" + ntasks: "1" + ntasks_per_node: "1" + oom_kill_step: "0" + prio_process: "0" + procid: "0" + submit_dir: /vast/users/xiaodan/zhangjian/A1/launch_scripts + submit_host: auh-1b-cpu-login-001 + task_pid: "3348477" + tasks_per_node: "1" + topology_addr: auh7-1b-gpu-266 + topology_addr_pattern: node + startedAt: "2025-09-30T12:39:35.212827Z" + writerId: ddpjl58x45oxalea5he84xtx2t0hmwvi + m: [] + python_version: 3.10.18 + t: + "1": + - 1 + - 2 + - 3 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 63 + - 71 + - 83 + - 95 + - 105 + "2": + - 1 + - 2 + - 3 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 63 + - 71 + - 83 + - 95 + - 105 + "3": + - 2 + - 13 + - 15 + - 16 + - 61 + "4": 3.10.18 + "5": 0.21.4 + "6": 4.56.1 + "10": + - 19 + "12": 0.21.4 + "13": linux-x86_64 +activation_checkpointing: + value: whole_layer +allow_resume: + value: false +batch_divisor: + value: global_batch +canceled_check_interval: + value: 50 +checkpoint_dir: + value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 +compile: + value: null +console_log_interval: + value: 1 +data: + value: + dataset: vla_dataset_realworld + drop_last: true + for_inference: false + lerobot_episode_index_end: null + lerobot_episode_index_start: null + mixture: null + multi_modal: torch + num_workers: 0 + pad: to_max + persistent_workers: false + pin_memory: true + prefetch_factor: null + rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds + rlds_dataset_name: libero_4_task_suites_no_noops + rlds_read_threads: 8 + rlds_shuffle_buffer_size: 100000 + rlds_traj_threads: 8 + root_size_mixture: null + seed: 95818 + sequence_length: 1600 + shuffle: true + shuffle_messages: false + split: train + timeout: 0 + use_proprio: true + use_wrist_image: true +device_eval_batch_size: + value: 4 +device_inf_eval_batch_size: + value: 16 +device_train_batch_size: + value: 15 +device_train_grad_accum: + value: 0 +device_train_microbatch_size: + value: 16 +dry_run: + value: false +early_exit: + value: false +epoch: + value: null +eval_interval: + value: 0 +eval_on_load: + value: false +eval_subset_num_batches: + value: -1 +evaluators: + value: + - data: + dataset: vla_dataset_realworld + drop_last: true + for_inference: false + lerobot_episode_index_end: 765 + lerobot_episode_index_start: 353 + mixture: null + multi_modal: torch + num_workers: 0 + pad: to_max + persistent_workers: true + pin_memory: true + prefetch_factor: null + rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds + rlds_dataset_name: libero_4_task_suites_no_noops + rlds_read_threads: 8 + rlds_shuffle_buffer_size: 256000 + rlds_traj_threads: 8 + root_size_mixture: null + seed: null + sequence_length: 1600 + shuffle: false + shuffle_messages: false + split: validation + timeout: 0 + use_proprio: true + use_wrist_image: true + device_eval_batch_size: null + eval_name: null + label: val + max_examples: null + max_new_tokens: 448 + mm_evaluator: null + save_dir: null + save_to_checkpoint_dir: false + skip_if_metrics_cached: true + subset_num_batches: 64 +extra_steps_after_cancel: + value: 10 +fast_forward_batches: + value: null +force_save_unsharded: + value: false +fsdp: + value: + hybrid_sharding_num_model_replicas: null + precision: float + sharding_strategy: FULL_SHARD + use_orig_params: true + wrapping_strategy: by_block_and_size +ft_connector: + value: false +ft_embedding: + value: lm_head +ft_llm: + value: true +ft_vit: + value: false +fused_loss: + value: null +gen1_gc_interval: + value: 1 +global_train_batch_size: + value: 126 +inf_eval_interval: + value: -1 +inf_eval_subset_num_batches: + value: -1 +inf_evaluators: + value: [] +initial_model_checkpoint: + value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924 +keep_lr_on_load: + value: true +load_model_config: + value: null +load_path: + value: null +load_path_sharded_checkpointer: + value: null +lora: + value: false +lora_connector: + value: false +lora_llm: + value: false +lora_rank: + value: 8 +lora_vit: + value: false +max_duration: + value: 500000 +max_grad_norm: + value: 1 +max_grad_norm_ratio: + value: null +model: + value: + action_dim: 7 + action_head: l1_regression + action_head_dit_depth: 28 + action_head_dit_hidden_size: 1152 + action_head_dit_num_heads: 16 + action_tokenizer: + identifier: physical-intelligence/fast + tokenizer_dir: null + action_use_left_eef: false + action_use_mobile_base: false + activation_type: swiglu + additional_vocab_size: 128 + always_start_with_space: true + attention_dropout: 0 + attention_layer_norm: false + attention_layer_norm_with_affine: true + attention_type: sdpa + bias_for_layer_norm: null + block_group_size: 1 + block_type: sequential + clip_qkv: null + crop_mode: overlap-and-resize-c2 + d_model: 3584 + default_inference_len: 65 + embedding_dropout: 0 + embedding_size: 152064 + ff_out_size: 0 + fix_image_padding: true + float32_attention: true + head_dim: null + horizon: 8 + image_feature_dropout: 0 + image_padding_embed: pad_and_partial_pad + image_pooling_2d: attention_meanq + image_pooling_h: 2 + image_pooling_w: 2 + image_projector: mlp + include_bias: false + init_cutoff_factor: null + init_device: null + init_fn: normal + init_std: 0.02 + initializer_range: 0.02 + layer_norm_eps: 1e-06 + layer_norm_type: rms + layer_norm_with_affine: true + llm_causal_attention: false + llm_load_path: /weka/oe-training-default/mm-olmo/pretrained_llms/qwen2-7b.pt + low_cpu_fsdp: true + max_crops: 12 + max_position_embeddings: null + max_sequence_length: 4096 + message_formatting: role + mlp_hidden_size: 37888 + mlp_ratio: 4 + moe_capacity_factor: 1.25 + moe_dropless: true + moe_interleave: false + moe_lbl_in_fp32: false + moe_log_expert_assignment: false + moe_loss_weight: 0.1 + moe_mlp_impl: sparse + moe_num_experts: 8 + moe_shared_expert: false + moe_top_k: 2 + moe_zloss_weight: null + multi_annotation_weighting: root_subsegments + n_heads: 28 + n_kv_heads: 4 + n_layers: 28 + new_embedding_init_range: 0.02 + norm_after: false + normalize_input_embeds: false + num_diffusion_inference_steps: 30 + num_diffusion_steps: 1000 + overlap_margins: + - 4 + - 4 + pad_tokenizer: true + pad_value: 0 + precision: amp_bf16 + prompt_type: uber_model + qkv_bias: true + residual_dropout: 0.1 + response_residual_dropout: 0 + rope: true + rope_full_precision: true + rope_theta: 1e+06 + scale_logits: false + system_prompt_kind: demo_or_style + tokenizer: + identifier: Qwen/Qwen2-7B + tokenizer_dir: null + use_col_tokens: true + use_position_ids: true + use_proprio: true + vision_backbone: + attention_dropout: 0 + fsdp_wrap: false + image_default_input_size: + - 336 + - 336 + image_dropout_rate: 0 + image_emb_dim: 1024 + image_head_dim: 64 + image_mlp_activations: quick_gelu + image_mlp_dim: 4096 + image_model_type: openai + image_norm_eps: 1e-05 + image_num_heads: 16 + image_num_key_value_heads: 16 + image_num_layers: 23 + image_num_pos: 577 + image_patch_size: 14 + image_pos_patch_size: 14 + initializer_range: 0.02 + residual_dropout: 0 + resize_mode: default + vit_layers: + - -2 + - -9 + vit_load_path: /weka/oe-training-default/mm-olmo/pretrained_image_encoders/vit-l-14-336.pt + vocab_size: 152064 + weight_tying: false +multi_component_grad_norm: + value: true +no_pre_train_checkpoint: + value: true +optimizer: + value: + betas: + - 0.9 + - 0.95 + connector_betas: + - 0.9 + - 0.95 + connector_eps: 1e-06 + connector_learning_rate: 0.0002 + connector_weight_decay: 0 + eps: 1e-05 + learning_rate: 0.0001 + llm_betas: + - 0.9 + - 0.95 + llm_eps: 1e-06 + llm_learning_rate: 5e-05 + llm_weight_decay: 0 + metrics_log_interval: 20 + name: adamw + vit_betas: + - 0.9 + - 0.95 + vit_eps: 1e-06 + vit_learning_rate: 6e-06 + vit_weight_decay: 0 + weight_decay: 0.01 +precision: + value: amp_bf16 +python_profiling: + value: false +remote_save_folder: + value: null +reset_dataloader_state: + value: false +reset_optimizer_state: + value: false +reset_trainer_state: + value: false +restore_dataloader: + value: true +run_name: + value: realworld_20250930_123854 +save_dataloader_state: + value: false +save_folder: + value: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1 +save_interval: + value: 500 +save_interval_action_head: + value: 500 +save_interval_ephemeral: + value: null +save_interval_unsharded: + value: 500 +save_num_action_head_checkpoints_to_keep: + value: 2 +save_num_checkpoints_to_keep: + value: 1 +save_num_unsharded_checkpoints_to_keep: + value: 1 +save_overwrite: + value: true +scheduler: + value: + alpha_f: 0.1 + connector_t_warmup: 200 + grad_clip_warmup_factor: null + grad_clip_warmup_steps: null + llm_t_warmup: 2000 + name: multimodal + t_max: null + t_warmup: 100 + units: steps + vit_t_warmup: 2000 + warmup_min_lr: 0 +seed: + value: 6198 +sharded_checkpointer: + value: torch_legacy +softmax_auxiliary_loss: + value: true +softmax_auxiliary_loss_scale: + value: 0.0001 +speed_monitor: + value: + gpu_flops_available: null + window_size: 20 +stop_after: + value: null +stop_at: + value: 500000 +time_limit: + value: null +torch_profiling: + value: false +train_exit_random_layer: + value: false +use_lora: + value: true diff --git a/all_l1/wandb/wandb/run-20250930_123935-on919ofh/files/output.log b/all_l1/wandb/wandb/run-20250930_123935-on919ofh/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..585b382786b5a8abbabdb5e699595af70046f329 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_123935-on919ofh/files/output.log @@ -0,0 +1,122 @@ +wandb: Detected [openai] in use. +wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script. +wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/ +09/30 [12:39:37] WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No warnings.py:109 + device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once + +****** vla_cfg: {'datasets': {'rlds': {'name': None, 'path': None, 'weight': 1.0, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe', 1, 'bounds']], 'open-source-real-world': {'rlds': {'name': 'a1_real_world', 'path': '/vast/users/xiaodan/zhangjian/datasets/OXE', 'weight': 8, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [], 'agibot': {'path': None, 'weight': 8, 'action_proprio_normalization_type': None}}}, 'model': {'action_head': {'action_dim': 16, 'proprio_dim': 16, 'num_actions_chunk': 8, 'action_tokens_mapping': {'left_end_effector': 8, 'right_end_effector': 8}, 'use_left_eef': True, 'use_mobile_base': False}}} +****** Skip RLDS main; path not found: None +****** start build LeRobot main... +build_tokenizer, cache_dir None tokenizer_dir None +09/30 [12:39:43] INFO | >> Padding tokenizer with 418 tokens tokenizer.py:130 + INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk +****** length of the dataset: 72641 +09/30 [12:39:52] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50 +****** length of the dataset: 27906 +09/30 [12:39:53] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser +****** length of the dataset: 13441 +09/30 [12:39:54] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue +****** length of the dataset: 10316 +09/30 [12:39:55] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen +****** length of the dataset: 17131 +09/30 [12:39:57] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote +****** length of the dataset: 15765 +09/30 [12:39:58] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue +****** length of the dataset: 90 + INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe +****** length of the dataset: 18397 +****** Skip RLDS open-source-real-world; mixture 'a1_real_world' not found under: /vast/users/xiaodan/zhangjian/datasets/OXE +****** Expect one of: [] +****** path: None +****** Skip AgiBotWorld-Alpha open-source-real-world; path not found: None +****** After build vla train dataset... +****** iterable_sources: [, , , , , , , ] +****** Before build mixed iterable dataset... +****** Build vla train dataloader successfully! +************************* Build train_dataloader successful! +************************* Before build_inf_evaluators +09/30 [12:39:59] WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No warnings.py:109 + device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once + +************************* Build evaluators successful! +************************* Early exit flags: early_exit=False +************************* Initialize model successful! +***** state_dict_path: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924/model.pt +***** Load checkpoint successful! +missing keys: ['action_head.model.layer_norm1.weight', 'action_head.model.layer_norm1.bias', 'action_head.model.fc1.weight', 'action_head.model.fc1.bias', 'action_head.model.mlp_resnet_blocks.0.ffn.0.weight', 'action_head.model.mlp_resnet_blocks.0.ffn.0.bias', 'action_head.model.mlp_resnet_blocks.0.ffn.1.weight', 'action_head.model.mlp_resnet_blocks.0.ffn.1.bias', 'action_head.model.mlp_resnet_blocks.1.ffn.0.weight', 'action_head.model.mlp_resnet_blocks.1.ffn.0.bias', 'action_head.model.mlp_resnet_blocks.1.ffn.1.weight', 'action_head.model.mlp_resnet_blocks.1.ffn.1.bias', 'action_head.model.layer_norm2.weight', 'action_head.model.layer_norm2.bias', 'action_head.model.fc2.weight', 'action_head.model.fc2.bias', 'proprio_projector.fc1.weight', 'proprio_projector.fc1.bias', 'proprio_projector.fc2.weight', 'proprio_projector.fc2.bias'] +unexpected keys: [] +************************* Initialize model successful! +************************* LoRA flags: use_lora=True, lora_llm=False, lora_vit=False, lora_connector=False +************************* Before add lora to model +************************* Before FSDP model wrapping +************************* FSDP model wrapping successful! +************************* Before building optimizer and scheduler +************* Before get lora params +************* After get lora params successfully +09/30 [12:41:26] INFO | >> Constructing optimizer with 2 param groups optim.py:1283 +************************************************** +After building optimizer and scheduler and model, before training, peak GPU memory (MB): 35614 +************************* VLATrainer initialized successfully! +************************* Before trainer.fit() +Pre-train system metrics + System/Peak GPU Memory (MB)=35,614 + WARNING | >> /vast/users/xiaodan/zhangjian/A1/olmo/data/collator.py:200: UserWarning: To copy construct from a tensor, it is recommended to use warnings.py:109 + sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + timestep_list = [torch.tensor(ex["timestep"], dtype=torch.int64) for ex in batch] + +!!!Training failed: Given normalized_shape=[57344], expected input with shape [*, 57344], but got input of size[15, 8, 28672] +Traceback (most recent call last): + File "/vast/users/xiaodan/zhangjian/A1/scripts/train_for_action.py", line 593, in main + trainer.fit() + File "/vast/users/xiaodan/zhangjian/A1/olmo/train.py", line 2331, in fit + metrics = self.train_step(batch, reduce_global_loss=should_log_this_step) + File "/vast/users/xiaodan/zhangjian/A1/olmo/train.py", line 2038, in train_step + ce_batch_loss, z_batch_loss, batch_accuracy, lb_batch_loss, moe_z_batch_loss, expert_assignments,action_loss = self.train_batch(batch) + File "/vast/users/xiaodan/zhangjian/A1/olmo/train.py", line 1956, in train_batch + accuracy, ce_loss, z_loss, logits, action_loss = self.model_forward( + File "/vast/users/xiaodan/zhangjian/A1/olmo/train.py", line 1813, in model_forward + outputs = self.fsdp_model.forward( + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py", line 854, in forward + output = self._fsdp_wrapped_module(*args, **kwargs) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl + return forward_call(*args, **kwargs) + File "/vast/users/xiaodan/zhangjian/A1/olmo/vla/affordvla.py", line 1042, in forward + predicted_actions = self.action_head.predict_action(action_hidden_states) + File "/vast/users/xiaodan/zhangjian/A1/olmo/vla/action_heads.py", line 249, in predict_action + action = self.model(rearranged_actions_hidden_states) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl + return forward_call(*args, **kwargs) + File "/vast/users/xiaodan/zhangjian/A1/olmo/vla/action_heads.py", line 210, in forward + x = self.layer_norm1(x) # shape: (batch_size, input_dim) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl + return forward_call(*args, **kwargs) + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/normalization.py", line 217, in forward + return F.layer_norm( + File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/functional.py", line 2905, in layer_norm + return torch.layer_norm( +RuntimeError: Given normalized_shape=[57344], expected input with shape [*, 57344], but got input of size[15, 8, 28672] +wandb: WARNING The `quiet` argument to `wandb.run.finish()` is deprecated, use `wandb.Settings(quiet=...)` to set this instead. diff --git a/all_l1/wandb/wandb/run-20250930_123935-on919ofh/files/requirements.txt b/all_l1/wandb/wandb/run-20250930_123935-on919ofh/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..5a5e971a5d9f3b6f6a75248e203682c5fbcd69e8 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_123935-on919ofh/files/requirements.txt @@ -0,0 +1,286 @@ +ai2-molmo==0.0.0 +astunparse==1.6.3 +flatbuffers==25.2.10 +gast==0.6.0 +google-pasta==0.2.0 +h5py==3.14.0 +libclang==18.1.1 +Markdown==3.9 +namex==0.1.0 +opt_einsum==3.4.0 +optree==0.17.0 +tensorboard-data-server==0.7.2 +tensorflow-io-gcs-filesystem==0.37.1 +termcolor==3.1.0 +Werkzeug==3.1.3 +Brotli==1.1.0 +Farama-Notifications==0.0.4 +MarkupSafe==2.1.5 +PyYAML==6.0.2 +absl-py==2.3.1 +accelerate==1.10.1 +ai2-molmo==0.0.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +antlr4-python3-runtime==4.9.3 +anyio==4.10.0 +array_record==0.8.1 +async-timeout==5.0.1 +attrs==25.3.0 +av==15.1.0 +backports.tarfile==1.2.0 +beaker-gantry==3.2.0 +beaker-py==2.5.0 +black==23.12.1 +blinker==1.9.0 +boltons==25.0.0 +boto3==1.40.33 +botocore==1.40.33 +build==1.3.0 +cached_path==1.7.3 +cached-property==2.0.1 +cachetools==5.5.2 +certifi==2025.8.3 +cffi==2.0.0 +charset-normalizer==3.4.3 +click==8.2.1 +click-help-colors==0.9.4 +click-option-group==0.5.7 +cloudpickle==3.1.1 +cmake==4.1.0 +contourpy==1.3.2 +cryptography==46.0.1 +cycler==0.12.1 +dataclass-extensions==0.2.3 +datasets==3.6.0 +decorator==5.2.1 +deepdiff==8.6.1 +diffusers==0.35.1 +dill==0.3.8 +distro==1.9.0 +dlimp==0.0.1 +dm-tree==0.1.9 +docutils==0.22.1 +draccus==0.10.0 +editdistance==0.8.1 +einops==0.8.1 +einops-exts==0.0.4 +et_xmlfile==2.0.0 +etils==1.13.0 +evdev==1.9.2 +exceptiongroup==1.3.0 +face==24.0.0 +fastapi==0.116.2 +ffmpy==0.6.1 +fiddle==0.3.0 +filelock==3.13.1 +Flask==3.1.2 +fonttools==4.60.0 +frozenlist==1.7.0 +fsspec==2023.9.2 +ftfy==6.3.1 +gcsfs==2023.9.2 +gitdb==4.0.12 +GitPython==3.1.45 +glom==24.11.0 +google-api-core==2.25.1 +google-auth==2.40.3 +google-auth-oauthlib==1.2.2 +google-cloud-core==2.4.3 +google-cloud-storage==2.19.0 +google-crc32c==1.7.1 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +gradio==5.46.0 +gradio_client==1.13.0 +graphviz==0.21 +groovy==0.1.2 +grpcio==1.75.0 +gymnasium==0.29.1 +h11==0.16.0 +hf_transfer==0.1.9 +hf-xet==1.1.10 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==0.35.0 +id==1.5.0 +idna==3.10 +imageio==2.37.0 +imageio-ffmpeg==0.6.0 +importlib_metadata==8.7.0 +importlib_resources==6.5.2 +iniconfig==2.1.0 +inquirerpy==0.3.4 +isort==5.12.0 +itsdangerous==2.2.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.3.0 +jeepney==0.9.0 +Jinja2==3.1.4 +jiter==0.11.0 +jmespath==1.0.1 +joblib==1.5.2 +jsonlines==4.0.0 +keras==2.15.0 +keyring==25.6.0 +kiwisolver==1.4.9 +latex2sympy2_extended==1.10.2 +lerobot==0.3.4 +Levenshtein==0.27.1 +libcst==1.8.4 +lightning-utilities==0.15.2 +markdown-it-py==4.0.0 +math-verify==0.8.0 +matplotlib==3.10.6 +mdurl==0.1.2 +mergedeep==1.3.4 +ml-dtypes==0.2.0 +ml_dtypes==0.5.3 +more-itertools==10.8.0 +mpmath==1.3.0 +msgspec==0.19.0 +multidict==6.6.4 +multiprocess==0.70.16 +mypy==1.3.0 +mypy_extensions==1.1.0 +necessary==0.4.3 +networkx==3.3 +nh3==0.3.0 +nltk==3.9.1 +numpy==1.26.4 +oauthlib==3.3.1 +omegaconf==2.3.0 +openai==1.108.0 +opencv-python-headless==4.12.0.88 +OpenEXR==3.4.0 +openpyxl==3.1.5 +orderly-set==5.5.0 +orjson==3.11.3 +packaging==25.0 +pandas==2.3.2 +pathspec==0.12.1 +petname==2.6 +pfzy==0.3.4 +pillow==11.0.0 +pip==25.2 +platformdirs==4.4.0 +pluggy==1.6.0 +promise==2.3 +prompt_toolkit==3.0.52 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==4.21.12 +protobuf==6.32.1 +psutil==7.1.0 +pyarrow==21.0.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.23 +pydantic==2.11.9 +pydantic_core==2.33.2 +pydub==0.25.1 +Pygments==2.19.2 +pynput==1.8.1 +pyparsing==3.2.4 +pyproject_hooks==1.2.0 +pyserial==3.5 +pytest==8.4.2 +pytest-sphinx==0.6.3 +python-dateutil==2.9.0.post0 +python-Levenshtein==0.27.1 +python-multipart==0.0.20 +python-xlib==0.33 +pytorch-triton-rocm==3.4.0 +pytz==2025.2 +pyyaml-include==1.4.1 +RapidFuzz==3.14.1 +readme_renderer==44.0 +regex==2025.9.1 +requests==2.32.5 +requests-oauthlib==2.0.0 +requests-toolbelt==1.0.0 +requirements-parser==0.13.0 +rerun-sdk==0.22.1 +rfc3986==2.0.0 +rich==13.9.4 +rsa==4.9.1 +ruff==0.13.0 +s3transfer==0.14.0 +safehttpx==0.1.6 +safetensors==0.6.2 +scikit-learn==1.7.2 +scipy==1.15.3 +SecretStorage==3.4.0 +semantic-version==2.10.0 +sentencepiece==0.2.1 +sentry-sdk==2.38.0 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.17.0 +smart_open==7.3.1 +smashed==0.21.5 +smmap==5.0.2 +sniffio==1.3.1 +starlette==0.48.0 +sympy==1.13.3 +tensorboard==2.15.2 +tensorboard==2.19.0 +tensorflow==2.15.0 +tensorflow-addons==0.23.0 +tensorflow-datasets==4.9.3 +tensorflow-estimator==2.15.0 +tensorflow-graphics==2021.12.3 +tensorflow-metadata==1.17.2 +threadpoolctl==3.6.0 +timm==1.0.19 +tokenizers==0.22.0 +toml==0.10.2 +tomli==2.2.1 +tomlkit==0.13.3 +torch==2.8.0+rocm6.4 +torchcodec==0.5 +torchmetrics==1.8.2 +torchvision==0.23.0+rocm6.4 +tqdm==4.67.1 +transformers==4.56.1 +trimesh==4.8.2 +trouting==0.3.3 +twine==6.2.0 +typeguard==2.13.3 +typer==0.17.4 +typing_extensions==4.15.0 +typing-inspect==0.9.0 +typing-inspection==0.4.1 +tzdata==2025.2 +urllib3==2.5.0 +uvicorn==0.35.0 +wandb==0.21.4 +wcwidth==0.2.13 +websockets==15.0.1 +wheel==0.45.1 +wrapt==1.14.2 +xxhash==3.5.0 +yarl==1.20.1 +zipp==3.23.0 +lerobot==0.3.4 +minLoRA==0.1.0 +autocommand==2.2.2 +backports.tarfile==1.2.0 +importlib_metadata==8.0.0 +inflect==7.3.1 +jaraco.collections==5.1.0 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +jaraco.text==3.12.1 +more-itertools==10.3.0 +packaging==24.2 +platformdirs==4.2.2 +tomli==2.0.1 +typeguard==4.3.0 +typing_extensions==4.12.2 +wheel==0.45.1 +zipp==3.19.2 diff --git a/all_l1/wandb/wandb/run-20250930_123935-on919ofh/files/wandb-metadata.json b/all_l1/wandb/wandb/run-20250930_123935-on919ofh/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5762a7bb91d928d9a80f47410b88ca55fb42a8a6 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_123935-on919ofh/files/wandb-metadata.json @@ -0,0 +1,202 @@ +{ + "os": "Linux-5.15.0-140-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.18", + "startedAt": "2025-09-30T12:39:35.212827Z", + "args": [ + "qwen2_7b", + "save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1", + "--vision_backbone", + "openai", + "--action_head", + "l1_regression", + "--seq_len", + "1600", + "--ft_llm", + "--checkpoint", + "/vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924", + "--device_train_microbatch_size", + "16", + "--global_batch_size", + "126", + "--dataset", + "vla_dataset_realworld", + "--llm_learning_rate", + "5e-5", + "--wandb_entity", + "henryeap", + "--wandb_project", + "a1-realworld", + "--wandb_run_name", + "realworld", + "--save_overwrite" + ], + "program": "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py", + "codePath": "launch_scripts/train_vla.py", + "codePathLocal": "launch_scripts/train_vla.py", + "git": { + "remote": "https://github.com/Spatialtemporal-AI/A1.git", + "commit": "f2afcc15e05f491a8e50add64395fc1db0a1188d" + }, + "email": "ihenrykwok@outlook.com", + "root": "/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb", + "host": "auh7-1b-gpu-266", + "executable": "/vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10", + "cpu_count": 64, + "cpu_count_logical": 128, + "gpu": "Instinct MI210", + "gpu_count": 8, + "disk": { + "/": { + "total": "470343073792", + "used": "50365325312" + } + }, + "memory": { + "total": "2434606915584" + }, + "gpu_amd": [ + { + "id": "5", + "uniqueId": "0x55366988951d07ad", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "2", + "uniqueId": "0x433fd00772eb3fb7", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "7", + "uniqueId": "0xec87d8b9ba4d4aa9", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "3", + "uniqueId": "0xdaf8107e479aecf0", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "4", + "uniqueId": "0xe7ce40a1ea1cbac2", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "1", + "uniqueId": "0x1633d5497b099b7f", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "0", + "uniqueId": "0xef820a5c24b0c740", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "6", + "uniqueId": "0x38b61e086b79539", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + } + ], + "slurm": { + "cluster_name": "ai-04r", + "conf": "/etc/slurm/slurm.conf", + "cpus_on_node": "128", + "gpus_on_node": "8", + "gtids": "0", + "job_account": "faculty-acc", + "job_cpus_per_node": "128", + "job_end_time": "1759495098", + "job_gid": "2000", + "job_gpus": "0,1,2,3,4,5,6,7", + "job_id": "1992", + "job_name": "mh_realworld", + "job_nodelist": "auh7-1b-gpu-266", + "job_num_nodes": "1", + "job_partition": "faculty", + "job_qos": "xdqos", + "job_start_time": "1759235898", + "job_uid": "2013", + "job_user": "xiaodan", + "jobid": "1992", + "localid": "0", + "nnodes": "1", + "nodeid": "0", + "nodelist": "auh7-1b-gpu-266", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "oom_kill_step": "0", + "prio_process": "0", + "procid": "0", + "submit_dir": "/vast/users/xiaodan/zhangjian/A1/launch_scripts", + "submit_host": "auh-1b-cpu-login-001", + "task_pid": "3348477", + "tasks_per_node": "1", + "topology_addr": "auh7-1b-gpu-266", + "topology_addr_pattern": "node" + }, + "writerId": "ddpjl58x45oxalea5he84xtx2t0hmwvi" +} \ No newline at end of file diff --git a/all_l1/wandb/wandb/run-20250930_123935-on919ofh/files/wandb-summary.json b/all_l1/wandb/wandb/run-20250930_123935-on919ofh/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..2845286b5b86f811ffeb1607ee5b370273cdb1d3 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_123935-on919ofh/files/wandb-summary.json @@ -0,0 +1 @@ +{"System/Peak GPU Memory (MB)":35614.78125,"_wandb":{"runtime":117},"_runtime":117.495642133,"_timestamp":1.7592360860861056e+09,"_step":0} \ No newline at end of file diff --git a/all_l1/wandb/wandb/run-20250930_123935-on919ofh/logs/debug-core.log b/all_l1/wandb/wandb/run-20250930_123935-on919ofh/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..7da1929de6f736f50f9ef711615d13821c5a9f9e --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_123935-on919ofh/logs/debug-core.log @@ -0,0 +1,16 @@ +{"time":"2025-09-30T12:39:35.531288321Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpfxbw7uln/port-3348659.txt","pid":3348659,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-09-30T12:39:35.533267395Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3348659} +{"time":"2025-09-30T12:39:35.534759111Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3348659-3348834-3851720927/socket","Net":"unix"}} +{"time":"2025-09-30T12:39:35.667946335Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-09-30T12:39:35.680795333Z","level":"INFO","msg":"handleInformInit: received","streamId":"on919ofh","id":"1(@)"} +{"time":"2025-09-30T12:39:37.028776178Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"on919ofh","id":"1(@)"} +{"time":"2025-09-30T12:41:36.891520431Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"on919ofh","id":"1(@)"} +{"time":"2025-09-30T12:41:36.894745416Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"on919ofh","id":"1(@)"} +{"time":"2025-09-30T12:41:36.894752626Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"} +{"time":"2025-09-30T12:41:36.894759416Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"} +{"time":"2025-09-30T12:41:36.894768886Z","level":"INFO","msg":"server is shutting down"} +{"time":"2025-09-30T12:41:36.894767326Z","level":"INFO","msg":"connection: closing","id":"1(@)"} +{"time":"2025-09-30T12:41:36.894807567Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"} +{"time":"2025-09-30T12:41:36.894811637Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"} +{"time":"2025-09-30T12:41:36.894869608Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3348659-3348834-3851720927/socket","Net":"unix"}} +{"time":"2025-09-30T12:41:36.894890678Z","level":"INFO","msg":"server is closed"} diff --git a/all_l1/wandb/wandb/run-20250930_123935-on919ofh/logs/debug-internal.log b/all_l1/wandb/wandb/run-20250930_123935-on919ofh/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..0361c0ad20a583e125b493480b5bf29a6b61525a --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_123935-on919ofh/logs/debug-internal.log @@ -0,0 +1,12 @@ +{"time":"2025-09-30T12:39:35.682847778Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-09-30T12:39:37.028720157Z","level":"INFO","msg":"stream: created new stream","id":"on919ofh"} +{"time":"2025-09-30T12:39:37.028770978Z","level":"INFO","msg":"stream: started","id":"on919ofh"} +{"time":"2025-09-30T12:39:37.028806479Z","level":"INFO","msg":"writer: started","stream_id":"on919ofh"} +{"time":"2025-09-30T12:39:37.028813859Z","level":"INFO","msg":"handler: started","stream_id":"on919ofh"} +{"time":"2025-09-30T12:39:37.028829659Z","level":"INFO","msg":"sender: started","stream_id":"on919ofh"} +{"time":"2025-09-30T12:41:34.878202555Z","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run metadata","runtime_seconds":0.001991154}],"total_operations":1}} +{"time":"2025-09-30T12:41:36.521701684Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-09-30T12:41:36.891885208Z","level":"INFO","msg":"stream: closing","id":"on919ofh"} +{"time":"2025-09-30T12:41:36.891903148Z","level":"INFO","msg":"handler: closed","stream_id":"on919ofh"} +{"time":"2025-09-30T12:41:36.893444534Z","level":"INFO","msg":"sender: closed","stream_id":"on919ofh"} +{"time":"2025-09-30T12:41:36.893451044Z","level":"INFO","msg":"stream: closed","id":"on919ofh"} diff --git a/all_l1/wandb/wandb/run-20250930_123935-on919ofh/logs/debug.log b/all_l1/wandb/wandb/run-20250930_123935-on919ofh/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/all_l1/wandb/wandb/run-20250930_123935-on919ofh/run-on919ofh.wandb b/all_l1/wandb/wandb/run-20250930_123935-on919ofh/run-on919ofh.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e87d6aa220ca0a894a8c1eac0b724132e5fa6348 Binary files /dev/null and b/all_l1/wandb/wandb/run-20250930_123935-on919ofh/run-on919ofh.wandb differ diff --git a/all_l1/wandb/wandb/run-20250930_131250-ea1k0g3y/files/output.log b/all_l1/wandb/wandb/run-20250930_131250-ea1k0g3y/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1bb88f27b27896eb4b02da358511d15067d0b892 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_131250-ea1k0g3y/files/output.log @@ -0,0 +1,45224 @@ +wandb: Detected [openai] in use. +wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script. +wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/ +09/30 [13:12:52] WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No warnings.py:109 + device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once + +****** vla_cfg: {'datasets': {'rlds': {'name': None, 'path': None, 'weight': 1.0, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe', 1, 'bounds']], 'open-source-real-world': {'rlds': {'name': 'a1_real_world', 'path': '/vast/users/xiaodan/zhangjian/datasets/OXE', 'weight': 8, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [], 'agibot': {'path': None, 'weight': 8, 'action_proprio_normalization_type': None}}}, 'model': {'action_head': {'action_dim': 16, 'proprio_dim': 16, 'num_actions_chunk': 8, 'action_tokens_mapping': {'left_end_effector': 8, 'right_end_effector': 8}, 'use_left_eef': True, 'use_mobile_base': False}}} +****** Skip RLDS main; path not found: None +****** start build LeRobot main... +build_tokenizer, cache_dir None tokenizer_dir None +09/30 [13:12:53] INFO | >> Padding tokenizer with 418 tokens tokenizer.py:130 +09/30 [13:12:54] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk +****** length of the dataset: 72641 +09/30 [13:13:02] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50 +****** length of the dataset: 27906 +09/30 [13:13:09] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser +****** length of the dataset: 13441 +09/30 [13:13:10] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue +****** length of the dataset: 10316 +09/30 [13:13:11] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen +****** length of the dataset: 17131 +09/30 [13:13:12] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote +****** length of the dataset: 15765 +09/30 [13:13:13] INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue +****** length of the dataset: 90 + INFO | >> Loading train dataset: vla_dataset_realworld/train __init__.py:435 +****** before LeRobot dataset... +****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe +****** length of the dataset: 18397 +****** Skip RLDS open-source-real-world; mixture 'a1_real_world' not found under: /vast/users/xiaodan/zhangjian/datasets/OXE +****** Expect one of: [] +****** path: None +****** Skip AgiBotWorld-Alpha open-source-real-world; path not found: None +****** After build vla train dataset... +****** iterable_sources: [, , , , , , , ] +****** Before build mixed iterable dataset... +****** Build vla train dataloader successfully! +************************* Build train_dataloader successful! +************************* Before build_inf_evaluators +09/30 [13:13:14] WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No warnings.py:109 + device id is provided via `init_process_group` or `barrier `. Using the current device set by the user. + warnings.warn( # warn only once + +************************* Build evaluators successful! +************************* Early exit flags: early_exit=False +************************* Initialize model successful! +***** state_dict_path: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924/model.pt +***** Load checkpoint successful! +missing keys: ['action_head.model.layer_norm1.weight', 'action_head.model.layer_norm1.bias', 'action_head.model.fc1.weight', 'action_head.model.fc1.bias', 'action_head.model.mlp_resnet_blocks.0.ffn.0.weight', 'action_head.model.mlp_resnet_blocks.0.ffn.0.bias', 'action_head.model.mlp_resnet_blocks.0.ffn.1.weight', 'action_head.model.mlp_resnet_blocks.0.ffn.1.bias', 'action_head.model.mlp_resnet_blocks.1.ffn.0.weight', 'action_head.model.mlp_resnet_blocks.1.ffn.0.bias', 'action_head.model.mlp_resnet_blocks.1.ffn.1.weight', 'action_head.model.mlp_resnet_blocks.1.ffn.1.bias', 'action_head.model.layer_norm2.weight', 'action_head.model.layer_norm2.bias', 'action_head.model.fc2.weight', 'action_head.model.fc2.bias', 'proprio_projector.fc1.weight', 'proprio_projector.fc1.bias', 'proprio_projector.fc2.weight', 'proprio_projector.fc2.bias'] +unexpected keys: [] +************************* Initialize model successful! +************************* LoRA flags: use_lora=True, lora_llm=False, lora_vit=False, lora_connector=False +************************* Before add lora to model +************************* Before FSDP model wrapping +************************* FSDP model wrapping successful! +************************* Before building optimizer and scheduler +************* Before get lora params +************* After get lora params successfully +09/30 [13:14:27] INFO | >> Constructing optimizer with 2 param groups optim.py:1283 +************************************************** +After building optimizer and scheduler and model, before training, peak GPU memory (MB): 35614 +************************* VLATrainer initialized successfully! +************************* Before trainer.fit() +Pre-train system metrics + System/Peak GPU Memory (MB)=35,614 + WARNING | >> /vast/users/xiaodan/zhangjian/A1/olmo/data/collator.py:200: UserWarning: To copy construct from a tensor, it is recommended to use warnings.py:109 + sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor). + timestep_list = [torch.tensor(ex["timestep"], dtype=torch.int64) for ex in batch] + +[step=1/500000] + train/ActionL1Loss=0.6793 + throughput/total_tokens=192,000 + System/Peak GPU Memory (MB)=40,145 +[step=2/500000] + train/ActionL1Loss=0.5930 + throughput/total_tokens=384,000 + throughput/device/tokens_per_second=1,200 + throughput/device/batches_per_second=0.0500 + System/Peak GPU Memory (MB)=46,917 +[step=3/500000] + train/ActionL1Loss=0.6559 + throughput/total_tokens=576,000 + throughput/device/tokens_per_second=1,172 + throughput/device/batches_per_second=0.0489 +[step=4/500000] + train/ActionL1Loss=0.6283 + throughput/total_tokens=768,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5/500000] + train/ActionL1Loss=0.5817 + throughput/total_tokens=960,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=6/500000] + train/ActionL1Loss=0.5941 + throughput/total_tokens=1,152,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=7/500000] + train/ActionL1Loss=0.5921 + throughput/total_tokens=1,344,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=8/500000] + train/ActionL1Loss=0.5627 + throughput/total_tokens=1,536,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=9/500000] + train/ActionL1Loss=0.5788 + throughput/total_tokens=1,728,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=10/500000] + train/ActionL1Loss=0.5913 + throughput/total_tokens=1,920,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=11/500000] + train/ActionL1Loss=0.5938 + throughput/total_tokens=2,112,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=12/500000] + train/ActionL1Loss=0.5775 + throughput/total_tokens=2,304,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=13/500000] + train/ActionL1Loss=0.5672 + throughput/total_tokens=2,496,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=14/500000] + train/ActionL1Loss=0.5261 + throughput/total_tokens=2,688,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=15/500000] + train/ActionL1Loss=0.5075 + throughput/total_tokens=2,880,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=16/500000] + train/ActionL1Loss=0.5681 + throughput/total_tokens=3,072,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=17/500000] + train/ActionL1Loss=0.5502 + throughput/total_tokens=3,264,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=18/500000] + train/ActionL1Loss=0.5460 + throughput/total_tokens=3,456,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=19/500000] + train/ActionL1Loss=0.5226 + throughput/total_tokens=3,648,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=20/500000] + optim/total_grad_norm=19.49 + train/ActionL1Loss=0.5248 + throughput/total_tokens=3,840,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 + System/Peak GPU Memory (MB)=46,917 +[step=21/500000] + train/ActionL1Loss=0.6231 + throughput/total_tokens=4,032,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=22/500000] + train/ActionL1Loss=0.5242 + throughput/total_tokens=4,224,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=23/500000] + train/ActionL1Loss=0.5158 + throughput/total_tokens=4,416,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=24/500000] + train/ActionL1Loss=0.5881 + throughput/total_tokens=4,608,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=25/500000] + train/ActionL1Loss=0.4934 + throughput/total_tokens=4,800,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=26/500000] + train/ActionL1Loss=0.5082 + throughput/total_tokens=4,992,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=27/500000] + train/ActionL1Loss=0.5737 + throughput/total_tokens=5,184,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=28/500000] + train/ActionL1Loss=0.5663 + throughput/total_tokens=5,376,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=29/500000] + train/ActionL1Loss=0.5887 + throughput/total_tokens=5,568,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=30/500000] + train/ActionL1Loss=0.5154 + throughput/total_tokens=5,760,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0481 + System/Peak GPU Memory (MB)=46,917 +[step=31/500000] + train/ActionL1Loss=0.5592 + throughput/total_tokens=5,952,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0481 +[step=32/500000] + train/ActionL1Loss=0.5018 + throughput/total_tokens=6,144,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0481 +[step=33/500000] + train/ActionL1Loss=0.5105 + throughput/total_tokens=6,336,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=34/500000] + train/ActionL1Loss=0.6220 + throughput/total_tokens=6,528,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=35/500000] + train/ActionL1Loss=0.5397 + throughput/total_tokens=6,720,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=36/500000] + train/ActionL1Loss=0.5327 + throughput/total_tokens=6,912,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=37/500000] + train/ActionL1Loss=0.5355 + throughput/total_tokens=7,104,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=38/500000] + train/ActionL1Loss=0.4923 + throughput/total_tokens=7,296,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=39/500000] + train/ActionL1Loss=0.5807 + throughput/total_tokens=7,488,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=40/500000] + optim/total_grad_norm=15.46 + train/ActionL1Loss=0.4993 + throughput/total_tokens=7,680,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 + System/Peak GPU Memory (MB)=46,917 +[step=41/500000] + train/ActionL1Loss=0.5361 + throughput/total_tokens=7,872,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=42/500000] + train/ActionL1Loss=0.5536 + throughput/total_tokens=8,064,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=43/500000] + train/ActionL1Loss=0.5528 + throughput/total_tokens=8,256,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=44/500000] + train/ActionL1Loss=0.6008 + throughput/total_tokens=8,448,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=45/500000] + train/ActionL1Loss=0.5148 + throughput/total_tokens=8,640,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=46/500000] + train/ActionL1Loss=0.5229 + throughput/total_tokens=8,832,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=47/500000] + train/ActionL1Loss=0.5188 + throughput/total_tokens=9,024,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=48/500000] + train/ActionL1Loss=0.5267 + throughput/total_tokens=9,216,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=49/500000] + train/ActionL1Loss=0.5068 + throughput/total_tokens=9,408,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=50/500000] + train/ActionL1Loss=0.5800 + throughput/total_tokens=9,600,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0481 + System/Peak GPU Memory (MB)=46,917 +[step=51/500000] + train/ActionL1Loss=0.5701 + throughput/total_tokens=9,792,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 +[step=52/500000] + train/ActionL1Loss=0.5594 + throughput/total_tokens=9,984,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 +[step=53/500000] + train/ActionL1Loss=0.5317 + throughput/total_tokens=10,176,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=54/500000] + train/ActionL1Loss=0.5531 + throughput/total_tokens=10,368,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=55/500000] + train/ActionL1Loss=0.4996 + throughput/total_tokens=10,560,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=56/500000] + train/ActionL1Loss=0.5236 + throughput/total_tokens=10,752,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=57/500000] + train/ActionL1Loss=0.5663 + throughput/total_tokens=10,944,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=58/500000] + train/ActionL1Loss=0.5745 + throughput/total_tokens=11,136,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=59/500000] + train/ActionL1Loss=0.5446 + throughput/total_tokens=11,328,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=60/500000] + optim/total_grad_norm=23.71 + train/ActionL1Loss=0.5159 + throughput/total_tokens=11,520,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0480 + System/Peak GPU Memory (MB)=46,917 +[step=61/500000] + train/ActionL1Loss=0.5215 + throughput/total_tokens=11,712,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=62/500000] + train/ActionL1Loss=0.5102 + throughput/total_tokens=11,904,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 +[step=63/500000] + train/ActionL1Loss=0.4673 + throughput/total_tokens=12,096,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0480 +[step=64/500000] + train/ActionL1Loss=0.5513 + throughput/total_tokens=12,288,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0480 +[step=65/500000] + train/ActionL1Loss=0.5553 + throughput/total_tokens=12,480,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0480 +[step=66/500000] + train/ActionL1Loss=0.5202 + throughput/total_tokens=12,672,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=67/500000] + train/ActionL1Loss=0.5049 + throughput/total_tokens=12,864,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=68/500000] + train/ActionL1Loss=0.5740 + throughput/total_tokens=13,056,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=69/500000] + train/ActionL1Loss=0.5250 + throughput/total_tokens=13,248,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=70/500000] + train/ActionL1Loss=0.5484 + throughput/total_tokens=13,440,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=71/500000] + train/ActionL1Loss=0.5563 + throughput/total_tokens=13,632,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=72/500000] + train/ActionL1Loss=0.4984 + throughput/total_tokens=13,824,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=73/500000] + train/ActionL1Loss=0.5155 + throughput/total_tokens=14,016,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=74/500000] + train/ActionL1Loss=0.5304 + throughput/total_tokens=14,208,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=75/500000] + train/ActionL1Loss=0.5154 + throughput/total_tokens=14,400,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=76/500000] + train/ActionL1Loss=0.4780 + throughput/total_tokens=14,592,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=77/500000] + train/ActionL1Loss=0.4628 + throughput/total_tokens=14,784,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=78/500000] + train/ActionL1Loss=0.5457 + throughput/total_tokens=14,976,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=79/500000] + train/ActionL1Loss=0.5861 + throughput/total_tokens=15,168,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=80/500000] + optim/total_grad_norm=22.39 + train/ActionL1Loss=0.4898 + throughput/total_tokens=15,360,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=81/500000] + train/ActionL1Loss=0.6331 + throughput/total_tokens=15,552,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=82/500000] + train/ActionL1Loss=0.5654 + throughput/total_tokens=15,744,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=83/500000] + train/ActionL1Loss=0.5584 + throughput/total_tokens=15,936,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=84/500000] + train/ActionL1Loss=0.5468 + throughput/total_tokens=16,128,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=85/500000] + train/ActionL1Loss=0.5248 + throughput/total_tokens=16,320,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=86/500000] + train/ActionL1Loss=0.5029 + throughput/total_tokens=16,512,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=87/500000] + train/ActionL1Loss=0.5210 + throughput/total_tokens=16,704,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=88/500000] + train/ActionL1Loss=0.5187 + throughput/total_tokens=16,896,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=89/500000] + train/ActionL1Loss=0.5067 + throughput/total_tokens=17,088,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=90/500000] + train/ActionL1Loss=0.4625 + throughput/total_tokens=17,280,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=91/500000] + train/ActionL1Loss=0.5168 + throughput/total_tokens=17,472,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=92/500000] + train/ActionL1Loss=0.5409 + throughput/total_tokens=17,664,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=93/500000] + train/ActionL1Loss=0.5601 + throughput/total_tokens=17,856,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=94/500000] + train/ActionL1Loss=0.4243 + throughput/total_tokens=18,048,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=95/500000] + train/ActionL1Loss=0.5194 + throughput/total_tokens=18,240,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=96/500000] + train/ActionL1Loss=0.5665 + throughput/total_tokens=18,432,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=97/500000] + train/ActionL1Loss=0.4418 + throughput/total_tokens=18,624,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=98/500000] + train/ActionL1Loss=0.5506 + throughput/total_tokens=18,816,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=99/500000] + train/ActionL1Loss=0.4717 + throughput/total_tokens=19,008,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=100/500000] + optim/total_grad_norm=17.61 + train/ActionL1Loss=0.5248 + throughput/total_tokens=19,200,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=101/500000] + train/ActionL1Loss=0.5125 + throughput/total_tokens=19,392,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=102/500000] + train/ActionL1Loss=0.5354 + throughput/total_tokens=19,584,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=103/500000] + train/ActionL1Loss=0.5102 + throughput/total_tokens=19,776,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=104/500000] + train/ActionL1Loss=0.5482 + throughput/total_tokens=19,968,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=105/500000] + train/ActionL1Loss=0.5705 + throughput/total_tokens=20,160,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=106/500000] + train/ActionL1Loss=0.4926 + throughput/total_tokens=20,352,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=107/500000] + train/ActionL1Loss=0.5551 + throughput/total_tokens=20,544,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=108/500000] + train/ActionL1Loss=0.5551 + throughput/total_tokens=20,736,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=109/500000] + train/ActionL1Loss=0.4997 + throughput/total_tokens=20,928,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=110/500000] + train/ActionL1Loss=0.5270 + throughput/total_tokens=21,120,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 + System/Peak GPU Memory (MB)=46,917 +[step=111/500000] + train/ActionL1Loss=0.5325 + throughput/total_tokens=21,312,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=112/500000] + train/ActionL1Loss=0.4957 + throughput/total_tokens=21,504,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=113/500000] + train/ActionL1Loss=0.5392 + throughput/total_tokens=21,696,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=114/500000] + train/ActionL1Loss=0.5045 + throughput/total_tokens=21,888,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=115/500000] + train/ActionL1Loss=0.5695 + throughput/total_tokens=22,080,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=116/500000] + train/ActionL1Loss=0.5164 + throughput/total_tokens=22,272,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=117/500000] + train/ActionL1Loss=0.5523 + throughput/total_tokens=22,464,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=118/500000] + train/ActionL1Loss=0.5445 + throughput/total_tokens=22,656,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=119/500000] + train/ActionL1Loss=0.4686 + throughput/total_tokens=22,848,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=120/500000] + optim/total_grad_norm=14.19 + train/ActionL1Loss=0.5430 + throughput/total_tokens=23,040,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=121/500000] + train/ActionL1Loss=0.5473 + throughput/total_tokens=23,232,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=122/500000] + train/ActionL1Loss=0.5987 + throughput/total_tokens=23,424,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=123/500000] + train/ActionL1Loss=0.5515 + throughput/total_tokens=23,616,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=124/500000] + train/ActionL1Loss=0.5459 + throughput/total_tokens=23,808,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=125/500000] + train/ActionL1Loss=0.5786 + throughput/total_tokens=24,000,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=126/500000] + train/ActionL1Loss=0.5208 + throughput/total_tokens=24,192,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=127/500000] + train/ActionL1Loss=0.5503 + throughput/total_tokens=24,384,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=128/500000] + train/ActionL1Loss=0.5833 + throughput/total_tokens=24,576,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=129/500000] + train/ActionL1Loss=0.5338 + throughput/total_tokens=24,768,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=130/500000] + train/ActionL1Loss=0.5428 + throughput/total_tokens=24,960,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=131/500000] + train/ActionL1Loss=0.4783 + throughput/total_tokens=25,152,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=132/500000] + train/ActionL1Loss=0.5188 + throughput/total_tokens=25,344,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=133/500000] + train/ActionL1Loss=0.5806 + throughput/total_tokens=25,536,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=134/500000] + train/ActionL1Loss=0.5368 + throughput/total_tokens=25,728,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=135/500000] + train/ActionL1Loss=0.5088 + throughput/total_tokens=25,920,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=136/500000] + train/ActionL1Loss=0.5313 + throughput/total_tokens=26,112,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=137/500000] + train/ActionL1Loss=0.5029 + throughput/total_tokens=26,304,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=138/500000] + train/ActionL1Loss=0.5067 + throughput/total_tokens=26,496,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=139/500000] + train/ActionL1Loss=0.5152 + throughput/total_tokens=26,688,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=140/500000] + optim/total_grad_norm=23.63 + train/ActionL1Loss=0.5234 + throughput/total_tokens=26,880,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=141/500000] + train/ActionL1Loss=0.4837 + throughput/total_tokens=27,072,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=142/500000] + train/ActionL1Loss=0.4590 + throughput/total_tokens=27,264,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=143/500000] + train/ActionL1Loss=0.5121 + throughput/total_tokens=27,456,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=144/500000] + train/ActionL1Loss=0.4559 + throughput/total_tokens=27,648,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=145/500000] + train/ActionL1Loss=0.4668 + throughput/total_tokens=27,840,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=146/500000] + train/ActionL1Loss=0.4233 + throughput/total_tokens=28,032,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=147/500000] + train/ActionL1Loss=0.4365 + throughput/total_tokens=28,224,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=148/500000] + train/ActionL1Loss=0.4604 + throughput/total_tokens=28,416,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=149/500000] + train/ActionL1Loss=0.3949 + throughput/total_tokens=28,608,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=150/500000] + train/ActionL1Loss=0.3900 + throughput/total_tokens=28,800,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=151/500000] + train/ActionL1Loss=0.3793 + throughput/total_tokens=28,992,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=152/500000] + train/ActionL1Loss=0.3793 + throughput/total_tokens=29,184,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=153/500000] + train/ActionL1Loss=0.3564 + throughput/total_tokens=29,376,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=154/500000] + train/ActionL1Loss=0.3782 + throughput/total_tokens=29,568,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=155/500000] + train/ActionL1Loss=0.3420 + throughput/total_tokens=29,760,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=156/500000] + train/ActionL1Loss=0.3574 + throughput/total_tokens=29,952,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=157/500000] + train/ActionL1Loss=0.3317 + throughput/total_tokens=30,144,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=158/500000] + train/ActionL1Loss=0.3487 + throughput/total_tokens=30,336,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=159/500000] + train/ActionL1Loss=0.3473 + throughput/total_tokens=30,528,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=160/500000] + optim/total_grad_norm=25.91 + train/ActionL1Loss=0.3520 + throughput/total_tokens=30,720,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 + System/Peak GPU Memory (MB)=46,917 +[step=161/500000] + train/ActionL1Loss=0.3086 + throughput/total_tokens=30,912,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=162/500000] + train/ActionL1Loss=0.3156 + throughput/total_tokens=31,104,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=163/500000] + train/ActionL1Loss=0.3163 + throughput/total_tokens=31,296,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=164/500000] + train/ActionL1Loss=0.3323 + throughput/total_tokens=31,488,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=165/500000] + train/ActionL1Loss=0.3094 + throughput/total_tokens=31,680,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=166/500000] + train/ActionL1Loss=0.3056 + throughput/total_tokens=31,872,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=167/500000] + train/ActionL1Loss=0.3072 + throughput/total_tokens=32,064,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=168/500000] + train/ActionL1Loss=0.3139 + throughput/total_tokens=32,256,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=169/500000] + train/ActionL1Loss=0.2983 + throughput/total_tokens=32,448,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=170/500000] + train/ActionL1Loss=0.3091 + throughput/total_tokens=32,640,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=171/500000] + train/ActionL1Loss=0.3064 + throughput/total_tokens=32,832,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=172/500000] + train/ActionL1Loss=0.2767 + throughput/total_tokens=33,024,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=173/500000] + train/ActionL1Loss=0.2648 + throughput/total_tokens=33,216,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=174/500000] + train/ActionL1Loss=0.2654 + throughput/total_tokens=33,408,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=175/500000] + train/ActionL1Loss=0.3006 + throughput/total_tokens=33,600,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=176/500000] + train/ActionL1Loss=0.2798 + throughput/total_tokens=33,792,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=177/500000] + train/ActionL1Loss=0.2637 + throughput/total_tokens=33,984,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=178/500000] + train/ActionL1Loss=0.2645 + throughput/total_tokens=34,176,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=179/500000] + train/ActionL1Loss=0.2754 + throughput/total_tokens=34,368,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=180/500000] + optim/total_grad_norm=27.13 + train/ActionL1Loss=0.3134 + throughput/total_tokens=34,560,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=181/500000] + train/ActionL1Loss=0.2563 + throughput/total_tokens=34,752,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=182/500000] + train/ActionL1Loss=0.2556 + throughput/total_tokens=34,944,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=183/500000] + train/ActionL1Loss=0.2431 + throughput/total_tokens=35,136,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=184/500000] + train/ActionL1Loss=0.2556 + throughput/total_tokens=35,328,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=185/500000] + train/ActionL1Loss=0.2454 + throughput/total_tokens=35,520,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=186/500000] + train/ActionL1Loss=0.2586 + throughput/total_tokens=35,712,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=187/500000] + train/ActionL1Loss=0.2259 + throughput/total_tokens=35,904,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=188/500000] + train/ActionL1Loss=0.2343 + throughput/total_tokens=36,096,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=189/500000] + train/ActionL1Loss=0.2712 + throughput/total_tokens=36,288,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=190/500000] + train/ActionL1Loss=0.2490 + throughput/total_tokens=36,480,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=191/500000] + train/ActionL1Loss=0.2473 + throughput/total_tokens=36,672,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=192/500000] + train/ActionL1Loss=0.2364 + throughput/total_tokens=36,864,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=193/500000] + train/ActionL1Loss=0.2644 + throughput/total_tokens=37,056,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=194/500000] + train/ActionL1Loss=0.2711 + throughput/total_tokens=37,248,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=195/500000] + train/ActionL1Loss=0.2709 + throughput/total_tokens=37,440,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=196/500000] + train/ActionL1Loss=0.2424 + throughput/total_tokens=37,632,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=197/500000] + train/ActionL1Loss=0.2658 + throughput/total_tokens=37,824,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=198/500000] + train/ActionL1Loss=0.2573 + throughput/total_tokens=38,016,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=199/500000] + train/ActionL1Loss=0.2192 + throughput/total_tokens=38,208,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=200/500000] + optim/total_grad_norm=23.02 + train/ActionL1Loss=0.2533 + throughput/total_tokens=38,400,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=201/500000] + train/ActionL1Loss=0.2330 + throughput/total_tokens=38,592,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=202/500000] + train/ActionL1Loss=0.2304 + throughput/total_tokens=38,784,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=203/500000] + train/ActionL1Loss=0.2472 + throughput/total_tokens=38,976,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=204/500000] + train/ActionL1Loss=0.2292 + throughput/total_tokens=39,168,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=205/500000] + train/ActionL1Loss=0.2286 + throughput/total_tokens=39,360,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=206/500000] + train/ActionL1Loss=0.2284 + throughput/total_tokens=39,552,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=207/500000] + train/ActionL1Loss=0.2619 + throughput/total_tokens=39,744,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=208/500000] + train/ActionL1Loss=0.2163 + throughput/total_tokens=39,936,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=209/500000] + train/ActionL1Loss=0.2141 + throughput/total_tokens=40,128,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=210/500000] + train/ActionL1Loss=0.2182 + throughput/total_tokens=40,320,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=211/500000] + train/ActionL1Loss=0.2002 + throughput/total_tokens=40,512,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=212/500000] + train/ActionL1Loss=0.2113 + throughput/total_tokens=40,704,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=213/500000] + train/ActionL1Loss=0.2084 + throughput/total_tokens=40,896,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=214/500000] + train/ActionL1Loss=0.2268 + throughput/total_tokens=41,088,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=215/500000] + train/ActionL1Loss=0.2274 + throughput/total_tokens=41,280,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=216/500000] + train/ActionL1Loss=0.2201 + throughput/total_tokens=41,472,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=217/500000] + train/ActionL1Loss=0.2271 + throughput/total_tokens=41,664,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=218/500000] + train/ActionL1Loss=0.2161 + throughput/total_tokens=41,856,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0481 +[step=219/500000] + train/ActionL1Loss=0.2182 + throughput/total_tokens=42,048,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=220/500000] + optim/total_grad_norm=15.42 + train/ActionL1Loss=0.2233 + throughput/total_tokens=42,240,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=221/500000] + train/ActionL1Loss=0.2247 + throughput/total_tokens=42,432,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=222/500000] + train/ActionL1Loss=0.2127 + throughput/total_tokens=42,624,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=223/500000] + train/ActionL1Loss=0.2059 + throughput/total_tokens=42,816,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=224/500000] + train/ActionL1Loss=0.2405 + throughput/total_tokens=43,008,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=225/500000] + train/ActionL1Loss=0.2014 + throughput/total_tokens=43,200,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=226/500000] + train/ActionL1Loss=0.2097 + throughput/total_tokens=43,392,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=227/500000] + train/ActionL1Loss=0.2146 + throughput/total_tokens=43,584,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=228/500000] + train/ActionL1Loss=0.2103 + throughput/total_tokens=43,776,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=229/500000] + train/ActionL1Loss=0.2259 + throughput/total_tokens=43,968,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=230/500000] + train/ActionL1Loss=0.2152 + throughput/total_tokens=44,160,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=231/500000] + train/ActionL1Loss=0.2642 + throughput/total_tokens=44,352,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=232/500000] + train/ActionL1Loss=0.2295 + throughput/total_tokens=44,544,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=233/500000] + train/ActionL1Loss=0.2271 + throughput/total_tokens=44,736,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=234/500000] + train/ActionL1Loss=0.2425 + throughput/total_tokens=44,928,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=235/500000] + train/ActionL1Loss=0.2361 + throughput/total_tokens=45,120,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=236/500000] + train/ActionL1Loss=0.2297 + throughput/total_tokens=45,312,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=237/500000] + train/ActionL1Loss=0.2281 + throughput/total_tokens=45,504,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=238/500000] + train/ActionL1Loss=0.2035 + throughput/total_tokens=45,696,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=239/500000] + train/ActionL1Loss=0.2001 + throughput/total_tokens=45,888,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=240/500000] + optim/total_grad_norm=16.29 + train/ActionL1Loss=0.1758 + throughput/total_tokens=46,080,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=241/500000] + train/ActionL1Loss=0.2035 + throughput/total_tokens=46,272,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=242/500000] + train/ActionL1Loss=0.2020 + throughput/total_tokens=46,464,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=243/500000] + train/ActionL1Loss=0.2050 + throughput/total_tokens=46,656,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=244/500000] + train/ActionL1Loss=0.2206 + throughput/total_tokens=46,848,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=245/500000] + train/ActionL1Loss=0.2011 + throughput/total_tokens=47,040,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=246/500000] + train/ActionL1Loss=0.2121 + throughput/total_tokens=47,232,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=247/500000] + train/ActionL1Loss=0.1984 + throughput/total_tokens=47,424,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=248/500000] + train/ActionL1Loss=0.2219 + throughput/total_tokens=47,616,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=249/500000] + train/ActionL1Loss=0.2219 + throughput/total_tokens=47,808,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=250/500000] + train/ActionL1Loss=0.1928 + throughput/total_tokens=48,000,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0481 + System/Peak GPU Memory (MB)=46,917 +[step=251/500000] + train/ActionL1Loss=0.1916 + throughput/total_tokens=48,192,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 +[step=252/500000] + train/ActionL1Loss=0.1974 + throughput/total_tokens=48,384,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 +[step=253/500000] + train/ActionL1Loss=0.2183 + throughput/total_tokens=48,576,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 +[step=254/500000] + train/ActionL1Loss=0.1849 + throughput/total_tokens=48,768,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0480 +[step=255/500000] + train/ActionL1Loss=0.1801 + throughput/total_tokens=48,960,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 +[step=256/500000] + train/ActionL1Loss=0.1961 + throughput/total_tokens=49,152,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 +[step=257/500000] + train/ActionL1Loss=0.2249 + throughput/total_tokens=49,344,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 +[step=258/500000] + train/ActionL1Loss=0.2318 + throughput/total_tokens=49,536,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 +[step=259/500000] + train/ActionL1Loss=0.2273 + throughput/total_tokens=49,728,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0480 +[step=260/500000] + optim/total_grad_norm=23.18 + train/ActionL1Loss=0.1848 + throughput/total_tokens=49,920,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 + System/Peak GPU Memory (MB)=46,917 +[step=261/500000] + train/ActionL1Loss=0.2027 + throughput/total_tokens=50,112,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=262/500000] + train/ActionL1Loss=0.2094 + throughput/total_tokens=50,304,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0480 +[step=263/500000] + train/ActionL1Loss=0.1992 + throughput/total_tokens=50,496,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0480 +[step=264/500000] + train/ActionL1Loss=0.1909 + throughput/total_tokens=50,688,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0480 +[step=265/500000] + train/ActionL1Loss=0.2196 + throughput/total_tokens=50,880,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 +[step=266/500000] + train/ActionL1Loss=0.2268 + throughput/total_tokens=51,072,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=267/500000] + train/ActionL1Loss=0.2233 + throughput/total_tokens=51,264,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=268/500000] + train/ActionL1Loss=0.2066 + throughput/total_tokens=51,456,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=269/500000] + train/ActionL1Loss=0.2016 + throughput/total_tokens=51,648,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=270/500000] + train/ActionL1Loss=0.2015 + throughput/total_tokens=51,840,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=271/500000] + train/ActionL1Loss=0.2055 + throughput/total_tokens=52,032,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=272/500000] + train/ActionL1Loss=0.2198 + throughput/total_tokens=52,224,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=273/500000] + train/ActionL1Loss=0.1757 + throughput/total_tokens=52,416,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=274/500000] + train/ActionL1Loss=0.1993 + throughput/total_tokens=52,608,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=275/500000] + train/ActionL1Loss=0.1859 + throughput/total_tokens=52,800,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=276/500000] + train/ActionL1Loss=0.2143 + throughput/total_tokens=52,992,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=277/500000] + train/ActionL1Loss=0.1780 + throughput/total_tokens=53,184,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=278/500000] + train/ActionL1Loss=0.1863 + throughput/total_tokens=53,376,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=279/500000] + train/ActionL1Loss=0.2004 + throughput/total_tokens=53,568,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=280/500000] + optim/total_grad_norm=17.31 + train/ActionL1Loss=0.2156 + throughput/total_tokens=53,760,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=281/500000] + train/ActionL1Loss=0.1881 + throughput/total_tokens=53,952,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=282/500000] + train/ActionL1Loss=0.1926 + throughput/total_tokens=54,144,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=283/500000] + train/ActionL1Loss=0.1987 + throughput/total_tokens=54,336,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=284/500000] + train/ActionL1Loss=0.1991 + throughput/total_tokens=54,528,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=285/500000] + train/ActionL1Loss=0.2128 + throughput/total_tokens=54,720,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=286/500000] + train/ActionL1Loss=0.2022 + throughput/total_tokens=54,912,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=287/500000] + train/ActionL1Loss=0.2106 + throughput/total_tokens=55,104,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=288/500000] + train/ActionL1Loss=0.1919 + throughput/total_tokens=55,296,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=289/500000] + train/ActionL1Loss=0.2025 + throughput/total_tokens=55,488,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=290/500000] + train/ActionL1Loss=0.2146 + throughput/total_tokens=55,680,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=291/500000] + train/ActionL1Loss=0.1919 + throughput/total_tokens=55,872,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=292/500000] + train/ActionL1Loss=0.1888 + throughput/total_tokens=56,064,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=293/500000] + train/ActionL1Loss=0.2064 + throughput/total_tokens=56,256,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=294/500000] + train/ActionL1Loss=0.2027 + throughput/total_tokens=56,448,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=295/500000] + train/ActionL1Loss=0.1783 + throughput/total_tokens=56,640,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=296/500000] + train/ActionL1Loss=0.1850 + throughput/total_tokens=56,832,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=297/500000] + train/ActionL1Loss=0.2083 + throughput/total_tokens=57,024,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=298/500000] + train/ActionL1Loss=0.2021 + throughput/total_tokens=57,216,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=299/500000] + train/ActionL1Loss=0.1845 + throughput/total_tokens=57,408,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=300/500000] + optim/total_grad_norm=16.60 + train/ActionL1Loss=0.1845 + throughput/total_tokens=57,600,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=301/500000] + train/ActionL1Loss=0.2018 + throughput/total_tokens=57,792,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=302/500000] + train/ActionL1Loss=0.1957 + throughput/total_tokens=57,984,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=303/500000] + train/ActionL1Loss=0.1652 + throughput/total_tokens=58,176,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=304/500000] + train/ActionL1Loss=0.2014 + throughput/total_tokens=58,368,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=305/500000] + train/ActionL1Loss=0.1828 + throughput/total_tokens=58,560,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=306/500000] + train/ActionL1Loss=0.1950 + throughput/total_tokens=58,752,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=307/500000] + train/ActionL1Loss=0.2061 + throughput/total_tokens=58,944,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=308/500000] + train/ActionL1Loss=0.1981 + throughput/total_tokens=59,136,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=309/500000] + train/ActionL1Loss=0.1972 + throughput/total_tokens=59,328,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=310/500000] + train/ActionL1Loss=0.1936 + throughput/total_tokens=59,520,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=311/500000] + train/ActionL1Loss=0.1842 + throughput/total_tokens=59,712,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=312/500000] + train/ActionL1Loss=0.1736 + throughput/total_tokens=59,904,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=313/500000] + train/ActionL1Loss=0.2145 + throughput/total_tokens=60,096,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=314/500000] + train/ActionL1Loss=0.1929 + throughput/total_tokens=60,288,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=315/500000] + train/ActionL1Loss=0.1741 + throughput/total_tokens=60,480,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=316/500000] + train/ActionL1Loss=0.1897 + throughput/total_tokens=60,672,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=317/500000] + train/ActionL1Loss=0.1783 + throughput/total_tokens=60,864,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=318/500000] + train/ActionL1Loss=0.2116 + throughput/total_tokens=61,056,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=319/500000] + train/ActionL1Loss=0.1984 + throughput/total_tokens=61,248,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=320/500000] + optim/total_grad_norm=18.84 + train/ActionL1Loss=0.1753 + throughput/total_tokens=61,440,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=321/500000] + train/ActionL1Loss=0.1826 + throughput/total_tokens=61,632,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=322/500000] + train/ActionL1Loss=0.1789 + throughput/total_tokens=61,824,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=323/500000] + train/ActionL1Loss=0.2173 + throughput/total_tokens=62,016,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=324/500000] + train/ActionL1Loss=0.2000 + throughput/total_tokens=62,208,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=325/500000] + train/ActionL1Loss=0.1843 + throughput/total_tokens=62,400,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=326/500000] + train/ActionL1Loss=0.1770 + throughput/total_tokens=62,592,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=327/500000] + train/ActionL1Loss=0.1927 + throughput/total_tokens=62,784,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=328/500000] + train/ActionL1Loss=0.2250 + throughput/total_tokens=62,976,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=329/500000] + train/ActionL1Loss=0.2190 + throughput/total_tokens=63,168,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=330/500000] + train/ActionL1Loss=0.2269 + throughput/total_tokens=63,360,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=331/500000] + train/ActionL1Loss=0.1855 + throughput/total_tokens=63,552,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=332/500000] + train/ActionL1Loss=0.1776 + throughput/total_tokens=63,744,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=333/500000] + train/ActionL1Loss=0.1907 + throughput/total_tokens=63,936,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=334/500000] + train/ActionL1Loss=0.2260 + throughput/total_tokens=64,128,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=335/500000] + train/ActionL1Loss=0.2014 + throughput/total_tokens=64,320,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=336/500000] + train/ActionL1Loss=0.2003 + throughput/total_tokens=64,512,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=337/500000] + train/ActionL1Loss=0.1959 + throughput/total_tokens=64,704,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=338/500000] + train/ActionL1Loss=0.1735 + throughput/total_tokens=64,896,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=339/500000] + train/ActionL1Loss=0.1894 + throughput/total_tokens=65,088,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=340/500000] + optim/total_grad_norm=21.32 + train/ActionL1Loss=0.1724 + throughput/total_tokens=65,280,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=341/500000] + train/ActionL1Loss=0.1899 + throughput/total_tokens=65,472,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=342/500000] + train/ActionL1Loss=0.1680 + throughput/total_tokens=65,664,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=343/500000] + train/ActionL1Loss=0.1867 + throughput/total_tokens=65,856,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=344/500000] + train/ActionL1Loss=0.1897 + throughput/total_tokens=66,048,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=345/500000] + train/ActionL1Loss=0.1672 + throughput/total_tokens=66,240,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=346/500000] + train/ActionL1Loss=0.1895 + throughput/total_tokens=66,432,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=347/500000] + train/ActionL1Loss=0.1931 + throughput/total_tokens=66,624,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=348/500000] + train/ActionL1Loss=0.1768 + throughput/total_tokens=66,816,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=349/500000] + train/ActionL1Loss=0.1995 + throughput/total_tokens=67,008,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=350/500000] + train/ActionL1Loss=0.1850 + throughput/total_tokens=67,200,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=351/500000] + train/ActionL1Loss=0.1660 + throughput/total_tokens=67,392,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=352/500000] + train/ActionL1Loss=0.1564 + throughput/total_tokens=67,584,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=353/500000] + train/ActionL1Loss=0.2194 + throughput/total_tokens=67,776,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=354/500000] + train/ActionL1Loss=0.1869 + throughput/total_tokens=67,968,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=355/500000] + train/ActionL1Loss=0.1860 + throughput/total_tokens=68,160,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=356/500000] + train/ActionL1Loss=0.1781 + throughput/total_tokens=68,352,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=357/500000] + train/ActionL1Loss=0.1937 + throughput/total_tokens=68,544,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=358/500000] + train/ActionL1Loss=0.2198 + throughput/total_tokens=68,736,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=359/500000] + train/ActionL1Loss=0.2007 + throughput/total_tokens=68,928,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=360/500000] + optim/total_grad_norm=14.89 + train/ActionL1Loss=0.1978 + throughput/total_tokens=69,120,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 + System/Peak GPU Memory (MB)=46,917 +[step=361/500000] + train/ActionL1Loss=0.1887 + throughput/total_tokens=69,312,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=362/500000] + train/ActionL1Loss=0.2224 + throughput/total_tokens=69,504,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=363/500000] + train/ActionL1Loss=0.2033 + throughput/total_tokens=69,696,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=364/500000] + train/ActionL1Loss=0.1799 + throughput/total_tokens=69,888,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=365/500000] + train/ActionL1Loss=0.1892 + throughput/total_tokens=70,080,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=366/500000] + train/ActionL1Loss=0.2160 + throughput/total_tokens=70,272,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0480 +[step=367/500000] + train/ActionL1Loss=0.1924 + throughput/total_tokens=70,464,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=368/500000] + train/ActionL1Loss=0.1888 + throughput/total_tokens=70,656,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=369/500000] + train/ActionL1Loss=0.1720 + throughput/total_tokens=70,848,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=370/500000] + train/ActionL1Loss=0.1896 + throughput/total_tokens=71,040,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=371/500000] + train/ActionL1Loss=0.1847 + throughput/total_tokens=71,232,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=372/500000] + train/ActionL1Loss=0.2005 + throughput/total_tokens=71,424,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=373/500000] + train/ActionL1Loss=0.1846 + throughput/total_tokens=71,616,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=374/500000] + train/ActionL1Loss=0.1679 + throughput/total_tokens=71,808,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=375/500000] + train/ActionL1Loss=0.2013 + throughput/total_tokens=72,000,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=376/500000] + train/ActionL1Loss=0.2209 + throughput/total_tokens=72,192,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=377/500000] + train/ActionL1Loss=0.1855 + throughput/total_tokens=72,384,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=378/500000] + train/ActionL1Loss=0.1956 + throughput/total_tokens=72,576,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=379/500000] + train/ActionL1Loss=0.1849 + throughput/total_tokens=72,768,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=380/500000] + optim/total_grad_norm=15.45 + train/ActionL1Loss=0.1742 + throughput/total_tokens=72,960,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=381/500000] + train/ActionL1Loss=0.1951 + throughput/total_tokens=73,152,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=382/500000] + train/ActionL1Loss=0.1919 + throughput/total_tokens=73,344,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=383/500000] + train/ActionL1Loss=0.1954 + throughput/total_tokens=73,536,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=384/500000] + train/ActionL1Loss=0.1812 + throughput/total_tokens=73,728,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=385/500000] + train/ActionL1Loss=0.2024 + throughput/total_tokens=73,920,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=386/500000] + train/ActionL1Loss=0.1881 + throughput/total_tokens=74,112,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=387/500000] + train/ActionL1Loss=0.1644 + throughput/total_tokens=74,304,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=388/500000] + train/ActionL1Loss=0.1779 + throughput/total_tokens=74,496,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=389/500000] + train/ActionL1Loss=0.1972 + throughput/total_tokens=74,688,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=390/500000] + train/ActionL1Loss=0.1736 + throughput/total_tokens=74,880,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=391/500000] + train/ActionL1Loss=0.1827 + throughput/total_tokens=75,072,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=392/500000] + train/ActionL1Loss=0.1778 + throughput/total_tokens=75,264,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=393/500000] + train/ActionL1Loss=0.1618 + throughput/total_tokens=75,456,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=394/500000] + train/ActionL1Loss=0.1876 + throughput/total_tokens=75,648,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=395/500000] + train/ActionL1Loss=0.1848 + throughput/total_tokens=75,840,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=396/500000] + train/ActionL1Loss=0.2213 + throughput/total_tokens=76,032,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=397/500000] + train/ActionL1Loss=0.2074 + throughput/total_tokens=76,224,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=398/500000] + train/ActionL1Loss=0.2003 + throughput/total_tokens=76,416,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=399/500000] + train/ActionL1Loss=0.1818 + throughput/total_tokens=76,608,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=400/500000] + optim/total_grad_norm=17.86 + train/ActionL1Loss=0.2047 + throughput/total_tokens=76,800,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=401/500000] + train/ActionL1Loss=0.1926 + throughput/total_tokens=76,992,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=402/500000] + train/ActionL1Loss=0.1891 + throughput/total_tokens=77,184,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=403/500000] + train/ActionL1Loss=0.1927 + throughput/total_tokens=77,376,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=404/500000] + train/ActionL1Loss=0.1880 + throughput/total_tokens=77,568,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=405/500000] + train/ActionL1Loss=0.1822 + throughput/total_tokens=77,760,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=406/500000] + train/ActionL1Loss=0.1853 + throughput/total_tokens=77,952,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=407/500000] + train/ActionL1Loss=0.1951 + throughput/total_tokens=78,144,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=408/500000] + train/ActionL1Loss=0.1735 + throughput/total_tokens=78,336,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=409/500000] + train/ActionL1Loss=0.1942 + throughput/total_tokens=78,528,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=410/500000] + train/ActionL1Loss=0.1782 + throughput/total_tokens=78,720,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=411/500000] + train/ActionL1Loss=0.1767 + throughput/total_tokens=78,912,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=412/500000] + train/ActionL1Loss=0.1702 + throughput/total_tokens=79,104,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=413/500000] + train/ActionL1Loss=0.1537 + throughput/total_tokens=79,296,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=414/500000] + train/ActionL1Loss=0.1630 + throughput/total_tokens=79,488,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=415/500000] + train/ActionL1Loss=0.1654 + throughput/total_tokens=79,680,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=416/500000] + train/ActionL1Loss=0.2000 + throughput/total_tokens=79,872,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=417/500000] + train/ActionL1Loss=0.2001 + throughput/total_tokens=80,064,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=418/500000] + train/ActionL1Loss=0.1751 + throughput/total_tokens=80,256,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=419/500000] + train/ActionL1Loss=0.1956 + throughput/total_tokens=80,448,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=420/500000] + optim/total_grad_norm=14.71 + train/ActionL1Loss=0.1834 + throughput/total_tokens=80,640,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=421/500000] + train/ActionL1Loss=0.1955 + throughput/total_tokens=80,832,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=422/500000] + train/ActionL1Loss=0.1663 + throughput/total_tokens=81,024,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=423/500000] + train/ActionL1Loss=0.1624 + throughput/total_tokens=81,216,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=424/500000] + train/ActionL1Loss=0.1808 + throughput/total_tokens=81,408,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=425/500000] + train/ActionL1Loss=0.1633 + throughput/total_tokens=81,600,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=426/500000] + train/ActionL1Loss=0.1948 + throughput/total_tokens=81,792,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=427/500000] + train/ActionL1Loss=0.1933 + throughput/total_tokens=81,984,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=428/500000] + train/ActionL1Loss=0.2128 + throughput/total_tokens=82,176,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=429/500000] + train/ActionL1Loss=0.1875 + throughput/total_tokens=82,368,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=430/500000] + train/ActionL1Loss=0.1616 + throughput/total_tokens=82,560,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=431/500000] + train/ActionL1Loss=0.1755 + throughput/total_tokens=82,752,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=432/500000] + train/ActionL1Loss=0.1929 + throughput/total_tokens=82,944,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=433/500000] + train/ActionL1Loss=0.1802 + throughput/total_tokens=83,136,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=434/500000] + train/ActionL1Loss=0.1632 + throughput/total_tokens=83,328,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=435/500000] + train/ActionL1Loss=0.1780 + throughput/total_tokens=83,520,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=436/500000] + train/ActionL1Loss=0.1602 + throughput/total_tokens=83,712,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=437/500000] + train/ActionL1Loss=0.2114 + throughput/total_tokens=83,904,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=438/500000] + train/ActionL1Loss=0.1750 + throughput/total_tokens=84,096,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=439/500000] + train/ActionL1Loss=0.1717 + throughput/total_tokens=84,288,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=440/500000] + optim/total_grad_norm=15.10 + train/ActionL1Loss=0.1774 + throughput/total_tokens=84,480,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=441/500000] + train/ActionL1Loss=0.1699 + throughput/total_tokens=84,672,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=442/500000] + train/ActionL1Loss=0.1903 + throughput/total_tokens=84,864,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=443/500000] + train/ActionL1Loss=0.1792 + throughput/total_tokens=85,056,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=444/500000] + train/ActionL1Loss=0.1757 + throughput/total_tokens=85,248,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=445/500000] + train/ActionL1Loss=0.1856 + throughput/total_tokens=85,440,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=446/500000] + train/ActionL1Loss=0.1619 + throughput/total_tokens=85,632,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=447/500000] + train/ActionL1Loss=0.1671 + throughput/total_tokens=85,824,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=448/500000] + train/ActionL1Loss=0.1778 + throughput/total_tokens=86,016,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=449/500000] + train/ActionL1Loss=0.1607 + throughput/total_tokens=86,208,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=450/500000] + train/ActionL1Loss=0.1515 + throughput/total_tokens=86,400,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=451/500000] + train/ActionL1Loss=0.1881 + throughput/total_tokens=86,592,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=452/500000] + train/ActionL1Loss=0.1799 + throughput/total_tokens=86,784,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=453/500000] + train/ActionL1Loss=0.1364 + throughput/total_tokens=86,976,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=454/500000] + train/ActionL1Loss=0.1777 + throughput/total_tokens=87,168,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=455/500000] + train/ActionL1Loss=0.1706 + throughput/total_tokens=87,360,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=456/500000] + train/ActionL1Loss=0.1875 + throughput/total_tokens=87,552,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=457/500000] + train/ActionL1Loss=0.1717 + throughput/total_tokens=87,744,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=458/500000] + train/ActionL1Loss=0.1822 + throughput/total_tokens=87,936,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=459/500000] + train/ActionL1Loss=0.1582 + throughput/total_tokens=88,128,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=460/500000] + optim/total_grad_norm=13.39 + train/ActionL1Loss=0.1967 + throughput/total_tokens=88,320,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=461/500000] + train/ActionL1Loss=0.1721 + throughput/total_tokens=88,512,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=462/500000] + train/ActionL1Loss=0.1768 + throughput/total_tokens=88,704,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=463/500000] + train/ActionL1Loss=0.1719 + throughput/total_tokens=88,896,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=464/500000] + train/ActionL1Loss=0.1579 + throughput/total_tokens=89,088,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=465/500000] + train/ActionL1Loss=0.1571 + throughput/total_tokens=89,280,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=466/500000] + train/ActionL1Loss=0.1580 + throughput/total_tokens=89,472,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=467/500000] + train/ActionL1Loss=0.1895 + throughput/total_tokens=89,664,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=468/500000] + train/ActionL1Loss=0.1769 + throughput/total_tokens=89,856,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=469/500000] + train/ActionL1Loss=0.1762 + throughput/total_tokens=90,048,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=470/500000] + train/ActionL1Loss=0.1711 + throughput/total_tokens=90,240,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=471/500000] + train/ActionL1Loss=0.2014 + throughput/total_tokens=90,432,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=472/500000] + train/ActionL1Loss=0.1827 + throughput/total_tokens=90,624,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=473/500000] + train/ActionL1Loss=0.1709 + throughput/total_tokens=90,816,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=474/500000] + train/ActionL1Loss=0.1947 + throughput/total_tokens=91,008,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=475/500000] + train/ActionL1Loss=0.1680 + throughput/total_tokens=91,200,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=476/500000] + train/ActionL1Loss=0.1825 + throughput/total_tokens=91,392,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=477/500000] + train/ActionL1Loss=0.1651 + throughput/total_tokens=91,584,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=478/500000] + train/ActionL1Loss=0.1814 + throughput/total_tokens=91,776,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=479/500000] + train/ActionL1Loss=0.1909 + throughput/total_tokens=91,968,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=480/500000] + optim/total_grad_norm=13.61 + train/ActionL1Loss=0.1864 + throughput/total_tokens=92,160,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=481/500000] + train/ActionL1Loss=0.1661 + throughput/total_tokens=92,352,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=482/500000] + train/ActionL1Loss=0.2058 + throughput/total_tokens=92,544,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=483/500000] + train/ActionL1Loss=0.1784 + throughput/total_tokens=92,736,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=484/500000] + train/ActionL1Loss=0.1447 + throughput/total_tokens=92,928,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=485/500000] + train/ActionL1Loss=0.1619 + throughput/total_tokens=93,120,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=486/500000] + train/ActionL1Loss=0.2042 + throughput/total_tokens=93,312,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=487/500000] + train/ActionL1Loss=0.1800 + throughput/total_tokens=93,504,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=488/500000] + train/ActionL1Loss=0.1888 + throughput/total_tokens=93,696,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=489/500000] + train/ActionL1Loss=0.1719 + throughput/total_tokens=93,888,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=490/500000] + train/ActionL1Loss=0.1775 + throughput/total_tokens=94,080,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=491/500000] + train/ActionL1Loss=0.1873 + throughput/total_tokens=94,272,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=492/500000] + train/ActionL1Loss=0.1884 + throughput/total_tokens=94,464,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=493/500000] + train/ActionL1Loss=0.1794 + throughput/total_tokens=94,656,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=494/500000] + train/ActionL1Loss=0.1571 + throughput/total_tokens=94,848,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=495/500000] + train/ActionL1Loss=0.2027 + throughput/total_tokens=95,040,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=496/500000] + train/ActionL1Loss=0.1813 + throughput/total_tokens=95,232,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=497/500000] + train/ActionL1Loss=0.1860 + throughput/total_tokens=95,424,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=498/500000] + train/ActionL1Loss=0.1742 + throughput/total_tokens=95,616,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=499/500000] + train/ActionL1Loss=0.1614 + throughput/total_tokens=95,808,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=500/500000] + optim/total_grad_norm=13.84 + train/ActionL1Loss=0.1861 + throughput/total_tokens=96,000,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +09/30 [16:07:44] WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:678: warnings.py:109 + FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and + set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: + https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: + https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html . + warnings.warn( + + WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:860: UserWarning: warnings.py:109 + `_get_pg_default_device` will be deprecated, it only stays for backward-compatiblity reason. If you need to find a device for object + collectives, please use `_get_object_coll_device`. If you need to query the device types supported by group, please use + `_device_capability(group)`. + warnings.warn( + +09/30 [16:07:47] WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:722: FutureWarning: warnings.py:109 + Please use DTensor instead and we are deprecating ShardedTensor. + local_shape = tensor.shape + + WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:739: FutureWarning: warnings.py:109 + Please use DTensor instead and we are deprecating ShardedTensor. + tensor.shape, + + WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/fsdp/_state_dict_utils.py:741: FutureWarning: warnings.py:109 + Please use DTensor instead and we are deprecating ShardedTensor. + tensor.dtype, + +09/30 [16:07:51] WARNING | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/overrides.py:1747: FutureWarning: Please use DTensor warnings.py:109 + instead and we are deprecating ShardedTensor. + result = torch_func_method(public_api, types, args, kwargs) + +09/30 [16:08:23] INFO | >> Saving config... checkpoint.py:608 +09/30 [16:08:54] INFO | >> Saving model state... checkpoint.py:796 +09/30 [16:10:00] INFO | >> Saving optim state... checkpoint.py:811 +09/30 [16:11:31] INFO | >> Saving trainer state... checkpoint.py:670 + INFO | >> Saving config... checkpoint.py:608 +[step=501/500000] + train/ActionL1Loss=0.1892 + throughput/total_tokens=96,192,000 + throughput/device/tokens_per_second=1,207 + throughput/device/batches_per_second=0.0503 +[step=502/500000] + train/ActionL1Loss=0.1546 + throughput/total_tokens=96,384,000 + throughput/device/tokens_per_second=1,181 + throughput/device/batches_per_second=0.0492 +[step=503/500000] + train/ActionL1Loss=0.1637 + throughput/total_tokens=96,576,000 + throughput/device/tokens_per_second=1,171 + throughput/device/batches_per_second=0.0488 +[step=504/500000] + train/ActionL1Loss=0.1788 + throughput/total_tokens=96,768,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=505/500000] + train/ActionL1Loss=0.1868 + throughput/total_tokens=96,960,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=506/500000] + train/ActionL1Loss=0.1487 + throughput/total_tokens=97,152,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=507/500000] + train/ActionL1Loss=0.1739 + throughput/total_tokens=97,344,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=508/500000] + train/ActionL1Loss=0.1846 + throughput/total_tokens=97,536,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=509/500000] + train/ActionL1Loss=0.1693 + throughput/total_tokens=97,728,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=510/500000] + train/ActionL1Loss=0.1609 + throughput/total_tokens=97,920,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=511/500000] + train/ActionL1Loss=0.1648 + throughput/total_tokens=98,112,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=512/500000] + train/ActionL1Loss=0.1716 + throughput/total_tokens=98,304,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=513/500000] + train/ActionL1Loss=0.1814 + throughput/total_tokens=98,496,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=514/500000] + train/ActionL1Loss=0.1770 + throughput/total_tokens=98,688,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=515/500000] + train/ActionL1Loss=0.1785 + throughput/total_tokens=98,880,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=516/500000] + train/ActionL1Loss=0.1820 + throughput/total_tokens=99,072,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=517/500000] + train/ActionL1Loss=0.1843 + throughput/total_tokens=99,264,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=518/500000] + train/ActionL1Loss=0.1524 + throughput/total_tokens=99,456,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=519/500000] + train/ActionL1Loss=0.1808 + throughput/total_tokens=99,648,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=520/500000] + optim/total_grad_norm=14.66 + train/ActionL1Loss=0.1712 + throughput/total_tokens=99,840,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=521/500000] + train/ActionL1Loss=0.1766 + throughput/total_tokens=100,032,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=522/500000] + train/ActionL1Loss=0.1539 + throughput/total_tokens=100,224,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=523/500000] + train/ActionL1Loss=0.1631 + throughput/total_tokens=100,416,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=524/500000] + train/ActionL1Loss=0.1721 + throughput/total_tokens=100,608,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=525/500000] + train/ActionL1Loss=0.1662 + throughput/total_tokens=100,800,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=526/500000] + train/ActionL1Loss=0.1434 + throughput/total_tokens=100,992,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=527/500000] + train/ActionL1Loss=0.1556 + throughput/total_tokens=101,184,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=528/500000] + train/ActionL1Loss=0.1482 + throughput/total_tokens=101,376,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=529/500000] + train/ActionL1Loss=0.1653 + throughput/total_tokens=101,568,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=530/500000] + train/ActionL1Loss=0.2038 + throughput/total_tokens=101,760,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=531/500000] + train/ActionL1Loss=0.1562 + throughput/total_tokens=101,952,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=532/500000] + train/ActionL1Loss=0.1583 + throughput/total_tokens=102,144,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=533/500000] + train/ActionL1Loss=0.1652 + throughput/total_tokens=102,336,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=534/500000] + train/ActionL1Loss=0.1821 + throughput/total_tokens=102,528,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=535/500000] + train/ActionL1Loss=0.1932 + throughput/total_tokens=102,720,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=536/500000] + train/ActionL1Loss=0.1625 + throughput/total_tokens=102,912,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=537/500000] + train/ActionL1Loss=0.1979 + throughput/total_tokens=103,104,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=538/500000] + train/ActionL1Loss=0.1715 + throughput/total_tokens=103,296,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=539/500000] + train/ActionL1Loss=0.1873 + throughput/total_tokens=103,488,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=540/500000] + optim/total_grad_norm=10.94 + train/ActionL1Loss=0.1753 + throughput/total_tokens=103,680,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=541/500000] + train/ActionL1Loss=0.1665 + throughput/total_tokens=103,872,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=542/500000] + train/ActionL1Loss=0.1859 + throughput/total_tokens=104,064,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=543/500000] + train/ActionL1Loss=0.1495 + throughput/total_tokens=104,256,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=544/500000] + train/ActionL1Loss=0.1803 + throughput/total_tokens=104,448,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=545/500000] + train/ActionL1Loss=0.1802 + throughput/total_tokens=104,640,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=546/500000] + train/ActionL1Loss=0.1907 + throughput/total_tokens=104,832,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=547/500000] + train/ActionL1Loss=0.1633 + throughput/total_tokens=105,024,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=548/500000] + train/ActionL1Loss=0.1828 + throughput/total_tokens=105,216,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=549/500000] + train/ActionL1Loss=0.1400 + throughput/total_tokens=105,408,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=550/500000] + train/ActionL1Loss=0.1469 + throughput/total_tokens=105,600,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=551/500000] + train/ActionL1Loss=0.1730 + throughput/total_tokens=105,792,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=552/500000] + train/ActionL1Loss=0.1472 + throughput/total_tokens=105,984,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=553/500000] + train/ActionL1Loss=0.1625 + throughput/total_tokens=106,176,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=554/500000] + train/ActionL1Loss=0.1470 + throughput/total_tokens=106,368,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=555/500000] + train/ActionL1Loss=0.1795 + throughput/total_tokens=106,560,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=556/500000] + train/ActionL1Loss=0.1644 + throughput/total_tokens=106,752,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=557/500000] + train/ActionL1Loss=0.1542 + throughput/total_tokens=106,944,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=558/500000] + train/ActionL1Loss=0.1752 + throughput/total_tokens=107,136,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=559/500000] + train/ActionL1Loss=0.1490 + throughput/total_tokens=107,328,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=560/500000] + optim/total_grad_norm=12.44 + train/ActionL1Loss=0.1595 + throughput/total_tokens=107,520,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=561/500000] + train/ActionL1Loss=0.1708 + throughput/total_tokens=107,712,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=562/500000] + train/ActionL1Loss=0.1758 + throughput/total_tokens=107,904,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=563/500000] + train/ActionL1Loss=0.1604 + throughput/total_tokens=108,096,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=564/500000] + train/ActionL1Loss=0.1665 + throughput/total_tokens=108,288,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=565/500000] + train/ActionL1Loss=0.1680 + throughput/total_tokens=108,480,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=566/500000] + train/ActionL1Loss=0.1650 + throughput/total_tokens=108,672,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=567/500000] + train/ActionL1Loss=0.1549 + throughput/total_tokens=108,864,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=568/500000] + train/ActionL1Loss=0.1581 + throughput/total_tokens=109,056,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=569/500000] + train/ActionL1Loss=0.1455 + throughput/total_tokens=109,248,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=570/500000] + train/ActionL1Loss=0.1426 + throughput/total_tokens=109,440,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=571/500000] + train/ActionL1Loss=0.1497 + throughput/total_tokens=109,632,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=572/500000] + train/ActionL1Loss=0.1557 + throughput/total_tokens=109,824,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=573/500000] + train/ActionL1Loss=0.1850 + throughput/total_tokens=110,016,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=574/500000] + train/ActionL1Loss=0.1582 + throughput/total_tokens=110,208,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=575/500000] + train/ActionL1Loss=0.1534 + throughput/total_tokens=110,400,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=576/500000] + train/ActionL1Loss=0.1990 + throughput/total_tokens=110,592,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=577/500000] + train/ActionL1Loss=0.1684 + throughput/total_tokens=110,784,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=578/500000] + train/ActionL1Loss=0.1667 + throughput/total_tokens=110,976,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=579/500000] + train/ActionL1Loss=0.1699 + throughput/total_tokens=111,168,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=580/500000] + optim/total_grad_norm=12.20 + train/ActionL1Loss=0.1562 + throughput/total_tokens=111,360,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=581/500000] + train/ActionL1Loss=0.1721 + throughput/total_tokens=111,552,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=582/500000] + train/ActionL1Loss=0.1681 + throughput/total_tokens=111,744,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=583/500000] + train/ActionL1Loss=0.1900 + throughput/total_tokens=111,936,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=584/500000] + train/ActionL1Loss=0.1779 + throughput/total_tokens=112,128,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=585/500000] + train/ActionL1Loss=0.1568 + throughput/total_tokens=112,320,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=586/500000] + train/ActionL1Loss=0.1544 + throughput/total_tokens=112,512,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=587/500000] + train/ActionL1Loss=0.1512 + throughput/total_tokens=112,704,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=588/500000] + train/ActionL1Loss=0.1533 + throughput/total_tokens=112,896,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=589/500000] + train/ActionL1Loss=0.1479 + throughput/total_tokens=113,088,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=590/500000] + train/ActionL1Loss=0.1514 + throughput/total_tokens=113,280,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=591/500000] + train/ActionL1Loss=0.1670 + throughput/total_tokens=113,472,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=592/500000] + train/ActionL1Loss=0.1521 + throughput/total_tokens=113,664,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=593/500000] + train/ActionL1Loss=0.1696 + throughput/total_tokens=113,856,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=594/500000] + train/ActionL1Loss=0.1888 + throughput/total_tokens=114,048,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=595/500000] + train/ActionL1Loss=0.1999 + throughput/total_tokens=114,240,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=596/500000] + train/ActionL1Loss=0.1522 + throughput/total_tokens=114,432,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=597/500000] + train/ActionL1Loss=0.1696 + throughput/total_tokens=114,624,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=598/500000] + train/ActionL1Loss=0.1681 + throughput/total_tokens=114,816,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=599/500000] + train/ActionL1Loss=0.1478 + throughput/total_tokens=115,008,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=600/500000] + optim/total_grad_norm=11.63 + train/ActionL1Loss=0.1499 + throughput/total_tokens=115,200,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=601/500000] + train/ActionL1Loss=0.1504 + throughput/total_tokens=115,392,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=602/500000] + train/ActionL1Loss=0.1367 + throughput/total_tokens=115,584,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=603/500000] + train/ActionL1Loss=0.1357 + throughput/total_tokens=115,776,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=604/500000] + train/ActionL1Loss=0.1414 + throughput/total_tokens=115,968,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=605/500000] + train/ActionL1Loss=0.1497 + throughput/total_tokens=116,160,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=606/500000] + train/ActionL1Loss=0.1550 + throughput/total_tokens=116,352,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=607/500000] + train/ActionL1Loss=0.1435 + throughput/total_tokens=116,544,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=608/500000] + train/ActionL1Loss=0.1646 + throughput/total_tokens=116,736,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=609/500000] + train/ActionL1Loss=0.1828 + throughput/total_tokens=116,928,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=610/500000] + train/ActionL1Loss=0.1604 + throughput/total_tokens=117,120,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=611/500000] + train/ActionL1Loss=0.1711 + throughput/total_tokens=117,312,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=612/500000] + train/ActionL1Loss=0.1788 + throughput/total_tokens=117,504,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=613/500000] + train/ActionL1Loss=0.1762 + throughput/total_tokens=117,696,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=614/500000] + train/ActionL1Loss=0.1835 + throughput/total_tokens=117,888,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=615/500000] + train/ActionL1Loss=0.1961 + throughput/total_tokens=118,080,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=616/500000] + train/ActionL1Loss=0.1737 + throughput/total_tokens=118,272,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=617/500000] + train/ActionL1Loss=0.1824 + throughput/total_tokens=118,464,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=618/500000] + train/ActionL1Loss=0.1693 + throughput/total_tokens=118,656,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=619/500000] + train/ActionL1Loss=0.1639 + throughput/total_tokens=118,848,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=620/500000] + optim/total_grad_norm=10.83 + train/ActionL1Loss=0.1511 + throughput/total_tokens=119,040,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=621/500000] + train/ActionL1Loss=0.1893 + throughput/total_tokens=119,232,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=622/500000] + train/ActionL1Loss=0.1579 + throughput/total_tokens=119,424,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=623/500000] + train/ActionL1Loss=0.1715 + throughput/total_tokens=119,616,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=624/500000] + train/ActionL1Loss=0.1778 + throughput/total_tokens=119,808,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=625/500000] + train/ActionL1Loss=0.1698 + throughput/total_tokens=120,000,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=626/500000] + train/ActionL1Loss=0.1776 + throughput/total_tokens=120,192,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=627/500000] + train/ActionL1Loss=0.1676 + throughput/total_tokens=120,384,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=628/500000] + train/ActionL1Loss=0.1607 + throughput/total_tokens=120,576,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=629/500000] + train/ActionL1Loss=0.1709 + throughput/total_tokens=120,768,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=630/500000] + train/ActionL1Loss=0.1916 + throughput/total_tokens=120,960,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=631/500000] + train/ActionL1Loss=0.1805 + throughput/total_tokens=121,152,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=632/500000] + train/ActionL1Loss=0.1624 + throughput/total_tokens=121,344,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=633/500000] + train/ActionL1Loss=0.1753 + throughput/total_tokens=121,536,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=634/500000] + train/ActionL1Loss=0.1582 + throughput/total_tokens=121,728,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=635/500000] + train/ActionL1Loss=0.1804 + throughput/total_tokens=121,920,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=636/500000] + train/ActionL1Loss=0.1661 + throughput/total_tokens=122,112,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=637/500000] + train/ActionL1Loss=0.1705 + throughput/total_tokens=122,304,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=638/500000] + train/ActionL1Loss=0.1667 + throughput/total_tokens=122,496,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=639/500000] + train/ActionL1Loss=0.1588 + throughput/total_tokens=122,688,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=640/500000] + optim/total_grad_norm=7.792 + train/ActionL1Loss=0.1705 + throughput/total_tokens=122,880,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=641/500000] + train/ActionL1Loss=0.1604 + throughput/total_tokens=123,072,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=642/500000] + train/ActionL1Loss=0.1535 + throughput/total_tokens=123,264,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=643/500000] + train/ActionL1Loss=0.1611 + throughput/total_tokens=123,456,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=644/500000] + train/ActionL1Loss=0.1560 + throughput/total_tokens=123,648,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=645/500000] + train/ActionL1Loss=0.1524 + throughput/total_tokens=123,840,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=646/500000] + train/ActionL1Loss=0.1444 + throughput/total_tokens=124,032,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=647/500000] + train/ActionL1Loss=0.1762 + throughput/total_tokens=124,224,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=648/500000] + train/ActionL1Loss=0.1676 + throughput/total_tokens=124,416,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=649/500000] + train/ActionL1Loss=0.1642 + throughput/total_tokens=124,608,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=650/500000] + train/ActionL1Loss=0.1733 + throughput/total_tokens=124,800,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=651/500000] + train/ActionL1Loss=0.1750 + throughput/total_tokens=124,992,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=652/500000] + train/ActionL1Loss=0.1695 + throughput/total_tokens=125,184,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=653/500000] + train/ActionL1Loss=0.1622 + throughput/total_tokens=125,376,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=654/500000] + train/ActionL1Loss=0.1497 + throughput/total_tokens=125,568,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=655/500000] + train/ActionL1Loss=0.1591 + throughput/total_tokens=125,760,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=656/500000] + train/ActionL1Loss=0.1495 + throughput/total_tokens=125,952,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=657/500000] + train/ActionL1Loss=0.1560 + throughput/total_tokens=126,144,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=658/500000] + train/ActionL1Loss=0.1567 + throughput/total_tokens=126,336,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=659/500000] + train/ActionL1Loss=0.1450 + throughput/total_tokens=126,528,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=660/500000] + optim/total_grad_norm=10.12 + train/ActionL1Loss=0.1396 + throughput/total_tokens=126,720,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=661/500000] + train/ActionL1Loss=0.1337 + throughput/total_tokens=126,912,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=662/500000] + train/ActionL1Loss=0.1720 + throughput/total_tokens=127,104,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=663/500000] + train/ActionL1Loss=0.1641 + throughput/total_tokens=127,296,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=664/500000] + train/ActionL1Loss=0.1631 + throughput/total_tokens=127,488,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=665/500000] + train/ActionL1Loss=0.1716 + throughput/total_tokens=127,680,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=666/500000] + train/ActionL1Loss=0.1773 + throughput/total_tokens=127,872,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=667/500000] + train/ActionL1Loss=0.1498 + throughput/total_tokens=128,064,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=668/500000] + train/ActionL1Loss=0.1626 + throughput/total_tokens=128,256,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=669/500000] + train/ActionL1Loss=0.1574 + throughput/total_tokens=128,448,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=670/500000] + train/ActionL1Loss=0.1628 + throughput/total_tokens=128,640,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=671/500000] + train/ActionL1Loss=0.1662 + throughput/total_tokens=128,832,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=672/500000] + train/ActionL1Loss=0.1703 + throughput/total_tokens=129,024,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=673/500000] + train/ActionL1Loss=0.1387 + throughput/total_tokens=129,216,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=674/500000] + train/ActionL1Loss=0.1716 + throughput/total_tokens=129,408,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=675/500000] + train/ActionL1Loss=0.1512 + throughput/total_tokens=129,600,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=676/500000] + train/ActionL1Loss=0.1651 + throughput/total_tokens=129,792,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=677/500000] + train/ActionL1Loss=0.1614 + throughput/total_tokens=129,984,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=678/500000] + train/ActionL1Loss=0.1721 + throughput/total_tokens=130,176,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=679/500000] + train/ActionL1Loss=0.1599 + throughput/total_tokens=130,368,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=680/500000] + optim/total_grad_norm=9.022 + train/ActionL1Loss=0.1375 + throughput/total_tokens=130,560,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=681/500000] + train/ActionL1Loss=0.1611 + throughput/total_tokens=130,752,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=682/500000] + train/ActionL1Loss=0.1788 + throughput/total_tokens=130,944,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=683/500000] + train/ActionL1Loss=0.1680 + throughput/total_tokens=131,136,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=684/500000] + train/ActionL1Loss=0.1723 + throughput/total_tokens=131,328,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=685/500000] + train/ActionL1Loss=0.1567 + throughput/total_tokens=131,520,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=686/500000] + train/ActionL1Loss=0.1299 + throughput/total_tokens=131,712,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=687/500000] + train/ActionL1Loss=0.1533 + throughput/total_tokens=131,904,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=688/500000] + train/ActionL1Loss=0.1442 + throughput/total_tokens=132,096,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=689/500000] + train/ActionL1Loss=0.1522 + throughput/total_tokens=132,288,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=690/500000] + train/ActionL1Loss=0.1421 + throughput/total_tokens=132,480,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=691/500000] + train/ActionL1Loss=0.1593 + throughput/total_tokens=132,672,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=692/500000] + train/ActionL1Loss=0.1645 + throughput/total_tokens=132,864,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=693/500000] + train/ActionL1Loss=0.1529 + throughput/total_tokens=133,056,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=694/500000] + train/ActionL1Loss=0.1509 + throughput/total_tokens=133,248,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=695/500000] + train/ActionL1Loss=0.1389 + throughput/total_tokens=133,440,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=696/500000] + train/ActionL1Loss=0.1402 + throughput/total_tokens=133,632,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=697/500000] + train/ActionL1Loss=0.1487 + throughput/total_tokens=133,824,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=698/500000] + train/ActionL1Loss=0.1517 + throughput/total_tokens=134,016,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=699/500000] + train/ActionL1Loss=0.1461 + throughput/total_tokens=134,208,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=700/500000] + optim/total_grad_norm=9.758 + train/ActionL1Loss=0.1691 + throughput/total_tokens=134,400,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=701/500000] + train/ActionL1Loss=0.1614 + throughput/total_tokens=134,592,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=702/500000] + train/ActionL1Loss=0.1607 + throughput/total_tokens=134,784,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=703/500000] + train/ActionL1Loss=0.1637 + throughput/total_tokens=134,976,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=704/500000] + train/ActionL1Loss=0.1802 + throughput/total_tokens=135,168,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=705/500000] + train/ActionL1Loss=0.1469 + throughput/total_tokens=135,360,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=706/500000] + train/ActionL1Loss=0.1766 + throughput/total_tokens=135,552,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=707/500000] + train/ActionL1Loss=0.1778 + throughput/total_tokens=135,744,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=708/500000] + train/ActionL1Loss=0.1418 + throughput/total_tokens=135,936,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=709/500000] + train/ActionL1Loss=0.1731 + throughput/total_tokens=136,128,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=710/500000] + train/ActionL1Loss=0.1640 + throughput/total_tokens=136,320,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=711/500000] + train/ActionL1Loss=0.1540 + throughput/total_tokens=136,512,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=712/500000] + train/ActionL1Loss=0.1474 + throughput/total_tokens=136,704,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=713/500000] + train/ActionL1Loss=0.1622 + throughput/total_tokens=136,896,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=714/500000] + train/ActionL1Loss=0.1737 + throughput/total_tokens=137,088,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=715/500000] + train/ActionL1Loss=0.1523 + throughput/total_tokens=137,280,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=716/500000] + train/ActionL1Loss=0.1387 + throughput/total_tokens=137,472,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=717/500000] + train/ActionL1Loss=0.1657 + throughput/total_tokens=137,664,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=718/500000] + train/ActionL1Loss=0.1606 + throughput/total_tokens=137,856,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=719/500000] + train/ActionL1Loss=0.1646 + throughput/total_tokens=138,048,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=720/500000] + optim/total_grad_norm=10.56 + train/ActionL1Loss=0.1626 + throughput/total_tokens=138,240,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=721/500000] + train/ActionL1Loss=0.1555 + throughput/total_tokens=138,432,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=722/500000] + train/ActionL1Loss=0.1369 + throughput/total_tokens=138,624,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=723/500000] + train/ActionL1Loss=0.1379 + throughput/total_tokens=138,816,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=724/500000] + train/ActionL1Loss=0.1367 + throughput/total_tokens=139,008,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=725/500000] + train/ActionL1Loss=0.1612 + throughput/total_tokens=139,200,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=726/500000] + train/ActionL1Loss=0.1397 + throughput/total_tokens=139,392,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=727/500000] + train/ActionL1Loss=0.1375 + throughput/total_tokens=139,584,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=728/500000] + train/ActionL1Loss=0.1399 + throughput/total_tokens=139,776,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=729/500000] + train/ActionL1Loss=0.1435 + throughput/total_tokens=139,968,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=730/500000] + train/ActionL1Loss=0.1410 + throughput/total_tokens=140,160,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=731/500000] + train/ActionL1Loss=0.1403 + throughput/total_tokens=140,352,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=732/500000] + train/ActionL1Loss=0.1429 + throughput/total_tokens=140,544,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=733/500000] + train/ActionL1Loss=0.1438 + throughput/total_tokens=140,736,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=734/500000] + train/ActionL1Loss=0.1489 + throughput/total_tokens=140,928,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=735/500000] + train/ActionL1Loss=0.1561 + throughput/total_tokens=141,120,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=736/500000] + train/ActionL1Loss=0.1653 + throughput/total_tokens=141,312,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=737/500000] + train/ActionL1Loss=0.1463 + throughput/total_tokens=141,504,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=738/500000] + train/ActionL1Loss=0.1842 + throughput/total_tokens=141,696,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=739/500000] + train/ActionL1Loss=0.1796 + throughput/total_tokens=141,888,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=740/500000] + optim/total_grad_norm=13.07 + train/ActionL1Loss=0.1747 + throughput/total_tokens=142,080,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=741/500000] + train/ActionL1Loss=0.1499 + throughput/total_tokens=142,272,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=742/500000] + train/ActionL1Loss=0.1509 + throughput/total_tokens=142,464,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=743/500000] + train/ActionL1Loss=0.1874 + throughput/total_tokens=142,656,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=744/500000] + train/ActionL1Loss=0.1866 + throughput/total_tokens=142,848,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=745/500000] + train/ActionL1Loss=0.1692 + throughput/total_tokens=143,040,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=746/500000] + train/ActionL1Loss=0.1704 + throughput/total_tokens=143,232,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=747/500000] + train/ActionL1Loss=0.1718 + throughput/total_tokens=143,424,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=748/500000] + train/ActionL1Loss=0.1846 + throughput/total_tokens=143,616,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=749/500000] + train/ActionL1Loss=0.1342 + throughput/total_tokens=143,808,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=750/500000] + train/ActionL1Loss=0.1687 + throughput/total_tokens=144,000,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=751/500000] + train/ActionL1Loss=0.1737 + throughput/total_tokens=144,192,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=752/500000] + train/ActionL1Loss=0.1428 + throughput/total_tokens=144,384,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=753/500000] + train/ActionL1Loss=0.1427 + throughput/total_tokens=144,576,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=754/500000] + train/ActionL1Loss=0.1580 + throughput/total_tokens=144,768,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=755/500000] + train/ActionL1Loss=0.1741 + throughput/total_tokens=144,960,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=756/500000] + train/ActionL1Loss=0.1640 + throughput/total_tokens=145,152,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=757/500000] + train/ActionL1Loss=0.1556 + throughput/total_tokens=145,344,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=758/500000] + train/ActionL1Loss=0.1626 + throughput/total_tokens=145,536,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=759/500000] + train/ActionL1Loss=0.1660 + throughput/total_tokens=145,728,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=760/500000] + optim/total_grad_norm=11.49 + train/ActionL1Loss=0.1739 + throughput/total_tokens=145,920,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=761/500000] + train/ActionL1Loss=0.1826 + throughput/total_tokens=146,112,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=762/500000] + train/ActionL1Loss=0.1863 + throughput/total_tokens=146,304,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=763/500000] + train/ActionL1Loss=0.1606 + throughput/total_tokens=146,496,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=764/500000] + train/ActionL1Loss=0.1537 + throughput/total_tokens=146,688,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=765/500000] + train/ActionL1Loss=0.1648 + throughput/total_tokens=146,880,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=766/500000] + train/ActionL1Loss=0.1854 + throughput/total_tokens=147,072,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=767/500000] + train/ActionL1Loss=0.1783 + throughput/total_tokens=147,264,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=768/500000] + train/ActionL1Loss=0.1668 + throughput/total_tokens=147,456,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=769/500000] + train/ActionL1Loss=0.1766 + throughput/total_tokens=147,648,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=770/500000] + train/ActionL1Loss=0.1656 + throughput/total_tokens=147,840,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=771/500000] + train/ActionL1Loss=0.1889 + throughput/total_tokens=148,032,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=772/500000] + train/ActionL1Loss=0.1883 + throughput/total_tokens=148,224,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=773/500000] + train/ActionL1Loss=0.1834 + throughput/total_tokens=148,416,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=774/500000] + train/ActionL1Loss=0.1763 + throughput/total_tokens=148,608,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=775/500000] + train/ActionL1Loss=0.1466 + throughput/total_tokens=148,800,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=776/500000] + train/ActionL1Loss=0.1513 + throughput/total_tokens=148,992,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=777/500000] + train/ActionL1Loss=0.1714 + throughput/total_tokens=149,184,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=778/500000] + train/ActionL1Loss=0.1584 + throughput/total_tokens=149,376,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=779/500000] + train/ActionL1Loss=0.1744 + throughput/total_tokens=149,568,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=780/500000] + optim/total_grad_norm=11.31 + train/ActionL1Loss=0.1472 + throughput/total_tokens=149,760,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=781/500000] + train/ActionL1Loss=0.1600 + throughput/total_tokens=149,952,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=782/500000] + train/ActionL1Loss=0.1603 + throughput/total_tokens=150,144,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=783/500000] + train/ActionL1Loss=0.1467 + throughput/total_tokens=150,336,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=784/500000] + train/ActionL1Loss=0.1657 + throughput/total_tokens=150,528,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=785/500000] + train/ActionL1Loss=0.1410 + throughput/total_tokens=150,720,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=786/500000] + train/ActionL1Loss=0.1562 + throughput/total_tokens=150,912,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=787/500000] + train/ActionL1Loss=0.1957 + throughput/total_tokens=151,104,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=788/500000] + train/ActionL1Loss=0.1814 + throughput/total_tokens=151,296,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=789/500000] + train/ActionL1Loss=0.1674 + throughput/total_tokens=151,488,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=790/500000] + train/ActionL1Loss=0.1786 + throughput/total_tokens=151,680,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=791/500000] + train/ActionL1Loss=0.1749 + throughput/total_tokens=151,872,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=792/500000] + train/ActionL1Loss=0.1912 + throughput/total_tokens=152,064,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=793/500000] + train/ActionL1Loss=0.1536 + throughput/total_tokens=152,256,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=794/500000] + train/ActionL1Loss=0.1580 + throughput/total_tokens=152,448,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=795/500000] + train/ActionL1Loss=0.1824 + throughput/total_tokens=152,640,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=796/500000] + train/ActionL1Loss=0.1493 + throughput/total_tokens=152,832,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=797/500000] + train/ActionL1Loss=0.1488 + throughput/total_tokens=153,024,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=798/500000] + train/ActionL1Loss=0.1645 + throughput/total_tokens=153,216,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=799/500000] + train/ActionL1Loss=0.1973 + throughput/total_tokens=153,408,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=800/500000] + optim/total_grad_norm=9.145 + train/ActionL1Loss=0.1505 + throughput/total_tokens=153,600,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=801/500000] + train/ActionL1Loss=0.1653 + throughput/total_tokens=153,792,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=802/500000] + train/ActionL1Loss=0.1586 + throughput/total_tokens=153,984,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=803/500000] + train/ActionL1Loss=0.1435 + throughput/total_tokens=154,176,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=804/500000] + train/ActionL1Loss=0.1600 + throughput/total_tokens=154,368,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=805/500000] + train/ActionL1Loss=0.1432 + throughput/total_tokens=154,560,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=806/500000] + train/ActionL1Loss=0.1667 + throughput/total_tokens=154,752,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=807/500000] + train/ActionL1Loss=0.1732 + throughput/total_tokens=154,944,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=808/500000] + train/ActionL1Loss=0.1419 + throughput/total_tokens=155,136,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=809/500000] + train/ActionL1Loss=0.1734 + throughput/total_tokens=155,328,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=810/500000] + train/ActionL1Loss=0.1556 + throughput/total_tokens=155,520,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=811/500000] + train/ActionL1Loss=0.1434 + throughput/total_tokens=155,712,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=812/500000] + train/ActionL1Loss=0.1686 + throughput/total_tokens=155,904,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=813/500000] + train/ActionL1Loss=0.1577 + throughput/total_tokens=156,096,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=814/500000] + train/ActionL1Loss=0.1399 + throughput/total_tokens=156,288,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=815/500000] + train/ActionL1Loss=0.1553 + throughput/total_tokens=156,480,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=816/500000] + train/ActionL1Loss=0.1439 + throughput/total_tokens=156,672,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=817/500000] + train/ActionL1Loss=0.1585 + throughput/total_tokens=156,864,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=818/500000] + train/ActionL1Loss=0.1449 + throughput/total_tokens=157,056,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=819/500000] + train/ActionL1Loss=0.1586 + throughput/total_tokens=157,248,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=820/500000] + optim/total_grad_norm=7.629 + train/ActionL1Loss=0.1517 + throughput/total_tokens=157,440,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=821/500000] + train/ActionL1Loss=0.1475 + throughput/total_tokens=157,632,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=822/500000] + train/ActionL1Loss=0.1590 + throughput/total_tokens=157,824,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=823/500000] + train/ActionL1Loss=0.1694 + throughput/total_tokens=158,016,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=824/500000] + train/ActionL1Loss=0.1549 + throughput/total_tokens=158,208,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=825/500000] + train/ActionL1Loss=0.1279 + throughput/total_tokens=158,400,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=826/500000] + train/ActionL1Loss=0.1445 + throughput/total_tokens=158,592,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=827/500000] + train/ActionL1Loss=0.1516 + throughput/total_tokens=158,784,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=828/500000] + train/ActionL1Loss=0.1690 + throughput/total_tokens=158,976,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=829/500000] + train/ActionL1Loss=0.1370 + throughput/total_tokens=159,168,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=830/500000] + train/ActionL1Loss=0.1352 + throughput/total_tokens=159,360,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=831/500000] + train/ActionL1Loss=0.1387 + throughput/total_tokens=159,552,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=832/500000] + train/ActionL1Loss=0.1737 + throughput/total_tokens=159,744,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=833/500000] + train/ActionL1Loss=0.1650 + throughput/total_tokens=159,936,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=834/500000] + train/ActionL1Loss=0.1371 + throughput/total_tokens=160,128,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=835/500000] + train/ActionL1Loss=0.1424 + throughput/total_tokens=160,320,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=836/500000] + train/ActionL1Loss=0.1396 + throughput/total_tokens=160,512,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=837/500000] + train/ActionL1Loss=0.1503 + throughput/total_tokens=160,704,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=838/500000] + train/ActionL1Loss=0.1433 + throughput/total_tokens=160,896,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=839/500000] + train/ActionL1Loss=0.1452 + throughput/total_tokens=161,088,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=840/500000] + optim/total_grad_norm=9.636 + train/ActionL1Loss=0.1341 + throughput/total_tokens=161,280,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=841/500000] + train/ActionL1Loss=0.1695 + throughput/total_tokens=161,472,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=842/500000] + train/ActionL1Loss=0.1436 + throughput/total_tokens=161,664,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=843/500000] + train/ActionL1Loss=0.1424 + throughput/total_tokens=161,856,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=844/500000] + train/ActionL1Loss=0.1423 + throughput/total_tokens=162,048,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=845/500000] + train/ActionL1Loss=0.1476 + throughput/total_tokens=162,240,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=846/500000] + train/ActionL1Loss=0.1308 + throughput/total_tokens=162,432,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=847/500000] + train/ActionL1Loss=0.1360 + throughput/total_tokens=162,624,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=848/500000] + train/ActionL1Loss=0.1466 + throughput/total_tokens=162,816,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=849/500000] + train/ActionL1Loss=0.1697 + throughput/total_tokens=163,008,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=850/500000] + train/ActionL1Loss=0.1445 + throughput/total_tokens=163,200,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=851/500000] + train/ActionL1Loss=0.1282 + throughput/total_tokens=163,392,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=852/500000] + train/ActionL1Loss=0.1381 + throughput/total_tokens=163,584,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=853/500000] + train/ActionL1Loss=0.1740 + throughput/total_tokens=163,776,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=854/500000] + train/ActionL1Loss=0.1745 + throughput/total_tokens=163,968,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=855/500000] + train/ActionL1Loss=0.1307 + throughput/total_tokens=164,160,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=856/500000] + train/ActionL1Loss=0.1560 + throughput/total_tokens=164,352,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=857/500000] + train/ActionL1Loss=0.1933 + throughput/total_tokens=164,544,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=858/500000] + train/ActionL1Loss=0.1667 + throughput/total_tokens=164,736,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=859/500000] + train/ActionL1Loss=0.1447 + throughput/total_tokens=164,928,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=860/500000] + optim/total_grad_norm=10.16 + train/ActionL1Loss=0.1464 + throughput/total_tokens=165,120,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=861/500000] + train/ActionL1Loss=0.1788 + throughput/total_tokens=165,312,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=862/500000] + train/ActionL1Loss=0.1575 + throughput/total_tokens=165,504,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=863/500000] + train/ActionL1Loss=0.1543 + throughput/total_tokens=165,696,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=864/500000] + train/ActionL1Loss=0.1463 + throughput/total_tokens=165,888,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=865/500000] + train/ActionL1Loss=0.1722 + throughput/total_tokens=166,080,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=866/500000] + train/ActionL1Loss=0.1714 + throughput/total_tokens=166,272,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=867/500000] + train/ActionL1Loss=0.1578 + throughput/total_tokens=166,464,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=868/500000] + train/ActionL1Loss=0.1680 + throughput/total_tokens=166,656,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=869/500000] + train/ActionL1Loss=0.1692 + throughput/total_tokens=166,848,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=870/500000] + train/ActionL1Loss=0.1468 + throughput/total_tokens=167,040,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=871/500000] + train/ActionL1Loss=0.1555 + throughput/total_tokens=167,232,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=872/500000] + train/ActionL1Loss=0.1717 + throughput/total_tokens=167,424,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=873/500000] + train/ActionL1Loss=0.1743 + throughput/total_tokens=167,616,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=874/500000] + train/ActionL1Loss=0.1614 + throughput/total_tokens=167,808,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=875/500000] + train/ActionL1Loss=0.1628 + throughput/total_tokens=168,000,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=876/500000] + train/ActionL1Loss=0.1586 + throughput/total_tokens=168,192,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=877/500000] + train/ActionL1Loss=0.1576 + throughput/total_tokens=168,384,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=878/500000] + train/ActionL1Loss=0.1539 + throughput/total_tokens=168,576,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=879/500000] + train/ActionL1Loss=0.1627 + throughput/total_tokens=168,768,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=880/500000] + optim/total_grad_norm=11.79 + train/ActionL1Loss=0.1584 + throughput/total_tokens=168,960,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=881/500000] + train/ActionL1Loss=0.1448 + throughput/total_tokens=169,152,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=882/500000] + train/ActionL1Loss=0.1452 + throughput/total_tokens=169,344,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=883/500000] + train/ActionL1Loss=0.1612 + throughput/total_tokens=169,536,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=884/500000] + train/ActionL1Loss=0.1500 + throughput/total_tokens=169,728,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=885/500000] + train/ActionL1Loss=0.1677 + throughput/total_tokens=169,920,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=886/500000] + train/ActionL1Loss=0.1593 + throughput/total_tokens=170,112,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=887/500000] + train/ActionL1Loss=0.1499 + throughput/total_tokens=170,304,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=888/500000] + train/ActionL1Loss=0.1639 + throughput/total_tokens=170,496,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=889/500000] + train/ActionL1Loss=0.1532 + throughput/total_tokens=170,688,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=890/500000] + train/ActionL1Loss=0.1922 + throughput/total_tokens=170,880,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=891/500000] + train/ActionL1Loss=0.1461 + throughput/total_tokens=171,072,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=892/500000] + train/ActionL1Loss=0.1343 + throughput/total_tokens=171,264,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=893/500000] + train/ActionL1Loss=0.1486 + throughput/total_tokens=171,456,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=894/500000] + train/ActionL1Loss=0.1501 + throughput/total_tokens=171,648,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=895/500000] + train/ActionL1Loss=0.1493 + throughput/total_tokens=171,840,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=896/500000] + train/ActionL1Loss=0.1372 + throughput/total_tokens=172,032,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=897/500000] + train/ActionL1Loss=0.1474 + throughput/total_tokens=172,224,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=898/500000] + train/ActionL1Loss=0.1300 + throughput/total_tokens=172,416,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=899/500000] + train/ActionL1Loss=0.1513 + throughput/total_tokens=172,608,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=900/500000] + optim/total_grad_norm=10.51 + train/ActionL1Loss=0.1317 + throughput/total_tokens=172,800,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=901/500000] + train/ActionL1Loss=0.1358 + throughput/total_tokens=172,992,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=902/500000] + train/ActionL1Loss=0.1654 + throughput/total_tokens=173,184,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=903/500000] + train/ActionL1Loss=0.1364 + throughput/total_tokens=173,376,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0482 +[step=904/500000] + train/ActionL1Loss=0.1497 + throughput/total_tokens=173,568,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=905/500000] + train/ActionL1Loss=0.1406 + throughput/total_tokens=173,760,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=906/500000] + train/ActionL1Loss=0.1363 + throughput/total_tokens=173,952,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=907/500000] + train/ActionL1Loss=0.1503 + throughput/total_tokens=174,144,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=908/500000] + train/ActionL1Loss=0.1787 + throughput/total_tokens=174,336,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=909/500000] + train/ActionL1Loss=0.1686 + throughput/total_tokens=174,528,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=910/500000] + train/ActionL1Loss=0.1576 + throughput/total_tokens=174,720,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=911/500000] + train/ActionL1Loss=0.1643 + throughput/total_tokens=174,912,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=912/500000] + train/ActionL1Loss=0.1362 + throughput/total_tokens=175,104,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=913/500000] + train/ActionL1Loss=0.1598 + throughput/total_tokens=175,296,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=914/500000] + train/ActionL1Loss=0.1930 + throughput/total_tokens=175,488,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=915/500000] + train/ActionL1Loss=0.1680 + throughput/total_tokens=175,680,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=916/500000] + train/ActionL1Loss=0.1669 + throughput/total_tokens=175,872,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=917/500000] + train/ActionL1Loss=0.1588 + throughput/total_tokens=176,064,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=918/500000] + train/ActionL1Loss=0.1490 + throughput/total_tokens=176,256,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=919/500000] + train/ActionL1Loss=0.1690 + throughput/total_tokens=176,448,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=920/500000] + optim/total_grad_norm=9.688 + train/ActionL1Loss=0.1555 + throughput/total_tokens=176,640,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=921/500000] + train/ActionL1Loss=0.1434 + throughput/total_tokens=176,832,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=922/500000] + train/ActionL1Loss=0.1468 + throughput/total_tokens=177,024,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=923/500000] + train/ActionL1Loss=0.1595 + throughput/total_tokens=177,216,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=924/500000] + train/ActionL1Loss=0.1475 + throughput/total_tokens=177,408,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=925/500000] + train/ActionL1Loss=0.1507 + throughput/total_tokens=177,600,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=926/500000] + train/ActionL1Loss=0.1558 + throughput/total_tokens=177,792,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=927/500000] + train/ActionL1Loss=0.1657 + throughput/total_tokens=177,984,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=928/500000] + train/ActionL1Loss=0.1596 + throughput/total_tokens=178,176,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=929/500000] + train/ActionL1Loss=0.1429 + throughput/total_tokens=178,368,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=930/500000] + train/ActionL1Loss=0.1374 + throughput/total_tokens=178,560,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=931/500000] + train/ActionL1Loss=0.1604 + throughput/total_tokens=178,752,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=932/500000] + train/ActionL1Loss=0.1634 + throughput/total_tokens=178,944,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=933/500000] + train/ActionL1Loss=0.1583 + throughput/total_tokens=179,136,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=934/500000] + train/ActionL1Loss=0.1590 + throughput/total_tokens=179,328,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=935/500000] + train/ActionL1Loss=0.1566 + throughput/total_tokens=179,520,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=936/500000] + train/ActionL1Loss=0.1686 + throughput/total_tokens=179,712,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=937/500000] + train/ActionL1Loss=0.1293 + throughput/total_tokens=179,904,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=938/500000] + train/ActionL1Loss=0.1465 + throughput/total_tokens=180,096,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=939/500000] + train/ActionL1Loss=0.1619 + throughput/total_tokens=180,288,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=940/500000] + optim/total_grad_norm=10.06 + train/ActionL1Loss=0.1491 + throughput/total_tokens=180,480,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=941/500000] + train/ActionL1Loss=0.1804 + throughput/total_tokens=180,672,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=942/500000] + train/ActionL1Loss=0.1496 + throughput/total_tokens=180,864,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=943/500000] + train/ActionL1Loss=0.1239 + throughput/total_tokens=181,056,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=944/500000] + train/ActionL1Loss=0.1376 + throughput/total_tokens=181,248,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=945/500000] + train/ActionL1Loss=0.1449 + throughput/total_tokens=181,440,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=946/500000] + train/ActionL1Loss=0.1448 + throughput/total_tokens=181,632,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=947/500000] + train/ActionL1Loss=0.1421 + throughput/total_tokens=181,824,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=948/500000] + train/ActionL1Loss=0.1656 + throughput/total_tokens=182,016,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=949/500000] + train/ActionL1Loss=0.1326 + throughput/total_tokens=182,208,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=950/500000] + train/ActionL1Loss=0.1490 + throughput/total_tokens=182,400,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=951/500000] + train/ActionL1Loss=0.1549 + throughput/total_tokens=182,592,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=952/500000] + train/ActionL1Loss=0.1402 + throughput/total_tokens=182,784,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=953/500000] + train/ActionL1Loss=0.1353 + throughput/total_tokens=182,976,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=954/500000] + train/ActionL1Loss=0.1348 + throughput/total_tokens=183,168,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=955/500000] + train/ActionL1Loss=0.1315 + throughput/total_tokens=183,360,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=956/500000] + train/ActionL1Loss=0.1447 + throughput/total_tokens=183,552,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=957/500000] + train/ActionL1Loss=0.1222 + throughput/total_tokens=183,744,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=958/500000] + train/ActionL1Loss=0.1395 + throughput/total_tokens=183,936,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=959/500000] + train/ActionL1Loss=0.1518 + throughput/total_tokens=184,128,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=960/500000] + optim/total_grad_norm=8.135 + train/ActionL1Loss=0.1476 + throughput/total_tokens=184,320,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=961/500000] + train/ActionL1Loss=0.1299 + throughput/total_tokens=184,512,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=962/500000] + train/ActionL1Loss=0.1476 + throughput/total_tokens=184,704,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=963/500000] + train/ActionL1Loss=0.1244 + throughput/total_tokens=184,896,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=964/500000] + train/ActionL1Loss=0.1293 + throughput/total_tokens=185,088,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=965/500000] + train/ActionL1Loss=0.1355 + throughput/total_tokens=185,280,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=966/500000] + train/ActionL1Loss=0.1192 + throughput/total_tokens=185,472,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=967/500000] + train/ActionL1Loss=0.1280 + throughput/total_tokens=185,664,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=968/500000] + train/ActionL1Loss=0.1541 + throughput/total_tokens=185,856,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=969/500000] + train/ActionL1Loss=0.1724 + throughput/total_tokens=186,048,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=970/500000] + train/ActionL1Loss=0.1471 + throughput/total_tokens=186,240,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=971/500000] + train/ActionL1Loss=0.1426 + throughput/total_tokens=186,432,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=972/500000] + train/ActionL1Loss=0.1510 + throughput/total_tokens=186,624,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=973/500000] + train/ActionL1Loss=0.1675 + throughput/total_tokens=186,816,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=974/500000] + train/ActionL1Loss=0.1430 + throughput/total_tokens=187,008,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=975/500000] + train/ActionL1Loss=0.1444 + throughput/total_tokens=187,200,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=976/500000] + train/ActionL1Loss=0.1458 + throughput/total_tokens=187,392,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=977/500000] + train/ActionL1Loss=0.1407 + throughput/total_tokens=187,584,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=978/500000] + train/ActionL1Loss=0.1449 + throughput/total_tokens=187,776,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=979/500000] + train/ActionL1Loss=0.1397 + throughput/total_tokens=187,968,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=980/500000] + optim/total_grad_norm=9.094 + train/ActionL1Loss=0.1425 + throughput/total_tokens=188,160,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=981/500000] + train/ActionL1Loss=0.1390 + throughput/total_tokens=188,352,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=982/500000] + train/ActionL1Loss=0.1411 + throughput/total_tokens=188,544,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=983/500000] + train/ActionL1Loss=0.1551 + throughput/total_tokens=188,736,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=984/500000] + train/ActionL1Loss=0.1177 + throughput/total_tokens=188,928,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=985/500000] + train/ActionL1Loss=0.1313 + throughput/total_tokens=189,120,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=986/500000] + train/ActionL1Loss=0.1226 + throughput/total_tokens=189,312,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=987/500000] + train/ActionL1Loss=0.1339 + throughput/total_tokens=189,504,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=988/500000] + train/ActionL1Loss=0.1336 + throughput/total_tokens=189,696,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=989/500000] + train/ActionL1Loss=0.1321 + throughput/total_tokens=189,888,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=990/500000] + train/ActionL1Loss=0.1378 + throughput/total_tokens=190,080,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=991/500000] + train/ActionL1Loss=0.1521 + throughput/total_tokens=190,272,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=992/500000] + train/ActionL1Loss=0.1443 + throughput/total_tokens=190,464,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=993/500000] + train/ActionL1Loss=0.1345 + throughput/total_tokens=190,656,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=994/500000] + train/ActionL1Loss=0.1419 + throughput/total_tokens=190,848,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=995/500000] + train/ActionL1Loss=0.1351 + throughput/total_tokens=191,040,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=996/500000] + train/ActionL1Loss=0.1348 + throughput/total_tokens=191,232,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=997/500000] + train/ActionL1Loss=0.1252 + throughput/total_tokens=191,424,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=998/500000] + train/ActionL1Loss=0.1528 + throughput/total_tokens=191,616,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=999/500000] + train/ActionL1Loss=0.1400 + throughput/total_tokens=191,808,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1000/500000] + optim/total_grad_norm=8.571 + train/ActionL1Loss=0.1402 + throughput/total_tokens=192,000,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +09/30 [19:05:19] INFO | >> Saving config... checkpoint.py:608 +09/30 [19:06:02] INFO | >> Saving model state... checkpoint.py:796 +09/30 [19:07:10] INFO | >> Saving optim state... checkpoint.py:811 +09/30 [19:08:42] INFO | >> Saving trainer state... checkpoint.py:670 + INFO | >> Saving config... checkpoint.py:608 +[step=1001/500000] + train/ActionL1Loss=0.1380 + throughput/total_tokens=192,192,000 + throughput/device/tokens_per_second=1,208 + throughput/device/batches_per_second=0.0504 +[step=1002/500000] + train/ActionL1Loss=0.1510 + throughput/total_tokens=192,384,000 + throughput/device/tokens_per_second=1,181 + throughput/device/batches_per_second=0.0492 +[step=1003/500000] + train/ActionL1Loss=0.1533 + throughput/total_tokens=192,576,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=1004/500000] + train/ActionL1Loss=0.1215 + throughput/total_tokens=192,768,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=1005/500000] + train/ActionL1Loss=0.1401 + throughput/total_tokens=192,960,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1006/500000] + train/ActionL1Loss=0.1456 + throughput/total_tokens=193,152,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1007/500000] + train/ActionL1Loss=0.1613 + throughput/total_tokens=193,344,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1008/500000] + train/ActionL1Loss=0.1520 + throughput/total_tokens=193,536,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1009/500000] + train/ActionL1Loss=0.1174 + throughput/total_tokens=193,728,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1010/500000] + train/ActionL1Loss=0.1458 + throughput/total_tokens=193,920,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1011/500000] + train/ActionL1Loss=0.1642 + throughput/total_tokens=194,112,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1012/500000] + train/ActionL1Loss=0.1559 + throughput/total_tokens=194,304,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1013/500000] + train/ActionL1Loss=0.1421 + throughput/total_tokens=194,496,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1014/500000] + train/ActionL1Loss=0.1449 + throughput/total_tokens=194,688,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1015/500000] + train/ActionL1Loss=0.1318 + throughput/total_tokens=194,880,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1016/500000] + train/ActionL1Loss=0.1640 + throughput/total_tokens=195,072,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1017/500000] + train/ActionL1Loss=0.1886 + throughput/total_tokens=195,264,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1018/500000] + train/ActionL1Loss=0.1359 + throughput/total_tokens=195,456,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1019/500000] + train/ActionL1Loss=0.1465 + throughput/total_tokens=195,648,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1020/500000] + optim/total_grad_norm=12.15 + train/ActionL1Loss=0.1839 + throughput/total_tokens=195,840,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=1021/500000] + train/ActionL1Loss=0.1961 + throughput/total_tokens=196,032,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1022/500000] + train/ActionL1Loss=0.1845 + throughput/total_tokens=196,224,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1023/500000] + train/ActionL1Loss=0.1471 + throughput/total_tokens=196,416,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1024/500000] + train/ActionL1Loss=0.1432 + throughput/total_tokens=196,608,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1025/500000] + train/ActionL1Loss=0.1258 + throughput/total_tokens=196,800,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1026/500000] + train/ActionL1Loss=0.1381 + throughput/total_tokens=196,992,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1027/500000] + train/ActionL1Loss=0.1532 + throughput/total_tokens=197,184,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1028/500000] + train/ActionL1Loss=0.1339 + throughput/total_tokens=197,376,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1029/500000] + train/ActionL1Loss=0.1344 + throughput/total_tokens=197,568,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1030/500000] + train/ActionL1Loss=0.1538 + throughput/total_tokens=197,760,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1031/500000] + train/ActionL1Loss=0.1487 + throughput/total_tokens=197,952,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1032/500000] + train/ActionL1Loss=0.1556 + throughput/total_tokens=198,144,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1033/500000] + train/ActionL1Loss=0.1649 + throughput/total_tokens=198,336,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1034/500000] + train/ActionL1Loss=0.1547 + throughput/total_tokens=198,528,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1035/500000] + train/ActionL1Loss=0.1602 + throughput/total_tokens=198,720,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1036/500000] + train/ActionL1Loss=0.1372 + throughput/total_tokens=198,912,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1037/500000] + train/ActionL1Loss=0.1455 + throughput/total_tokens=199,104,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1038/500000] + train/ActionL1Loss=0.1631 + throughput/total_tokens=199,296,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1039/500000] + train/ActionL1Loss=0.1459 + throughput/total_tokens=199,488,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1040/500000] + optim/total_grad_norm=7.662 + train/ActionL1Loss=0.1411 + throughput/total_tokens=199,680,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1041/500000] + train/ActionL1Loss=0.1381 + throughput/total_tokens=199,872,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1042/500000] + train/ActionL1Loss=0.1367 + throughput/total_tokens=200,064,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1043/500000] + train/ActionL1Loss=0.1429 + throughput/total_tokens=200,256,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1044/500000] + train/ActionL1Loss=0.1520 + throughput/total_tokens=200,448,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1045/500000] + train/ActionL1Loss=0.1299 + throughput/total_tokens=200,640,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1046/500000] + train/ActionL1Loss=0.1317 + throughput/total_tokens=200,832,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1047/500000] + train/ActionL1Loss=0.1491 + throughput/total_tokens=201,024,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1048/500000] + train/ActionL1Loss=0.1373 + throughput/total_tokens=201,216,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1049/500000] + train/ActionL1Loss=0.1405 + throughput/total_tokens=201,408,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1050/500000] + train/ActionL1Loss=0.1661 + throughput/total_tokens=201,600,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1051/500000] + train/ActionL1Loss=0.1252 + throughput/total_tokens=201,792,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1052/500000] + train/ActionL1Loss=0.1347 + throughput/total_tokens=201,984,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1053/500000] + train/ActionL1Loss=0.1521 + throughput/total_tokens=202,176,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1054/500000] + train/ActionL1Loss=0.1615 + throughput/total_tokens=202,368,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1055/500000] + train/ActionL1Loss=0.1477 + throughput/total_tokens=202,560,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1056/500000] + train/ActionL1Loss=0.1579 + throughput/total_tokens=202,752,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1057/500000] + train/ActionL1Loss=0.1211 + throughput/total_tokens=202,944,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1058/500000] + train/ActionL1Loss=0.1509 + throughput/total_tokens=203,136,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1059/500000] + train/ActionL1Loss=0.1479 + throughput/total_tokens=203,328,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1060/500000] + optim/total_grad_norm=8.221 + train/ActionL1Loss=0.1295 + throughput/total_tokens=203,520,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=1061/500000] + train/ActionL1Loss=0.1427 + throughput/total_tokens=203,712,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1062/500000] + train/ActionL1Loss=0.1454 + throughput/total_tokens=203,904,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1063/500000] + train/ActionL1Loss=0.1301 + throughput/total_tokens=204,096,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1064/500000] + train/ActionL1Loss=0.1516 + throughput/total_tokens=204,288,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1065/500000] + train/ActionL1Loss=0.1365 + throughput/total_tokens=204,480,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1066/500000] + train/ActionL1Loss=0.1634 + throughput/total_tokens=204,672,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1067/500000] + train/ActionL1Loss=0.1326 + throughput/total_tokens=204,864,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1068/500000] + train/ActionL1Loss=0.1410 + throughput/total_tokens=205,056,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1069/500000] + train/ActionL1Loss=0.1340 + throughput/total_tokens=205,248,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1070/500000] + train/ActionL1Loss=0.1576 + throughput/total_tokens=205,440,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1071/500000] + train/ActionL1Loss=0.1407 + throughput/total_tokens=205,632,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1072/500000] + train/ActionL1Loss=0.1353 + throughput/total_tokens=205,824,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1073/500000] + train/ActionL1Loss=0.1584 + throughput/total_tokens=206,016,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1074/500000] + train/ActionL1Loss=0.1409 + throughput/total_tokens=206,208,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1075/500000] + train/ActionL1Loss=0.1333 + throughput/total_tokens=206,400,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1076/500000] + train/ActionL1Loss=0.1377 + throughput/total_tokens=206,592,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1077/500000] + train/ActionL1Loss=0.1546 + throughput/total_tokens=206,784,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1078/500000] + train/ActionL1Loss=0.1409 + throughput/total_tokens=206,976,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1079/500000] + train/ActionL1Loss=0.1694 + throughput/total_tokens=207,168,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1080/500000] + optim/total_grad_norm=8.767 + train/ActionL1Loss=0.1425 + throughput/total_tokens=207,360,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1081/500000] + train/ActionL1Loss=0.1441 + throughput/total_tokens=207,552,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1082/500000] + train/ActionL1Loss=0.1550 + throughput/total_tokens=207,744,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1083/500000] + train/ActionL1Loss=0.1369 + throughput/total_tokens=207,936,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1084/500000] + train/ActionL1Loss=0.1423 + throughput/total_tokens=208,128,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1085/500000] + train/ActionL1Loss=0.1220 + throughput/total_tokens=208,320,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1086/500000] + train/ActionL1Loss=0.1557 + throughput/total_tokens=208,512,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1087/500000] + train/ActionL1Loss=0.1539 + throughput/total_tokens=208,704,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1088/500000] + train/ActionL1Loss=0.1349 + throughput/total_tokens=208,896,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1089/500000] + train/ActionL1Loss=0.1569 + throughput/total_tokens=209,088,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1090/500000] + train/ActionL1Loss=0.1452 + throughput/total_tokens=209,280,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1091/500000] + train/ActionL1Loss=0.1225 + throughput/total_tokens=209,472,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1092/500000] + train/ActionL1Loss=0.1646 + throughput/total_tokens=209,664,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1093/500000] + train/ActionL1Loss=0.1394 + throughput/total_tokens=209,856,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1094/500000] + train/ActionL1Loss=0.1264 + throughput/total_tokens=210,048,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1095/500000] + train/ActionL1Loss=0.1501 + throughput/total_tokens=210,240,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1096/500000] + train/ActionL1Loss=0.1453 + throughput/total_tokens=210,432,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1097/500000] + train/ActionL1Loss=0.1411 + throughput/total_tokens=210,624,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1098/500000] + train/ActionL1Loss=0.1420 + throughput/total_tokens=210,816,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1099/500000] + train/ActionL1Loss=0.1357 + throughput/total_tokens=211,008,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1100/500000] + optim/total_grad_norm=8.492 + train/ActionL1Loss=0.1488 + throughput/total_tokens=211,200,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1101/500000] + train/ActionL1Loss=0.1638 + throughput/total_tokens=211,392,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1102/500000] + train/ActionL1Loss=0.1508 + throughput/total_tokens=211,584,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1103/500000] + train/ActionL1Loss=0.1441 + throughput/total_tokens=211,776,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1104/500000] + train/ActionL1Loss=0.1328 + throughput/total_tokens=211,968,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1105/500000] + train/ActionL1Loss=0.1574 + throughput/total_tokens=212,160,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1106/500000] + train/ActionL1Loss=0.1525 + throughput/total_tokens=212,352,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1107/500000] + train/ActionL1Loss=0.1423 + throughput/total_tokens=212,544,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1108/500000] + train/ActionL1Loss=0.1139 + throughput/total_tokens=212,736,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1109/500000] + train/ActionL1Loss=0.1586 + throughput/total_tokens=212,928,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1110/500000] + train/ActionL1Loss=0.1805 + throughput/total_tokens=213,120,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=1111/500000] + train/ActionL1Loss=0.1477 + throughput/total_tokens=213,312,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1112/500000] + train/ActionL1Loss=0.1570 + throughput/total_tokens=213,504,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1113/500000] + train/ActionL1Loss=0.1554 + throughput/total_tokens=213,696,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1114/500000] + train/ActionL1Loss=0.1416 + throughput/total_tokens=213,888,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1115/500000] + train/ActionL1Loss=0.1719 + throughput/total_tokens=214,080,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1116/500000] + train/ActionL1Loss=0.1594 + throughput/total_tokens=214,272,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1117/500000] + train/ActionL1Loss=0.1447 + throughput/total_tokens=214,464,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1118/500000] + train/ActionL1Loss=0.1488 + throughput/total_tokens=214,656,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1119/500000] + train/ActionL1Loss=0.1463 + throughput/total_tokens=214,848,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1120/500000] + optim/total_grad_norm=7.752 + train/ActionL1Loss=0.1292 + throughput/total_tokens=215,040,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1121/500000] + train/ActionL1Loss=0.1561 + throughput/total_tokens=215,232,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1122/500000] + train/ActionL1Loss=0.1302 + throughput/total_tokens=215,424,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1123/500000] + train/ActionL1Loss=0.1409 + throughput/total_tokens=215,616,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1124/500000] + train/ActionL1Loss=0.1380 + throughput/total_tokens=215,808,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1125/500000] + train/ActionL1Loss=0.1431 + throughput/total_tokens=216,000,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1126/500000] + train/ActionL1Loss=0.1293 + throughput/total_tokens=216,192,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1127/500000] + train/ActionL1Loss=0.1570 + throughput/total_tokens=216,384,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1128/500000] + train/ActionL1Loss=0.1300 + throughput/total_tokens=216,576,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1129/500000] + train/ActionL1Loss=0.1595 + throughput/total_tokens=216,768,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1130/500000] + train/ActionL1Loss=0.1438 + throughput/total_tokens=216,960,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1131/500000] + train/ActionL1Loss=0.1298 + throughput/total_tokens=217,152,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1132/500000] + train/ActionL1Loss=0.1372 + throughput/total_tokens=217,344,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1133/500000] + train/ActionL1Loss=0.1329 + throughput/total_tokens=217,536,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1134/500000] + train/ActionL1Loss=0.1549 + throughput/total_tokens=217,728,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1135/500000] + train/ActionL1Loss=0.1307 + throughput/total_tokens=217,920,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1136/500000] + train/ActionL1Loss=0.1560 + throughput/total_tokens=218,112,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1137/500000] + train/ActionL1Loss=0.1163 + throughput/total_tokens=218,304,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1138/500000] + train/ActionL1Loss=0.1054 + throughput/total_tokens=218,496,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1139/500000] + train/ActionL1Loss=0.1550 + throughput/total_tokens=218,688,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1140/500000] + optim/total_grad_norm=8.216 + train/ActionL1Loss=0.1487 + throughput/total_tokens=218,880,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1141/500000] + train/ActionL1Loss=0.1262 + throughput/total_tokens=219,072,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1142/500000] + train/ActionL1Loss=0.1087 + throughput/total_tokens=219,264,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1143/500000] + train/ActionL1Loss=0.1503 + throughput/total_tokens=219,456,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1144/500000] + train/ActionL1Loss=0.1453 + throughput/total_tokens=219,648,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1145/500000] + train/ActionL1Loss=0.1532 + throughput/total_tokens=219,840,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1146/500000] + train/ActionL1Loss=0.1213 + throughput/total_tokens=220,032,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1147/500000] + train/ActionL1Loss=0.1415 + throughput/total_tokens=220,224,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1148/500000] + train/ActionL1Loss=0.1548 + throughput/total_tokens=220,416,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1149/500000] + train/ActionL1Loss=0.1579 + throughput/total_tokens=220,608,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1150/500000] + train/ActionL1Loss=0.1539 + throughput/total_tokens=220,800,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1151/500000] + train/ActionL1Loss=0.1339 + throughput/total_tokens=220,992,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1152/500000] + train/ActionL1Loss=0.1177 + throughput/total_tokens=221,184,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1153/500000] + train/ActionL1Loss=0.1384 + throughput/total_tokens=221,376,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1154/500000] + train/ActionL1Loss=0.1318 + throughput/total_tokens=221,568,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1155/500000] + train/ActionL1Loss=0.1365 + throughput/total_tokens=221,760,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1156/500000] + train/ActionL1Loss=0.1225 + throughput/total_tokens=221,952,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1157/500000] + train/ActionL1Loss=0.1310 + throughput/total_tokens=222,144,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1158/500000] + train/ActionL1Loss=0.1213 + throughput/total_tokens=222,336,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1159/500000] + train/ActionL1Loss=0.1269 + throughput/total_tokens=222,528,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1160/500000] + optim/total_grad_norm=5.809 + train/ActionL1Loss=0.1305 + throughput/total_tokens=222,720,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=1161/500000] + train/ActionL1Loss=0.1192 + throughput/total_tokens=222,912,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1162/500000] + train/ActionL1Loss=0.1218 + throughput/total_tokens=223,104,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1163/500000] + train/ActionL1Loss=0.1256 + throughput/total_tokens=223,296,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1164/500000] + train/ActionL1Loss=0.1264 + throughput/total_tokens=223,488,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1165/500000] + train/ActionL1Loss=0.1271 + throughput/total_tokens=223,680,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1166/500000] + train/ActionL1Loss=0.1281 + throughput/total_tokens=223,872,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1167/500000] + train/ActionL1Loss=0.1360 + throughput/total_tokens=224,064,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1168/500000] + train/ActionL1Loss=0.1420 + throughput/total_tokens=224,256,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1169/500000] + train/ActionL1Loss=0.1473 + throughput/total_tokens=224,448,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1170/500000] + train/ActionL1Loss=0.1551 + throughput/total_tokens=224,640,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1171/500000] + train/ActionL1Loss=0.1700 + throughput/total_tokens=224,832,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1172/500000] + train/ActionL1Loss=0.1373 + throughput/total_tokens=225,024,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1173/500000] + train/ActionL1Loss=0.1642 + throughput/total_tokens=225,216,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1174/500000] + train/ActionL1Loss=0.1725 + throughput/total_tokens=225,408,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1175/500000] + train/ActionL1Loss=0.1602 + throughput/total_tokens=225,600,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1176/500000] + train/ActionL1Loss=0.1536 + throughput/total_tokens=225,792,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1177/500000] + train/ActionL1Loss=0.1258 + throughput/total_tokens=225,984,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1178/500000] + train/ActionL1Loss=0.1451 + throughput/total_tokens=226,176,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1179/500000] + train/ActionL1Loss=0.1668 + throughput/total_tokens=226,368,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1180/500000] + optim/total_grad_norm=7.688 + train/ActionL1Loss=0.1467 + throughput/total_tokens=226,560,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1181/500000] + train/ActionL1Loss=0.1458 + throughput/total_tokens=226,752,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1182/500000] + train/ActionL1Loss=0.1621 + throughput/total_tokens=226,944,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1183/500000] + train/ActionL1Loss=0.1484 + throughput/total_tokens=227,136,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1184/500000] + train/ActionL1Loss=0.1393 + throughput/total_tokens=227,328,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1185/500000] + train/ActionL1Loss=0.1420 + throughput/total_tokens=227,520,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1186/500000] + train/ActionL1Loss=0.1262 + throughput/total_tokens=227,712,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1187/500000] + train/ActionL1Loss=0.1204 + throughput/total_tokens=227,904,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1188/500000] + train/ActionL1Loss=0.1081 + throughput/total_tokens=228,096,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1189/500000] + train/ActionL1Loss=0.1262 + throughput/total_tokens=228,288,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1190/500000] + train/ActionL1Loss=0.1213 + throughput/total_tokens=228,480,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1191/500000] + train/ActionL1Loss=0.1316 + throughput/total_tokens=228,672,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1192/500000] + train/ActionL1Loss=0.1308 + throughput/total_tokens=228,864,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1193/500000] + train/ActionL1Loss=0.1589 + throughput/total_tokens=229,056,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1194/500000] + train/ActionL1Loss=0.1537 + throughput/total_tokens=229,248,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1195/500000] + train/ActionL1Loss=0.1541 + throughput/total_tokens=229,440,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1196/500000] + train/ActionL1Loss=0.1420 + throughput/total_tokens=229,632,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1197/500000] + train/ActionL1Loss=0.1552 + throughput/total_tokens=229,824,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1198/500000] + train/ActionL1Loss=0.1492 + throughput/total_tokens=230,016,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1199/500000] + train/ActionL1Loss=0.1464 + throughput/total_tokens=230,208,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1200/500000] + optim/total_grad_norm=7.543 + train/ActionL1Loss=0.1461 + throughput/total_tokens=230,400,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1201/500000] + train/ActionL1Loss=0.1505 + throughput/total_tokens=230,592,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1202/500000] + train/ActionL1Loss=0.1520 + throughput/total_tokens=230,784,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1203/500000] + train/ActionL1Loss=0.1386 + throughput/total_tokens=230,976,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1204/500000] + train/ActionL1Loss=0.1367 + throughput/total_tokens=231,168,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1205/500000] + train/ActionL1Loss=0.1413 + throughput/total_tokens=231,360,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1206/500000] + train/ActionL1Loss=0.1485 + throughput/total_tokens=231,552,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1207/500000] + train/ActionL1Loss=0.1570 + throughput/total_tokens=231,744,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1208/500000] + train/ActionL1Loss=0.1298 + throughput/total_tokens=231,936,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1209/500000] + train/ActionL1Loss=0.1360 + throughput/total_tokens=232,128,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1210/500000] + train/ActionL1Loss=0.1354 + throughput/total_tokens=232,320,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=1211/500000] + train/ActionL1Loss=0.1497 + throughput/total_tokens=232,512,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1212/500000] + train/ActionL1Loss=0.1693 + throughput/total_tokens=232,704,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1213/500000] + train/ActionL1Loss=0.1594 + throughput/total_tokens=232,896,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1214/500000] + train/ActionL1Loss=0.1445 + throughput/total_tokens=233,088,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1215/500000] + train/ActionL1Loss=0.1590 + throughput/total_tokens=233,280,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1216/500000] + train/ActionL1Loss=0.1360 + throughput/total_tokens=233,472,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1217/500000] + train/ActionL1Loss=0.1358 + throughput/total_tokens=233,664,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1218/500000] + train/ActionL1Loss=0.1079 + throughput/total_tokens=233,856,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1219/500000] + train/ActionL1Loss=0.1408 + throughput/total_tokens=234,048,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1220/500000] + optim/total_grad_norm=9.819 + train/ActionL1Loss=0.1333 + throughput/total_tokens=234,240,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1221/500000] + train/ActionL1Loss=0.1422 + throughput/total_tokens=234,432,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1222/500000] + train/ActionL1Loss=0.1601 + throughput/total_tokens=234,624,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1223/500000] + train/ActionL1Loss=0.1318 + throughput/total_tokens=234,816,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1224/500000] + train/ActionL1Loss=0.1388 + throughput/total_tokens=235,008,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1225/500000] + train/ActionL1Loss=0.1417 + throughput/total_tokens=235,200,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1226/500000] + train/ActionL1Loss=0.1340 + throughput/total_tokens=235,392,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1227/500000] + train/ActionL1Loss=0.1388 + throughput/total_tokens=235,584,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1228/500000] + train/ActionL1Loss=0.1348 + throughput/total_tokens=235,776,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1229/500000] + train/ActionL1Loss=0.1389 + throughput/total_tokens=235,968,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1230/500000] + train/ActionL1Loss=0.1448 + throughput/total_tokens=236,160,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=1231/500000] + train/ActionL1Loss=0.1559 + throughput/total_tokens=236,352,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1232/500000] + train/ActionL1Loss=0.1377 + throughput/total_tokens=236,544,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1233/500000] + train/ActionL1Loss=0.1346 + throughput/total_tokens=236,736,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1234/500000] + train/ActionL1Loss=0.1393 + throughput/total_tokens=236,928,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1235/500000] + train/ActionL1Loss=0.1631 + throughput/total_tokens=237,120,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1236/500000] + train/ActionL1Loss=0.1488 + throughput/total_tokens=237,312,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1237/500000] + train/ActionL1Loss=0.1288 + throughput/total_tokens=237,504,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1238/500000] + train/ActionL1Loss=0.1342 + throughput/total_tokens=237,696,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1239/500000] + train/ActionL1Loss=0.1295 + throughput/total_tokens=237,888,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1240/500000] + optim/total_grad_norm=7.144 + train/ActionL1Loss=0.1241 + throughput/total_tokens=238,080,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=1241/500000] + train/ActionL1Loss=0.1370 + throughput/total_tokens=238,272,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1242/500000] + train/ActionL1Loss=0.1356 + throughput/total_tokens=238,464,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1243/500000] + train/ActionL1Loss=0.1256 + throughput/total_tokens=238,656,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1244/500000] + train/ActionL1Loss=0.1181 + throughput/total_tokens=238,848,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1245/500000] + train/ActionL1Loss=0.1372 + throughput/total_tokens=239,040,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1246/500000] + train/ActionL1Loss=0.1440 + throughput/total_tokens=239,232,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1247/500000] + train/ActionL1Loss=0.1233 + throughput/total_tokens=239,424,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1248/500000] + train/ActionL1Loss=0.1295 + throughput/total_tokens=239,616,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1249/500000] + train/ActionL1Loss=0.1121 + throughput/total_tokens=239,808,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1250/500000] + train/ActionL1Loss=0.1231 + throughput/total_tokens=240,000,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1251/500000] + train/ActionL1Loss=0.1248 + throughput/total_tokens=240,192,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1252/500000] + train/ActionL1Loss=0.1027 + throughput/total_tokens=240,384,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1253/500000] + train/ActionL1Loss=0.1307 + throughput/total_tokens=240,576,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1254/500000] + train/ActionL1Loss=0.1256 + throughput/total_tokens=240,768,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1255/500000] + train/ActionL1Loss=0.1470 + throughput/total_tokens=240,960,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1256/500000] + train/ActionL1Loss=0.1349 + throughput/total_tokens=241,152,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1257/500000] + train/ActionL1Loss=0.1329 + throughput/total_tokens=241,344,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1258/500000] + train/ActionL1Loss=0.1419 + throughput/total_tokens=241,536,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1259/500000] + train/ActionL1Loss=0.1498 + throughput/total_tokens=241,728,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1260/500000] + optim/total_grad_norm=6.370 + train/ActionL1Loss=0.1325 + throughput/total_tokens=241,920,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=1261/500000] + train/ActionL1Loss=0.1290 + throughput/total_tokens=242,112,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1262/500000] + train/ActionL1Loss=0.1184 + throughput/total_tokens=242,304,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1263/500000] + train/ActionL1Loss=0.1175 + throughput/total_tokens=242,496,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1264/500000] + train/ActionL1Loss=0.1442 + throughput/total_tokens=242,688,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1265/500000] + train/ActionL1Loss=0.1388 + throughput/total_tokens=242,880,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1266/500000] + train/ActionL1Loss=0.1236 + throughput/total_tokens=243,072,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1267/500000] + train/ActionL1Loss=0.1258 + throughput/total_tokens=243,264,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1268/500000] + train/ActionL1Loss=0.1291 + throughput/total_tokens=243,456,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1269/500000] + train/ActionL1Loss=0.1389 + throughput/total_tokens=243,648,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1270/500000] + train/ActionL1Loss=0.1507 + throughput/total_tokens=243,840,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1271/500000] + train/ActionL1Loss=0.1300 + throughput/total_tokens=244,032,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1272/500000] + train/ActionL1Loss=0.1334 + throughput/total_tokens=244,224,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1273/500000] + train/ActionL1Loss=0.1335 + throughput/total_tokens=244,416,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1274/500000] + train/ActionL1Loss=0.1073 + throughput/total_tokens=244,608,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1275/500000] + train/ActionL1Loss=0.1226 + throughput/total_tokens=244,800,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1276/500000] + train/ActionL1Loss=0.1335 + throughput/total_tokens=244,992,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1277/500000] + train/ActionL1Loss=0.1279 + throughput/total_tokens=245,184,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1278/500000] + train/ActionL1Loss=0.1314 + throughput/total_tokens=245,376,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1279/500000] + train/ActionL1Loss=0.1412 + throughput/total_tokens=245,568,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1280/500000] + optim/total_grad_norm=6.727 + train/ActionL1Loss=0.1450 + throughput/total_tokens=245,760,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1281/500000] + train/ActionL1Loss=0.1652 + throughput/total_tokens=245,952,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1282/500000] + train/ActionL1Loss=0.1230 + throughput/total_tokens=246,144,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1283/500000] + train/ActionL1Loss=0.1575 + throughput/total_tokens=246,336,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1284/500000] + train/ActionL1Loss=0.1443 + throughput/total_tokens=246,528,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1285/500000] + train/ActionL1Loss=0.1574 + throughput/total_tokens=246,720,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1286/500000] + train/ActionL1Loss=0.1436 + throughput/total_tokens=246,912,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1287/500000] + train/ActionL1Loss=0.1505 + throughput/total_tokens=247,104,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1288/500000] + train/ActionL1Loss=0.1489 + throughput/total_tokens=247,296,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1289/500000] + train/ActionL1Loss=0.1390 + throughput/total_tokens=247,488,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1290/500000] + train/ActionL1Loss=0.1546 + throughput/total_tokens=247,680,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1291/500000] + train/ActionL1Loss=0.1592 + throughput/total_tokens=247,872,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1292/500000] + train/ActionL1Loss=0.1238 + throughput/total_tokens=248,064,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1293/500000] + train/ActionL1Loss=0.1311 + throughput/total_tokens=248,256,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1294/500000] + train/ActionL1Loss=0.1376 + throughput/total_tokens=248,448,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1295/500000] + train/ActionL1Loss=0.1221 + throughput/total_tokens=248,640,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1296/500000] + train/ActionL1Loss=0.1473 + throughput/total_tokens=248,832,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1297/500000] + train/ActionL1Loss=0.1518 + throughput/total_tokens=249,024,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1298/500000] + train/ActionL1Loss=0.1385 + throughput/total_tokens=249,216,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1299/500000] + train/ActionL1Loss=0.1161 + throughput/total_tokens=249,408,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1300/500000] + optim/total_grad_norm=5.362 + train/ActionL1Loss=0.1453 + throughput/total_tokens=249,600,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1301/500000] + train/ActionL1Loss=0.1219 + throughput/total_tokens=249,792,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1302/500000] + train/ActionL1Loss=0.1241 + throughput/total_tokens=249,984,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1303/500000] + train/ActionL1Loss=0.1376 + throughput/total_tokens=250,176,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1304/500000] + train/ActionL1Loss=0.1133 + throughput/total_tokens=250,368,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1305/500000] + train/ActionL1Loss=0.1359 + throughput/total_tokens=250,560,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1306/500000] + train/ActionL1Loss=0.1425 + throughput/total_tokens=250,752,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1307/500000] + train/ActionL1Loss=0.1427 + throughput/total_tokens=250,944,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1308/500000] + train/ActionL1Loss=0.1258 + throughput/total_tokens=251,136,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1309/500000] + train/ActionL1Loss=0.1393 + throughput/total_tokens=251,328,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1310/500000] + train/ActionL1Loss=0.1246 + throughput/total_tokens=251,520,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=1311/500000] + train/ActionL1Loss=0.1249 + throughput/total_tokens=251,712,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1312/500000] + train/ActionL1Loss=0.1281 + throughput/total_tokens=251,904,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1313/500000] + train/ActionL1Loss=0.1425 + throughput/total_tokens=252,096,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1314/500000] + train/ActionL1Loss=0.1314 + throughput/total_tokens=252,288,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1315/500000] + train/ActionL1Loss=0.1308 + throughput/total_tokens=252,480,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1316/500000] + train/ActionL1Loss=0.1361 + throughput/total_tokens=252,672,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1317/500000] + train/ActionL1Loss=0.1119 + throughput/total_tokens=252,864,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1318/500000] + train/ActionL1Loss=0.1155 + throughput/total_tokens=253,056,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1319/500000] + train/ActionL1Loss=0.1346 + throughput/total_tokens=253,248,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1320/500000] + optim/total_grad_norm=7.233 + train/ActionL1Loss=0.1375 + throughput/total_tokens=253,440,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1321/500000] + train/ActionL1Loss=0.1251 + throughput/total_tokens=253,632,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1322/500000] + train/ActionL1Loss=0.1199 + throughput/total_tokens=253,824,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1323/500000] + train/ActionL1Loss=0.1376 + throughput/total_tokens=254,016,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1324/500000] + train/ActionL1Loss=0.1478 + throughput/total_tokens=254,208,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1325/500000] + train/ActionL1Loss=0.1874 + throughput/total_tokens=254,400,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1326/500000] + train/ActionL1Loss=0.1538 + throughput/total_tokens=254,592,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1327/500000] + train/ActionL1Loss=0.1445 + throughput/total_tokens=254,784,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1328/500000] + train/ActionL1Loss=0.1431 + throughput/total_tokens=254,976,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1329/500000] + train/ActionL1Loss=0.1451 + throughput/total_tokens=255,168,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1330/500000] + train/ActionL1Loss=0.1582 + throughput/total_tokens=255,360,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1331/500000] + train/ActionL1Loss=0.1758 + throughput/total_tokens=255,552,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1332/500000] + train/ActionL1Loss=0.1479 + throughput/total_tokens=255,744,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1333/500000] + train/ActionL1Loss=0.1345 + throughput/total_tokens=255,936,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1334/500000] + train/ActionL1Loss=0.1419 + throughput/total_tokens=256,128,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1335/500000] + train/ActionL1Loss=0.1356 + throughput/total_tokens=256,320,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1336/500000] + train/ActionL1Loss=0.1475 + throughput/total_tokens=256,512,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1337/500000] + train/ActionL1Loss=0.1760 + throughput/total_tokens=256,704,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1338/500000] + train/ActionL1Loss=0.1463 + throughput/total_tokens=256,896,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1339/500000] + train/ActionL1Loss=0.1332 + throughput/total_tokens=257,088,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1340/500000] + optim/total_grad_norm=7.615 + train/ActionL1Loss=0.1370 + throughput/total_tokens=257,280,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1341/500000] + train/ActionL1Loss=0.1413 + throughput/total_tokens=257,472,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1342/500000] + train/ActionL1Loss=0.1650 + throughput/total_tokens=257,664,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1343/500000] + train/ActionL1Loss=0.1446 + throughput/total_tokens=257,856,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1344/500000] + train/ActionL1Loss=0.1375 + throughput/total_tokens=258,048,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1345/500000] + train/ActionL1Loss=0.1129 + throughput/total_tokens=258,240,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1346/500000] + train/ActionL1Loss=0.1465 + throughput/total_tokens=258,432,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1347/500000] + train/ActionL1Loss=0.1300 + throughput/total_tokens=258,624,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1348/500000] + train/ActionL1Loss=0.1335 + throughput/total_tokens=258,816,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1349/500000] + train/ActionL1Loss=0.1213 + throughput/total_tokens=259,008,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1350/500000] + train/ActionL1Loss=0.1201 + throughput/total_tokens=259,200,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1351/500000] + train/ActionL1Loss=0.1236 + throughput/total_tokens=259,392,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1352/500000] + train/ActionL1Loss=0.1600 + throughput/total_tokens=259,584,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1353/500000] + train/ActionL1Loss=0.1190 + throughput/total_tokens=259,776,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1354/500000] + train/ActionL1Loss=0.1382 + throughput/total_tokens=259,968,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1355/500000] + train/ActionL1Loss=0.1104 + throughput/total_tokens=260,160,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1356/500000] + train/ActionL1Loss=0.1086 + throughput/total_tokens=260,352,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1357/500000] + train/ActionL1Loss=0.1223 + throughput/total_tokens=260,544,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1358/500000] + train/ActionL1Loss=0.1110 + throughput/total_tokens=260,736,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1359/500000] + train/ActionL1Loss=0.1124 + throughput/total_tokens=260,928,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1360/500000] + optim/total_grad_norm=6.653 + train/ActionL1Loss=0.1322 + throughput/total_tokens=261,120,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=1361/500000] + train/ActionL1Loss=0.1203 + throughput/total_tokens=261,312,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1362/500000] + train/ActionL1Loss=0.1239 + throughput/total_tokens=261,504,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1363/500000] + train/ActionL1Loss=0.1267 + throughput/total_tokens=261,696,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1364/500000] + train/ActionL1Loss=0.1272 + throughput/total_tokens=261,888,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1365/500000] + train/ActionL1Loss=0.1025 + throughput/total_tokens=262,080,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1366/500000] + train/ActionL1Loss=0.1544 + throughput/total_tokens=262,272,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1367/500000] + train/ActionL1Loss=0.1352 + throughput/total_tokens=262,464,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1368/500000] + train/ActionL1Loss=0.1276 + throughput/total_tokens=262,656,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1369/500000] + train/ActionL1Loss=0.1086 + throughput/total_tokens=262,848,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1370/500000] + train/ActionL1Loss=0.1242 + throughput/total_tokens=263,040,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1371/500000] + train/ActionL1Loss=0.1276 + throughput/total_tokens=263,232,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1372/500000] + train/ActionL1Loss=0.1497 + throughput/total_tokens=263,424,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1373/500000] + train/ActionL1Loss=0.1293 + throughput/total_tokens=263,616,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1374/500000] + train/ActionL1Loss=0.1309 + throughput/total_tokens=263,808,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1375/500000] + train/ActionL1Loss=0.1538 + throughput/total_tokens=264,000,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1376/500000] + train/ActionL1Loss=0.1135 + throughput/total_tokens=264,192,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1377/500000] + train/ActionL1Loss=0.1442 + throughput/total_tokens=264,384,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1378/500000] + train/ActionL1Loss=0.1341 + throughput/total_tokens=264,576,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1379/500000] + train/ActionL1Loss=0.1256 + throughput/total_tokens=264,768,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1380/500000] + optim/total_grad_norm=7.195 + train/ActionL1Loss=0.1380 + throughput/total_tokens=264,960,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1381/500000] + train/ActionL1Loss=0.1316 + throughput/total_tokens=265,152,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1382/500000] + train/ActionL1Loss=0.1433 + throughput/total_tokens=265,344,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1383/500000] + train/ActionL1Loss=0.1162 + throughput/total_tokens=265,536,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1384/500000] + train/ActionL1Loss=0.1250 + throughput/total_tokens=265,728,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1385/500000] + train/ActionL1Loss=0.1152 + throughput/total_tokens=265,920,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1386/500000] + train/ActionL1Loss=0.1170 + throughput/total_tokens=266,112,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1387/500000] + train/ActionL1Loss=0.1304 + throughput/total_tokens=266,304,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1388/500000] + train/ActionL1Loss=0.1353 + throughput/total_tokens=266,496,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1389/500000] + train/ActionL1Loss=0.1452 + throughput/total_tokens=266,688,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1390/500000] + train/ActionL1Loss=0.1180 + throughput/total_tokens=266,880,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1391/500000] + train/ActionL1Loss=0.1324 + throughput/total_tokens=267,072,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1392/500000] + train/ActionL1Loss=0.1401 + throughput/total_tokens=267,264,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1393/500000] + train/ActionL1Loss=0.1613 + throughput/total_tokens=267,456,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1394/500000] + train/ActionL1Loss=0.1478 + throughput/total_tokens=267,648,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1395/500000] + train/ActionL1Loss=0.1490 + throughput/total_tokens=267,840,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1396/500000] + train/ActionL1Loss=0.1517 + throughput/total_tokens=268,032,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1397/500000] + train/ActionL1Loss=0.1322 + throughput/total_tokens=268,224,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1398/500000] + train/ActionL1Loss=0.1342 + throughput/total_tokens=268,416,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1399/500000] + train/ActionL1Loss=0.1569 + throughput/total_tokens=268,608,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1400/500000] + optim/total_grad_norm=8.248 + train/ActionL1Loss=0.1506 + throughput/total_tokens=268,800,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=1401/500000] + train/ActionL1Loss=0.1506 + throughput/total_tokens=268,992,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1402/500000] + train/ActionL1Loss=0.1223 + throughput/total_tokens=269,184,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1403/500000] + train/ActionL1Loss=0.1229 + throughput/total_tokens=269,376,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1404/500000] + train/ActionL1Loss=0.1155 + throughput/total_tokens=269,568,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1405/500000] + train/ActionL1Loss=0.1210 + throughput/total_tokens=269,760,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1406/500000] + train/ActionL1Loss=0.1142 + throughput/total_tokens=269,952,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1407/500000] + train/ActionL1Loss=0.1230 + throughput/total_tokens=270,144,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1408/500000] + train/ActionL1Loss=0.1266 + throughput/total_tokens=270,336,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1409/500000] + train/ActionL1Loss=0.1407 + throughput/total_tokens=270,528,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1410/500000] + train/ActionL1Loss=0.1410 + throughput/total_tokens=270,720,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=1411/500000] + train/ActionL1Loss=0.1475 + throughput/total_tokens=270,912,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1412/500000] + train/ActionL1Loss=0.1205 + throughput/total_tokens=271,104,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1413/500000] + train/ActionL1Loss=0.1080 + throughput/total_tokens=271,296,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1414/500000] + train/ActionL1Loss=0.1421 + throughput/total_tokens=271,488,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1415/500000] + train/ActionL1Loss=0.1344 + throughput/total_tokens=271,680,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1416/500000] + train/ActionL1Loss=0.1256 + throughput/total_tokens=271,872,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1417/500000] + train/ActionL1Loss=0.1341 + throughput/total_tokens=272,064,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1418/500000] + train/ActionL1Loss=0.1571 + throughput/total_tokens=272,256,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1419/500000] + train/ActionL1Loss=0.1310 + throughput/total_tokens=272,448,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1420/500000] + optim/total_grad_norm=8.794 + train/ActionL1Loss=0.1353 + throughput/total_tokens=272,640,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1421/500000] + train/ActionL1Loss=0.1305 + throughput/total_tokens=272,832,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1422/500000] + train/ActionL1Loss=0.1373 + throughput/total_tokens=273,024,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1423/500000] + train/ActionL1Loss=0.1286 + throughput/total_tokens=273,216,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1424/500000] + train/ActionL1Loss=0.1239 + throughput/total_tokens=273,408,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1425/500000] + train/ActionL1Loss=0.1555 + throughput/total_tokens=273,600,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1426/500000] + train/ActionL1Loss=0.1343 + throughput/total_tokens=273,792,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1427/500000] + train/ActionL1Loss=0.1250 + throughput/total_tokens=273,984,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1428/500000] + train/ActionL1Loss=0.1308 + throughput/total_tokens=274,176,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1429/500000] + train/ActionL1Loss=0.1693 + throughput/total_tokens=274,368,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1430/500000] + train/ActionL1Loss=0.1576 + throughput/total_tokens=274,560,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1431/500000] + train/ActionL1Loss=0.1274 + throughput/total_tokens=274,752,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1432/500000] + train/ActionL1Loss=0.1520 + throughput/total_tokens=274,944,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1433/500000] + train/ActionL1Loss=0.1565 + throughput/total_tokens=275,136,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1434/500000] + train/ActionL1Loss=0.1439 + throughput/total_tokens=275,328,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1435/500000] + train/ActionL1Loss=0.1450 + throughput/total_tokens=275,520,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1436/500000] + train/ActionL1Loss=0.1111 + throughput/total_tokens=275,712,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1437/500000] + train/ActionL1Loss=0.1274 + throughput/total_tokens=275,904,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1438/500000] + train/ActionL1Loss=0.1375 + throughput/total_tokens=276,096,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1439/500000] + train/ActionL1Loss=0.1310 + throughput/total_tokens=276,288,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1440/500000] + optim/total_grad_norm=7.740 + train/ActionL1Loss=0.1322 + throughput/total_tokens=276,480,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1441/500000] + train/ActionL1Loss=0.1491 + throughput/total_tokens=276,672,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1442/500000] + train/ActionL1Loss=0.1312 + throughput/total_tokens=276,864,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1443/500000] + train/ActionL1Loss=0.1204 + throughput/total_tokens=277,056,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1444/500000] + train/ActionL1Loss=0.1348 + throughput/total_tokens=277,248,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1445/500000] + train/ActionL1Loss=0.1358 + throughput/total_tokens=277,440,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1446/500000] + train/ActionL1Loss=0.1351 + throughput/total_tokens=277,632,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1447/500000] + train/ActionL1Loss=0.1472 + throughput/total_tokens=277,824,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1448/500000] + train/ActionL1Loss=0.1221 + throughput/total_tokens=278,016,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1449/500000] + train/ActionL1Loss=0.1403 + throughput/total_tokens=278,208,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1450/500000] + train/ActionL1Loss=0.1352 + throughput/total_tokens=278,400,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1451/500000] + train/ActionL1Loss=0.1163 + throughput/total_tokens=278,592,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1452/500000] + train/ActionL1Loss=0.1503 + throughput/total_tokens=278,784,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1453/500000] + train/ActionL1Loss=0.1328 + throughput/total_tokens=278,976,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1454/500000] + train/ActionL1Loss=0.1385 + throughput/total_tokens=279,168,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1455/500000] + train/ActionL1Loss=0.1417 + throughput/total_tokens=279,360,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1456/500000] + train/ActionL1Loss=0.1407 + throughput/total_tokens=279,552,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1457/500000] + train/ActionL1Loss=0.1180 + throughput/total_tokens=279,744,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=1458/500000] + train/ActionL1Loss=0.1304 + throughput/total_tokens=279,936,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=1459/500000] + train/ActionL1Loss=0.1085 + throughput/total_tokens=280,128,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=1460/500000] + optim/total_grad_norm=7.945 + train/ActionL1Loss=0.1312 + throughput/total_tokens=280,320,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=1461/500000] + train/ActionL1Loss=0.1311 + throughput/total_tokens=280,512,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=1462/500000] + train/ActionL1Loss=0.1316 + throughput/total_tokens=280,704,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=1463/500000] + train/ActionL1Loss=0.1226 + throughput/total_tokens=280,896,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=1464/500000] + train/ActionL1Loss=0.1173 + throughput/total_tokens=281,088,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=1465/500000] + train/ActionL1Loss=0.1258 + throughput/total_tokens=281,280,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1466/500000] + train/ActionL1Loss=0.1223 + throughput/total_tokens=281,472,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1467/500000] + train/ActionL1Loss=0.1157 + throughput/total_tokens=281,664,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1468/500000] + train/ActionL1Loss=0.1444 + throughput/total_tokens=281,856,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1469/500000] + train/ActionL1Loss=0.1195 + throughput/total_tokens=282,048,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1470/500000] + train/ActionL1Loss=0.1511 + throughput/total_tokens=282,240,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1471/500000] + train/ActionL1Loss=0.1294 + throughput/total_tokens=282,432,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1472/500000] + train/ActionL1Loss=0.1206 + throughput/total_tokens=282,624,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1473/500000] + train/ActionL1Loss=0.1381 + throughput/total_tokens=282,816,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1474/500000] + train/ActionL1Loss=0.1220 + throughput/total_tokens=283,008,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1475/500000] + train/ActionL1Loss=0.1174 + throughput/total_tokens=283,200,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1476/500000] + train/ActionL1Loss=0.1187 + throughput/total_tokens=283,392,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1477/500000] + train/ActionL1Loss=0.1548 + throughput/total_tokens=283,584,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1478/500000] + train/ActionL1Loss=0.1351 + throughput/total_tokens=283,776,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1479/500000] + train/ActionL1Loss=0.1338 + throughput/total_tokens=283,968,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1480/500000] + optim/total_grad_norm=8.052 + train/ActionL1Loss=0.1208 + throughput/total_tokens=284,160,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1481/500000] + train/ActionL1Loss=0.1066 + throughput/total_tokens=284,352,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1482/500000] + train/ActionL1Loss=0.1101 + throughput/total_tokens=284,544,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1483/500000] + train/ActionL1Loss=0.1473 + throughput/total_tokens=284,736,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1484/500000] + train/ActionL1Loss=0.1344 + throughput/total_tokens=284,928,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1485/500000] + train/ActionL1Loss=0.1562 + throughput/total_tokens=285,120,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1486/500000] + train/ActionL1Loss=0.1381 + throughput/total_tokens=285,312,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1487/500000] + train/ActionL1Loss=0.1424 + throughput/total_tokens=285,504,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1488/500000] + train/ActionL1Loss=0.1360 + throughput/total_tokens=285,696,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1489/500000] + train/ActionL1Loss=0.1318 + throughput/total_tokens=285,888,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1490/500000] + train/ActionL1Loss=0.1576 + throughput/total_tokens=286,080,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1491/500000] + train/ActionL1Loss=0.1440 + throughput/total_tokens=286,272,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1492/500000] + train/ActionL1Loss=0.1497 + throughput/total_tokens=286,464,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1493/500000] + train/ActionL1Loss=0.1582 + throughput/total_tokens=286,656,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1494/500000] + train/ActionL1Loss=0.1376 + throughput/total_tokens=286,848,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1495/500000] + train/ActionL1Loss=0.1422 + throughput/total_tokens=287,040,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1496/500000] + train/ActionL1Loss=0.1451 + throughput/total_tokens=287,232,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1497/500000] + train/ActionL1Loss=0.1462 + throughput/total_tokens=287,424,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1498/500000] + train/ActionL1Loss=0.1181 + throughput/total_tokens=287,616,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1499/500000] + train/ActionL1Loss=0.1207 + throughput/total_tokens=287,808,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1500/500000] + optim/total_grad_norm=7.636 + train/ActionL1Loss=0.1383 + throughput/total_tokens=288,000,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +09/30 [22:02:43] INFO | >> Saving config... checkpoint.py:608 +09/30 [22:03:22] INFO | >> Saving model state... checkpoint.py:796 +09/30 [22:04:29] INFO | >> Saving optim state... checkpoint.py:811 +09/30 [22:05:55] INFO | >> Saving trainer state... checkpoint.py:670 + INFO | >> Saving config... checkpoint.py:608 +[step=1501/500000] + train/ActionL1Loss=0.1275 + throughput/total_tokens=288,192,000 + throughput/device/tokens_per_second=1,202 + throughput/device/batches_per_second=0.0501 +[step=1502/500000] + train/ActionL1Loss=0.1225 + throughput/total_tokens=288,384,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=1503/500000] + train/ActionL1Loss=0.1112 + throughput/total_tokens=288,576,000 + throughput/device/tokens_per_second=1,137 + throughput/device/batches_per_second=0.0474 +[step=1504/500000] + train/ActionL1Loss=0.1140 + throughput/total_tokens=288,768,000 + throughput/device/tokens_per_second=1,137 + throughput/device/batches_per_second=0.0474 +[step=1505/500000] + train/ActionL1Loss=0.1246 + throughput/total_tokens=288,960,000 + throughput/device/tokens_per_second=1,142 + throughput/device/batches_per_second=0.0476 +[step=1506/500000] + train/ActionL1Loss=0.1115 + throughput/total_tokens=289,152,000 + throughput/device/tokens_per_second=1,145 + throughput/device/batches_per_second=0.0477 +[step=1507/500000] + train/ActionL1Loss=0.1246 + throughput/total_tokens=289,344,000 + throughput/device/tokens_per_second=1,147 + throughput/device/batches_per_second=0.0478 +[step=1508/500000] + train/ActionL1Loss=0.1303 + throughput/total_tokens=289,536,000 + throughput/device/tokens_per_second=1,149 + throughput/device/batches_per_second=0.0479 +[step=1509/500000] + train/ActionL1Loss=0.1425 + throughput/total_tokens=289,728,000 + throughput/device/tokens_per_second=1,150 + throughput/device/batches_per_second=0.0480 +[step=1510/500000] + train/ActionL1Loss=0.1164 + throughput/total_tokens=289,920,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 + System/Peak GPU Memory (MB)=46,917 +[step=1511/500000] + train/ActionL1Loss=0.1368 + throughput/total_tokens=290,112,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 +[step=1512/500000] + train/ActionL1Loss=0.1081 + throughput/total_tokens=290,304,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0480 +[step=1513/500000] + train/ActionL1Loss=0.1348 + throughput/total_tokens=290,496,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=1514/500000] + train/ActionL1Loss=0.1435 + throughput/total_tokens=290,688,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=1515/500000] + train/ActionL1Loss=0.1187 + throughput/total_tokens=290,880,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=1516/500000] + train/ActionL1Loss=0.1186 + throughput/total_tokens=291,072,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=1517/500000] + train/ActionL1Loss=0.1336 + throughput/total_tokens=291,264,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0481 +[step=1518/500000] + train/ActionL1Loss=0.1360 + throughput/total_tokens=291,456,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0481 +[step=1519/500000] + train/ActionL1Loss=0.1327 + throughput/total_tokens=291,648,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0481 +[step=1520/500000] + optim/total_grad_norm=6.378 + train/ActionL1Loss=0.1259 + throughput/total_tokens=291,840,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0481 + System/Peak GPU Memory (MB)=46,917 +[step=1521/500000] + train/ActionL1Loss=0.1149 + throughput/total_tokens=292,032,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0481 +[step=1522/500000] + train/ActionL1Loss=0.1177 + throughput/total_tokens=292,224,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=1523/500000] + train/ActionL1Loss=0.1328 + throughput/total_tokens=292,416,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1524/500000] + train/ActionL1Loss=0.1238 + throughput/total_tokens=292,608,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1525/500000] + train/ActionL1Loss=0.1023 + throughput/total_tokens=292,800,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1526/500000] + train/ActionL1Loss=0.1133 + throughput/total_tokens=292,992,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1527/500000] + train/ActionL1Loss=0.1385 + throughput/total_tokens=293,184,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1528/500000] + train/ActionL1Loss=0.1301 + throughput/total_tokens=293,376,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1529/500000] + train/ActionL1Loss=0.1440 + throughput/total_tokens=293,568,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1530/500000] + train/ActionL1Loss=0.1258 + throughput/total_tokens=293,760,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1531/500000] + train/ActionL1Loss=0.1357 + throughput/total_tokens=293,952,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1532/500000] + train/ActionL1Loss=0.1379 + throughput/total_tokens=294,144,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1533/500000] + train/ActionL1Loss=0.1625 + throughput/total_tokens=294,336,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1534/500000] + train/ActionL1Loss=0.1576 + throughput/total_tokens=294,528,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1535/500000] + train/ActionL1Loss=0.1402 + throughput/total_tokens=294,720,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1536/500000] + train/ActionL1Loss=0.1493 + throughput/total_tokens=294,912,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1537/500000] + train/ActionL1Loss=0.1385 + throughput/total_tokens=295,104,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1538/500000] + train/ActionL1Loss=0.1422 + throughput/total_tokens=295,296,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1539/500000] + train/ActionL1Loss=0.1394 + throughput/total_tokens=295,488,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1540/500000] + optim/total_grad_norm=7.708 + train/ActionL1Loss=0.1152 + throughput/total_tokens=295,680,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1541/500000] + train/ActionL1Loss=0.1606 + throughput/total_tokens=295,872,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1542/500000] + train/ActionL1Loss=0.1394 + throughput/total_tokens=296,064,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1543/500000] + train/ActionL1Loss=0.1365 + throughput/total_tokens=296,256,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1544/500000] + train/ActionL1Loss=0.1441 + throughput/total_tokens=296,448,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1545/500000] + train/ActionL1Loss=0.1407 + throughput/total_tokens=296,640,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1546/500000] + train/ActionL1Loss=0.1312 + throughput/total_tokens=296,832,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1547/500000] + train/ActionL1Loss=0.1302 + throughput/total_tokens=297,024,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1548/500000] + train/ActionL1Loss=0.1332 + throughput/total_tokens=297,216,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1549/500000] + train/ActionL1Loss=0.1054 + throughput/total_tokens=297,408,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1550/500000] + train/ActionL1Loss=0.1300 + throughput/total_tokens=297,600,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1551/500000] + train/ActionL1Loss=0.1185 + throughput/total_tokens=297,792,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1552/500000] + train/ActionL1Loss=0.1445 + throughput/total_tokens=297,984,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1553/500000] + train/ActionL1Loss=0.1412 + throughput/total_tokens=298,176,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1554/500000] + train/ActionL1Loss=0.1318 + throughput/total_tokens=298,368,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1555/500000] + train/ActionL1Loss=0.1348 + throughput/total_tokens=298,560,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1556/500000] + train/ActionL1Loss=0.1449 + throughput/total_tokens=298,752,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1557/500000] + train/ActionL1Loss=0.1372 + throughput/total_tokens=298,944,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1558/500000] + train/ActionL1Loss=0.1174 + throughput/total_tokens=299,136,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1559/500000] + train/ActionL1Loss=0.1312 + throughput/total_tokens=299,328,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1560/500000] + optim/total_grad_norm=9.160 + train/ActionL1Loss=0.1198 + throughput/total_tokens=299,520,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=1561/500000] + train/ActionL1Loss=0.1346 + throughput/total_tokens=299,712,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1562/500000] + train/ActionL1Loss=0.1180 + throughput/total_tokens=299,904,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1563/500000] + train/ActionL1Loss=0.1159 + throughput/total_tokens=300,096,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1564/500000] + train/ActionL1Loss=0.1128 + throughput/total_tokens=300,288,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1565/500000] + train/ActionL1Loss=0.1182 + throughput/total_tokens=300,480,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1566/500000] + train/ActionL1Loss=0.1590 + throughput/total_tokens=300,672,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1567/500000] + train/ActionL1Loss=0.1340 + throughput/total_tokens=300,864,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1568/500000] + train/ActionL1Loss=0.1243 + throughput/total_tokens=301,056,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1569/500000] + train/ActionL1Loss=0.1207 + throughput/total_tokens=301,248,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1570/500000] + train/ActionL1Loss=0.1243 + throughput/total_tokens=301,440,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1571/500000] + train/ActionL1Loss=0.1343 + throughput/total_tokens=301,632,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1572/500000] + train/ActionL1Loss=0.1457 + throughput/total_tokens=301,824,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1573/500000] + train/ActionL1Loss=0.1323 + throughput/total_tokens=302,016,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1574/500000] + train/ActionL1Loss=0.1356 + throughput/total_tokens=302,208,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1575/500000] + train/ActionL1Loss=0.1278 + throughput/total_tokens=302,400,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1576/500000] + train/ActionL1Loss=0.1389 + throughput/total_tokens=302,592,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1577/500000] + train/ActionL1Loss=0.1212 + throughput/total_tokens=302,784,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1578/500000] + train/ActionL1Loss=0.1527 + throughput/total_tokens=302,976,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1579/500000] + train/ActionL1Loss=0.1575 + throughput/total_tokens=303,168,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1580/500000] + optim/total_grad_norm=7.087 + train/ActionL1Loss=0.1428 + throughput/total_tokens=303,360,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1581/500000] + train/ActionL1Loss=0.1164 + throughput/total_tokens=303,552,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1582/500000] + train/ActionL1Loss=0.1548 + throughput/total_tokens=303,744,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1583/500000] + train/ActionL1Loss=0.1475 + throughput/total_tokens=303,936,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1584/500000] + train/ActionL1Loss=0.1291 + throughput/total_tokens=304,128,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1585/500000] + train/ActionL1Loss=0.1186 + throughput/total_tokens=304,320,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1586/500000] + train/ActionL1Loss=0.1250 + throughput/total_tokens=304,512,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1587/500000] + train/ActionL1Loss=0.1141 + throughput/total_tokens=304,704,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1588/500000] + train/ActionL1Loss=0.1434 + throughput/total_tokens=304,896,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1589/500000] + train/ActionL1Loss=0.1331 + throughput/total_tokens=305,088,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1590/500000] + train/ActionL1Loss=0.1447 + throughput/total_tokens=305,280,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1591/500000] + train/ActionL1Loss=0.1083 + throughput/total_tokens=305,472,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1592/500000] + train/ActionL1Loss=0.1345 + throughput/total_tokens=305,664,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1593/500000] + train/ActionL1Loss=0.1257 + throughput/total_tokens=305,856,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1594/500000] + train/ActionL1Loss=0.1289 + throughput/total_tokens=306,048,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1595/500000] + train/ActionL1Loss=0.1292 + throughput/total_tokens=306,240,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1596/500000] + train/ActionL1Loss=0.1333 + throughput/total_tokens=306,432,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1597/500000] + train/ActionL1Loss=0.1198 + throughput/total_tokens=306,624,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1598/500000] + train/ActionL1Loss=0.1436 + throughput/total_tokens=306,816,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1599/500000] + train/ActionL1Loss=0.1276 + throughput/total_tokens=307,008,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1600/500000] + optim/total_grad_norm=7.182 + train/ActionL1Loss=0.1245 + throughput/total_tokens=307,200,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1601/500000] + train/ActionL1Loss=0.1173 + throughput/total_tokens=307,392,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1602/500000] + train/ActionL1Loss=0.1426 + throughput/total_tokens=307,584,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1603/500000] + train/ActionL1Loss=0.1151 + throughput/total_tokens=307,776,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1604/500000] + train/ActionL1Loss=0.1190 + throughput/total_tokens=307,968,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1605/500000] + train/ActionL1Loss=0.1216 + throughput/total_tokens=308,160,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1606/500000] + train/ActionL1Loss=0.1111 + throughput/total_tokens=308,352,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1607/500000] + train/ActionL1Loss=0.1425 + throughput/total_tokens=308,544,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1608/500000] + train/ActionL1Loss=0.1232 + throughput/total_tokens=308,736,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1609/500000] + train/ActionL1Loss=0.1208 + throughput/total_tokens=308,928,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1610/500000] + train/ActionL1Loss=0.1207 + throughput/total_tokens=309,120,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=1611/500000] + train/ActionL1Loss=0.1188 + throughput/total_tokens=309,312,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1612/500000] + train/ActionL1Loss=0.1502 + throughput/total_tokens=309,504,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1613/500000] + train/ActionL1Loss=0.1303 + throughput/total_tokens=309,696,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1614/500000] + train/ActionL1Loss=0.1202 + throughput/total_tokens=309,888,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1615/500000] + train/ActionL1Loss=0.0989 + throughput/total_tokens=310,080,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1616/500000] + train/ActionL1Loss=0.1282 + throughput/total_tokens=310,272,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1617/500000] + train/ActionL1Loss=0.1178 + throughput/total_tokens=310,464,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1618/500000] + train/ActionL1Loss=0.1190 + throughput/total_tokens=310,656,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1619/500000] + train/ActionL1Loss=0.1342 + throughput/total_tokens=310,848,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1620/500000] + optim/total_grad_norm=8.399 + train/ActionL1Loss=0.1457 + throughput/total_tokens=311,040,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1621/500000] + train/ActionL1Loss=0.1322 + throughput/total_tokens=311,232,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1622/500000] + train/ActionL1Loss=0.1429 + throughput/total_tokens=311,424,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1623/500000] + train/ActionL1Loss=0.1247 + throughput/total_tokens=311,616,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1624/500000] + train/ActionL1Loss=0.1304 + throughput/total_tokens=311,808,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1625/500000] + train/ActionL1Loss=0.1297 + throughput/total_tokens=312,000,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1626/500000] + train/ActionL1Loss=0.1488 + throughput/total_tokens=312,192,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1627/500000] + train/ActionL1Loss=0.1551 + throughput/total_tokens=312,384,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1628/500000] + train/ActionL1Loss=0.1441 + throughput/total_tokens=312,576,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1629/500000] + train/ActionL1Loss=0.1423 + throughput/total_tokens=312,768,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1630/500000] + train/ActionL1Loss=0.1343 + throughput/total_tokens=312,960,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1631/500000] + train/ActionL1Loss=0.1471 + throughput/total_tokens=313,152,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1632/500000] + train/ActionL1Loss=0.1151 + throughput/total_tokens=313,344,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1633/500000] + train/ActionL1Loss=0.1617 + throughput/total_tokens=313,536,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1634/500000] + train/ActionL1Loss=0.1292 + throughput/total_tokens=313,728,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1635/500000] + train/ActionL1Loss=0.1429 + throughput/total_tokens=313,920,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1636/500000] + train/ActionL1Loss=0.1340 + throughput/total_tokens=314,112,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1637/500000] + train/ActionL1Loss=0.1241 + throughput/total_tokens=314,304,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1638/500000] + train/ActionL1Loss=0.1405 + throughput/total_tokens=314,496,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1639/500000] + train/ActionL1Loss=0.1406 + throughput/total_tokens=314,688,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1640/500000] + optim/total_grad_norm=7.664 + train/ActionL1Loss=0.1350 + throughput/total_tokens=314,880,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1641/500000] + train/ActionL1Loss=0.1454 + throughput/total_tokens=315,072,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1642/500000] + train/ActionL1Loss=0.1199 + throughput/total_tokens=315,264,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1643/500000] + train/ActionL1Loss=0.1347 + throughput/total_tokens=315,456,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1644/500000] + train/ActionL1Loss=0.1134 + throughput/total_tokens=315,648,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1645/500000] + train/ActionL1Loss=0.1228 + throughput/total_tokens=315,840,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1646/500000] + train/ActionL1Loss=0.1242 + throughput/total_tokens=316,032,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1647/500000] + train/ActionL1Loss=0.1443 + throughput/total_tokens=316,224,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1648/500000] + train/ActionL1Loss=0.1228 + throughput/total_tokens=316,416,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1649/500000] + train/ActionL1Loss=0.1387 + throughput/total_tokens=316,608,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1650/500000] + train/ActionL1Loss=0.1187 + throughput/total_tokens=316,800,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1651/500000] + train/ActionL1Loss=0.1230 + throughput/total_tokens=316,992,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1652/500000] + train/ActionL1Loss=0.1342 + throughput/total_tokens=317,184,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1653/500000] + train/ActionL1Loss=0.1217 + throughput/total_tokens=317,376,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1654/500000] + train/ActionL1Loss=0.1194 + throughput/total_tokens=317,568,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1655/500000] + train/ActionL1Loss=0.1397 + throughput/total_tokens=317,760,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1656/500000] + train/ActionL1Loss=0.1094 + throughput/total_tokens=317,952,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1657/500000] + train/ActionL1Loss=0.1173 + throughput/total_tokens=318,144,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1658/500000] + train/ActionL1Loss=0.1162 + throughput/total_tokens=318,336,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1659/500000] + train/ActionL1Loss=0.1228 + throughput/total_tokens=318,528,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1660/500000] + optim/total_grad_norm=7.771 + train/ActionL1Loss=0.1096 + throughput/total_tokens=318,720,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=1661/500000] + train/ActionL1Loss=0.1266 + throughput/total_tokens=318,912,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1662/500000] + train/ActionL1Loss=0.1497 + throughput/total_tokens=319,104,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1663/500000] + train/ActionL1Loss=0.1125 + throughput/total_tokens=319,296,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1664/500000] + train/ActionL1Loss=0.1064 + throughput/total_tokens=319,488,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1665/500000] + train/ActionL1Loss=0.1133 + throughput/total_tokens=319,680,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1666/500000] + train/ActionL1Loss=0.1233 + throughput/total_tokens=319,872,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1667/500000] + train/ActionL1Loss=0.1262 + throughput/total_tokens=320,064,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1668/500000] + train/ActionL1Loss=0.1235 + throughput/total_tokens=320,256,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1669/500000] + train/ActionL1Loss=0.1206 + throughput/total_tokens=320,448,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1670/500000] + train/ActionL1Loss=0.1331 + throughput/total_tokens=320,640,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1671/500000] + train/ActionL1Loss=0.1418 + throughput/total_tokens=320,832,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1672/500000] + train/ActionL1Loss=0.1077 + throughput/total_tokens=321,024,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1673/500000] + train/ActionL1Loss=0.1443 + throughput/total_tokens=321,216,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1674/500000] + train/ActionL1Loss=0.1393 + throughput/total_tokens=321,408,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1675/500000] + train/ActionL1Loss=0.1443 + throughput/total_tokens=321,600,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1676/500000] + train/ActionL1Loss=0.1182 + throughput/total_tokens=321,792,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1677/500000] + train/ActionL1Loss=0.1537 + throughput/total_tokens=321,984,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1678/500000] + train/ActionL1Loss=0.1378 + throughput/total_tokens=322,176,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1679/500000] + train/ActionL1Loss=0.0973 + throughput/total_tokens=322,368,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1680/500000] + optim/total_grad_norm=6.686 + train/ActionL1Loss=0.1159 + throughput/total_tokens=322,560,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1681/500000] + train/ActionL1Loss=0.1433 + throughput/total_tokens=322,752,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1682/500000] + train/ActionL1Loss=0.1179 + throughput/total_tokens=322,944,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1683/500000] + train/ActionL1Loss=0.1247 + throughput/total_tokens=323,136,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1684/500000] + train/ActionL1Loss=0.1177 + throughput/total_tokens=323,328,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1685/500000] + train/ActionL1Loss=0.1365 + throughput/total_tokens=323,520,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1686/500000] + train/ActionL1Loss=0.1274 + throughput/total_tokens=323,712,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1687/500000] + train/ActionL1Loss=0.1358 + throughput/total_tokens=323,904,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1688/500000] + train/ActionL1Loss=0.1273 + throughput/total_tokens=324,096,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1689/500000] + train/ActionL1Loss=0.1295 + throughput/total_tokens=324,288,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1690/500000] + train/ActionL1Loss=0.1281 + throughput/total_tokens=324,480,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1691/500000] + train/ActionL1Loss=0.1116 + throughput/total_tokens=324,672,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1692/500000] + train/ActionL1Loss=0.1566 + throughput/total_tokens=324,864,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1693/500000] + train/ActionL1Loss=0.1226 + throughput/total_tokens=325,056,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1694/500000] + train/ActionL1Loss=0.1264 + throughput/total_tokens=325,248,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1695/500000] + train/ActionL1Loss=0.1124 + throughput/total_tokens=325,440,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1696/500000] + train/ActionL1Loss=0.1140 + throughput/total_tokens=325,632,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1697/500000] + train/ActionL1Loss=0.1233 + throughput/total_tokens=325,824,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1698/500000] + train/ActionL1Loss=0.1328 + throughput/total_tokens=326,016,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1699/500000] + train/ActionL1Loss=0.1218 + throughput/total_tokens=326,208,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1700/500000] + optim/total_grad_norm=6.495 + train/ActionL1Loss=0.1355 + throughput/total_tokens=326,400,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1701/500000] + train/ActionL1Loss=0.1130 + throughput/total_tokens=326,592,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1702/500000] + train/ActionL1Loss=0.1392 + throughput/total_tokens=326,784,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1703/500000] + train/ActionL1Loss=0.1317 + throughput/total_tokens=326,976,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1704/500000] + train/ActionL1Loss=0.1250 + throughput/total_tokens=327,168,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1705/500000] + train/ActionL1Loss=0.1324 + throughput/total_tokens=327,360,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1706/500000] + train/ActionL1Loss=0.1277 + throughput/total_tokens=327,552,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1707/500000] + train/ActionL1Loss=0.1442 + throughput/total_tokens=327,744,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1708/500000] + train/ActionL1Loss=0.1314 + throughput/total_tokens=327,936,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1709/500000] + train/ActionL1Loss=0.1377 + throughput/total_tokens=328,128,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1710/500000] + train/ActionL1Loss=0.1163 + throughput/total_tokens=328,320,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=1711/500000] + train/ActionL1Loss=0.1341 + throughput/total_tokens=328,512,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1712/500000] + train/ActionL1Loss=0.1191 + throughput/total_tokens=328,704,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1713/500000] + train/ActionL1Loss=0.1181 + throughput/total_tokens=328,896,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1714/500000] + train/ActionL1Loss=0.1404 + throughput/total_tokens=329,088,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1715/500000] + train/ActionL1Loss=0.1318 + throughput/total_tokens=329,280,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1716/500000] + train/ActionL1Loss=0.1226 + throughput/total_tokens=329,472,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1717/500000] + train/ActionL1Loss=0.1245 + throughput/total_tokens=329,664,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1718/500000] + train/ActionL1Loss=0.1339 + throughput/total_tokens=329,856,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1719/500000] + train/ActionL1Loss=0.1253 + throughput/total_tokens=330,048,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1720/500000] + optim/total_grad_norm=6.059 + train/ActionL1Loss=0.1084 + throughput/total_tokens=330,240,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1721/500000] + train/ActionL1Loss=0.1316 + throughput/total_tokens=330,432,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1722/500000] + train/ActionL1Loss=0.1212 + throughput/total_tokens=330,624,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1723/500000] + train/ActionL1Loss=0.1053 + throughput/total_tokens=330,816,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1724/500000] + train/ActionL1Loss=0.1140 + throughput/total_tokens=331,008,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1725/500000] + train/ActionL1Loss=0.1203 + throughput/total_tokens=331,200,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1726/500000] + train/ActionL1Loss=0.1227 + throughput/total_tokens=331,392,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1727/500000] + train/ActionL1Loss=0.1202 + throughput/total_tokens=331,584,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1728/500000] + train/ActionL1Loss=0.1108 + throughput/total_tokens=331,776,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1729/500000] + train/ActionL1Loss=0.1231 + throughput/total_tokens=331,968,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1730/500000] + train/ActionL1Loss=0.1167 + throughput/total_tokens=332,160,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1731/500000] + train/ActionL1Loss=0.1076 + throughput/total_tokens=332,352,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1732/500000] + train/ActionL1Loss=0.1055 + throughput/total_tokens=332,544,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1733/500000] + train/ActionL1Loss=0.1218 + throughput/total_tokens=332,736,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1734/500000] + train/ActionL1Loss=0.1024 + throughput/total_tokens=332,928,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1735/500000] + train/ActionL1Loss=0.1349 + throughput/total_tokens=333,120,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1736/500000] + train/ActionL1Loss=0.1193 + throughput/total_tokens=333,312,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1737/500000] + train/ActionL1Loss=0.1332 + throughput/total_tokens=333,504,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1738/500000] + train/ActionL1Loss=0.1352 + throughput/total_tokens=333,696,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1739/500000] + train/ActionL1Loss=0.1285 + throughput/total_tokens=333,888,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1740/500000] + optim/total_grad_norm=7.816 + train/ActionL1Loss=0.1246 + throughput/total_tokens=334,080,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1741/500000] + train/ActionL1Loss=0.1276 + throughput/total_tokens=334,272,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1742/500000] + train/ActionL1Loss=0.1122 + throughput/total_tokens=334,464,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1743/500000] + train/ActionL1Loss=0.1066 + throughput/total_tokens=334,656,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1744/500000] + train/ActionL1Loss=0.1154 + throughput/total_tokens=334,848,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1745/500000] + train/ActionL1Loss=0.1228 + throughput/total_tokens=335,040,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1746/500000] + train/ActionL1Loss=0.1136 + throughput/total_tokens=335,232,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1747/500000] + train/ActionL1Loss=0.1121 + throughput/total_tokens=335,424,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1748/500000] + train/ActionL1Loss=0.1299 + throughput/total_tokens=335,616,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1749/500000] + train/ActionL1Loss=0.1328 + throughput/total_tokens=335,808,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1750/500000] + train/ActionL1Loss=0.1173 + throughput/total_tokens=336,000,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1751/500000] + train/ActionL1Loss=0.1010 + throughput/total_tokens=336,192,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1752/500000] + train/ActionL1Loss=0.1000 + throughput/total_tokens=336,384,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1753/500000] + train/ActionL1Loss=0.1306 + throughput/total_tokens=336,576,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1754/500000] + train/ActionL1Loss=0.1382 + throughput/total_tokens=336,768,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1755/500000] + train/ActionL1Loss=0.1193 + throughput/total_tokens=336,960,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1756/500000] + train/ActionL1Loss=0.1240 + throughput/total_tokens=337,152,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1757/500000] + train/ActionL1Loss=0.1212 + throughput/total_tokens=337,344,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1758/500000] + train/ActionL1Loss=0.1504 + throughput/total_tokens=337,536,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1759/500000] + train/ActionL1Loss=0.1195 + throughput/total_tokens=337,728,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1760/500000] + optim/total_grad_norm=8.062 + train/ActionL1Loss=0.1374 + throughput/total_tokens=337,920,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=1761/500000] + train/ActionL1Loss=0.1258 + throughput/total_tokens=338,112,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1762/500000] + train/ActionL1Loss=0.1278 + throughput/total_tokens=338,304,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1763/500000] + train/ActionL1Loss=0.1236 + throughput/total_tokens=338,496,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1764/500000] + train/ActionL1Loss=0.1278 + throughput/total_tokens=338,688,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1765/500000] + train/ActionL1Loss=0.1362 + throughput/total_tokens=338,880,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1766/500000] + train/ActionL1Loss=0.1165 + throughput/total_tokens=339,072,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1767/500000] + train/ActionL1Loss=0.1157 + throughput/total_tokens=339,264,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1768/500000] + train/ActionL1Loss=0.1220 + throughput/total_tokens=339,456,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1769/500000] + train/ActionL1Loss=0.1346 + throughput/total_tokens=339,648,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=1770/500000] + train/ActionL1Loss=0.1043 + throughput/total_tokens=339,840,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1771/500000] + train/ActionL1Loss=0.1285 + throughput/total_tokens=340,032,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1772/500000] + train/ActionL1Loss=0.1312 + throughput/total_tokens=340,224,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1773/500000] + train/ActionL1Loss=0.1363 + throughput/total_tokens=340,416,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1774/500000] + train/ActionL1Loss=0.1284 + throughput/total_tokens=340,608,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1775/500000] + train/ActionL1Loss=0.1347 + throughput/total_tokens=340,800,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1776/500000] + train/ActionL1Loss=0.1258 + throughput/total_tokens=340,992,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1777/500000] + train/ActionL1Loss=0.1334 + throughput/total_tokens=341,184,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1778/500000] + train/ActionL1Loss=0.1274 + throughput/total_tokens=341,376,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1779/500000] + train/ActionL1Loss=0.1349 + throughput/total_tokens=341,568,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1780/500000] + optim/total_grad_norm=7.074 + train/ActionL1Loss=0.1169 + throughput/total_tokens=341,760,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1781/500000] + train/ActionL1Loss=0.1491 + throughput/total_tokens=341,952,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1782/500000] + train/ActionL1Loss=0.1368 + throughput/total_tokens=342,144,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1783/500000] + train/ActionL1Loss=0.1208 + throughput/total_tokens=342,336,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1784/500000] + train/ActionL1Loss=0.1177 + throughput/total_tokens=342,528,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1785/500000] + train/ActionL1Loss=0.1262 + throughput/total_tokens=342,720,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1786/500000] + train/ActionL1Loss=0.1297 + throughput/total_tokens=342,912,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1787/500000] + train/ActionL1Loss=0.1158 + throughput/total_tokens=343,104,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1788/500000] + train/ActionL1Loss=0.1335 + throughput/total_tokens=343,296,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1789/500000] + train/ActionL1Loss=0.1358 + throughput/total_tokens=343,488,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1790/500000] + train/ActionL1Loss=0.1350 + throughput/total_tokens=343,680,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1791/500000] + train/ActionL1Loss=0.1411 + throughput/total_tokens=343,872,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1792/500000] + train/ActionL1Loss=0.1310 + throughput/total_tokens=344,064,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1793/500000] + train/ActionL1Loss=0.1301 + throughput/total_tokens=344,256,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1794/500000] + train/ActionL1Loss=0.1295 + throughput/total_tokens=344,448,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1795/500000] + train/ActionL1Loss=0.1393 + throughput/total_tokens=344,640,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1796/500000] + train/ActionL1Loss=0.1361 + throughput/total_tokens=344,832,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=1797/500000] + train/ActionL1Loss=0.1509 + throughput/total_tokens=345,024,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=1798/500000] + train/ActionL1Loss=0.1361 + throughput/total_tokens=345,216,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1799/500000] + train/ActionL1Loss=0.1344 + throughput/total_tokens=345,408,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=1800/500000] + optim/total_grad_norm=6.884 + train/ActionL1Loss=0.1220 + throughput/total_tokens=345,600,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1801/500000] + train/ActionL1Loss=0.1274 + throughput/total_tokens=345,792,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1802/500000] + train/ActionL1Loss=0.1352 + throughput/total_tokens=345,984,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1803/500000] + train/ActionL1Loss=0.1229 + throughput/total_tokens=346,176,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1804/500000] + train/ActionL1Loss=0.1254 + throughput/total_tokens=346,368,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1805/500000] + train/ActionL1Loss=0.1270 + throughput/total_tokens=346,560,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1806/500000] + train/ActionL1Loss=0.1313 + throughput/total_tokens=346,752,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1807/500000] + train/ActionL1Loss=0.1262 + throughput/total_tokens=346,944,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1808/500000] + train/ActionL1Loss=0.1249 + throughput/total_tokens=347,136,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1809/500000] + train/ActionL1Loss=0.1243 + throughput/total_tokens=347,328,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1810/500000] + train/ActionL1Loss=0.1070 + throughput/total_tokens=347,520,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1811/500000] + train/ActionL1Loss=0.1143 + throughput/total_tokens=347,712,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1812/500000] + train/ActionL1Loss=0.1243 + throughput/total_tokens=347,904,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1813/500000] + train/ActionL1Loss=0.1261 + throughput/total_tokens=348,096,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1814/500000] + train/ActionL1Loss=0.1293 + throughput/total_tokens=348,288,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1815/500000] + train/ActionL1Loss=0.1232 + throughput/total_tokens=348,480,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1816/500000] + train/ActionL1Loss=0.1177 + throughput/total_tokens=348,672,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1817/500000] + train/ActionL1Loss=0.1379 + throughput/total_tokens=348,864,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1818/500000] + train/ActionL1Loss=0.1073 + throughput/total_tokens=349,056,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1819/500000] + train/ActionL1Loss=0.0967 + throughput/total_tokens=349,248,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1820/500000] + optim/total_grad_norm=6.608 + train/ActionL1Loss=0.1340 + throughput/total_tokens=349,440,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1821/500000] + train/ActionL1Loss=0.1264 + throughput/total_tokens=349,632,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1822/500000] + train/ActionL1Loss=0.1106 + throughput/total_tokens=349,824,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1823/500000] + train/ActionL1Loss=0.1461 + throughput/total_tokens=350,016,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1824/500000] + train/ActionL1Loss=0.1098 + throughput/total_tokens=350,208,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1825/500000] + train/ActionL1Loss=0.1303 + throughput/total_tokens=350,400,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=1826/500000] + train/ActionL1Loss=0.1287 + throughput/total_tokens=350,592,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=1827/500000] + train/ActionL1Loss=0.1205 + throughput/total_tokens=350,784,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1828/500000] + train/ActionL1Loss=0.1222 + throughput/total_tokens=350,976,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1829/500000] + train/ActionL1Loss=0.1257 + throughput/total_tokens=351,168,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1830/500000] + train/ActionL1Loss=0.1253 + throughput/total_tokens=351,360,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1831/500000] + train/ActionL1Loss=0.1301 + throughput/total_tokens=351,552,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1832/500000] + train/ActionL1Loss=0.1136 + throughput/total_tokens=351,744,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1833/500000] + train/ActionL1Loss=0.1411 + throughput/total_tokens=351,936,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1834/500000] + train/ActionL1Loss=0.1186 + throughput/total_tokens=352,128,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1835/500000] + train/ActionL1Loss=0.1363 + throughput/total_tokens=352,320,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1836/500000] + train/ActionL1Loss=0.1181 + throughput/total_tokens=352,512,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1837/500000] + train/ActionL1Loss=0.1460 + throughput/total_tokens=352,704,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1838/500000] + train/ActionL1Loss=0.1294 + throughput/total_tokens=352,896,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1839/500000] + train/ActionL1Loss=0.1030 + throughput/total_tokens=353,088,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1840/500000] + optim/total_grad_norm=7.481 + train/ActionL1Loss=0.1261 + throughput/total_tokens=353,280,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1841/500000] + train/ActionL1Loss=0.1356 + throughput/total_tokens=353,472,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1842/500000] + train/ActionL1Loss=0.1367 + throughput/total_tokens=353,664,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1843/500000] + train/ActionL1Loss=0.1547 + throughput/total_tokens=353,856,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1844/500000] + train/ActionL1Loss=0.1454 + throughput/total_tokens=354,048,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1845/500000] + train/ActionL1Loss=0.1411 + throughput/total_tokens=354,240,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1846/500000] + train/ActionL1Loss=0.1234 + throughput/total_tokens=354,432,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1847/500000] + train/ActionL1Loss=0.1215 + throughput/total_tokens=354,624,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1848/500000] + train/ActionL1Loss=0.1259 + throughput/total_tokens=354,816,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1849/500000] + train/ActionL1Loss=0.1216 + throughput/total_tokens=355,008,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1850/500000] + train/ActionL1Loss=0.1380 + throughput/total_tokens=355,200,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1851/500000] + train/ActionL1Loss=0.1497 + throughput/total_tokens=355,392,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1852/500000] + train/ActionL1Loss=0.1398 + throughput/total_tokens=355,584,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1853/500000] + train/ActionL1Loss=0.1382 + throughput/total_tokens=355,776,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1854/500000] + train/ActionL1Loss=0.1267 + throughput/total_tokens=355,968,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=1855/500000] + train/ActionL1Loss=0.1396 + throughput/total_tokens=356,160,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1856/500000] + train/ActionL1Loss=0.1118 + throughput/total_tokens=356,352,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1857/500000] + train/ActionL1Loss=0.1335 + throughput/total_tokens=356,544,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1858/500000] + train/ActionL1Loss=0.1329 + throughput/total_tokens=356,736,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1859/500000] + train/ActionL1Loss=0.1290 + throughput/total_tokens=356,928,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1860/500000] + optim/total_grad_norm=7.482 + train/ActionL1Loss=0.1414 + throughput/total_tokens=357,120,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=1861/500000] + train/ActionL1Loss=0.1384 + throughput/total_tokens=357,312,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1862/500000] + train/ActionL1Loss=0.1345 + throughput/total_tokens=357,504,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1863/500000] + train/ActionL1Loss=0.1194 + throughput/total_tokens=357,696,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1864/500000] + train/ActionL1Loss=0.1080 + throughput/total_tokens=357,888,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1865/500000] + train/ActionL1Loss=0.1301 + throughput/total_tokens=358,080,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1866/500000] + train/ActionL1Loss=0.1045 + throughput/total_tokens=358,272,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1867/500000] + train/ActionL1Loss=0.1193 + throughput/total_tokens=358,464,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1868/500000] + train/ActionL1Loss=0.1257 + throughput/total_tokens=358,656,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1869/500000] + train/ActionL1Loss=0.1228 + throughput/total_tokens=358,848,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1870/500000] + train/ActionL1Loss=0.1147 + throughput/total_tokens=359,040,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1871/500000] + train/ActionL1Loss=0.1252 + throughput/total_tokens=359,232,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1872/500000] + train/ActionL1Loss=0.1014 + throughput/total_tokens=359,424,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1873/500000] + train/ActionL1Loss=0.1130 + throughput/total_tokens=359,616,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1874/500000] + train/ActionL1Loss=0.1327 + throughput/total_tokens=359,808,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1875/500000] + train/ActionL1Loss=0.1279 + throughput/total_tokens=360,000,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1876/500000] + train/ActionL1Loss=0.1358 + throughput/total_tokens=360,192,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1877/500000] + train/ActionL1Loss=0.1457 + throughput/total_tokens=360,384,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1878/500000] + train/ActionL1Loss=0.1307 + throughput/total_tokens=360,576,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1879/500000] + train/ActionL1Loss=0.1368 + throughput/total_tokens=360,768,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1880/500000] + optim/total_grad_norm=8.312 + train/ActionL1Loss=0.1379 + throughput/total_tokens=360,960,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1881/500000] + train/ActionL1Loss=0.1503 + throughput/total_tokens=361,152,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1882/500000] + train/ActionL1Loss=0.1314 + throughput/total_tokens=361,344,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1883/500000] + train/ActionL1Loss=0.1381 + throughput/total_tokens=361,536,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1884/500000] + train/ActionL1Loss=0.1394 + throughput/total_tokens=361,728,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1885/500000] + train/ActionL1Loss=0.1157 + throughput/total_tokens=361,920,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1886/500000] + train/ActionL1Loss=0.1174 + throughput/total_tokens=362,112,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1887/500000] + train/ActionL1Loss=0.1202 + throughput/total_tokens=362,304,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1888/500000] + train/ActionL1Loss=0.1114 + throughput/total_tokens=362,496,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1889/500000] + train/ActionL1Loss=0.1199 + throughput/total_tokens=362,688,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1890/500000] + train/ActionL1Loss=0.1252 + throughput/total_tokens=362,880,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1891/500000] + train/ActionL1Loss=0.1054 + throughput/total_tokens=363,072,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1892/500000] + train/ActionL1Loss=0.1157 + throughput/total_tokens=363,264,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1893/500000] + train/ActionL1Loss=0.1078 + throughput/total_tokens=363,456,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1894/500000] + train/ActionL1Loss=0.1055 + throughput/total_tokens=363,648,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1895/500000] + train/ActionL1Loss=0.1367 + throughput/total_tokens=363,840,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1896/500000] + train/ActionL1Loss=0.1177 + throughput/total_tokens=364,032,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1897/500000] + train/ActionL1Loss=0.1111 + throughput/total_tokens=364,224,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1898/500000] + train/ActionL1Loss=0.1205 + throughput/total_tokens=364,416,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1899/500000] + train/ActionL1Loss=0.1224 + throughput/total_tokens=364,608,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1900/500000] + optim/total_grad_norm=7.113 + train/ActionL1Loss=0.1343 + throughput/total_tokens=364,800,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1901/500000] + train/ActionL1Loss=0.1142 + throughput/total_tokens=364,992,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1902/500000] + train/ActionL1Loss=0.1121 + throughput/total_tokens=365,184,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1903/500000] + train/ActionL1Loss=0.1150 + throughput/total_tokens=365,376,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1904/500000] + train/ActionL1Loss=0.1210 + throughput/total_tokens=365,568,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1905/500000] + train/ActionL1Loss=0.1109 + throughput/total_tokens=365,760,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1906/500000] + train/ActionL1Loss=0.1260 + throughput/total_tokens=365,952,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1907/500000] + train/ActionL1Loss=0.1176 + throughput/total_tokens=366,144,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1908/500000] + train/ActionL1Loss=0.1165 + throughput/total_tokens=366,336,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1909/500000] + train/ActionL1Loss=0.1252 + throughput/total_tokens=366,528,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1910/500000] + train/ActionL1Loss=0.1260 + throughput/total_tokens=366,720,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=1911/500000] + train/ActionL1Loss=0.1322 + throughput/total_tokens=366,912,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1912/500000] + train/ActionL1Loss=0.1298 + throughput/total_tokens=367,104,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1913/500000] + train/ActionL1Loss=0.1276 + throughput/total_tokens=367,296,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1914/500000] + train/ActionL1Loss=0.1183 + throughput/total_tokens=367,488,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1915/500000] + train/ActionL1Loss=0.1348 + throughput/total_tokens=367,680,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1916/500000] + train/ActionL1Loss=0.1060 + throughput/total_tokens=367,872,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1917/500000] + train/ActionL1Loss=0.1111 + throughput/total_tokens=368,064,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1918/500000] + train/ActionL1Loss=0.1161 + throughput/total_tokens=368,256,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1919/500000] + train/ActionL1Loss=0.1490 + throughput/total_tokens=368,448,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1920/500000] + optim/total_grad_norm=8.388 + train/ActionL1Loss=0.1171 + throughput/total_tokens=368,640,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1921/500000] + train/ActionL1Loss=0.1289 + throughput/total_tokens=368,832,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1922/500000] + train/ActionL1Loss=0.1358 + throughput/total_tokens=369,024,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=1923/500000] + train/ActionL1Loss=0.1328 + throughput/total_tokens=369,216,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1924/500000] + train/ActionL1Loss=0.1202 + throughput/total_tokens=369,408,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1925/500000] + train/ActionL1Loss=0.1167 + throughput/total_tokens=369,600,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=1926/500000] + train/ActionL1Loss=0.1156 + throughput/total_tokens=369,792,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=1927/500000] + train/ActionL1Loss=0.1159 + throughput/total_tokens=369,984,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1928/500000] + train/ActionL1Loss=0.1231 + throughput/total_tokens=370,176,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1929/500000] + train/ActionL1Loss=0.1123 + throughput/total_tokens=370,368,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1930/500000] + train/ActionL1Loss=0.1137 + throughput/total_tokens=370,560,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1931/500000] + train/ActionL1Loss=0.1014 + throughput/total_tokens=370,752,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1932/500000] + train/ActionL1Loss=0.1031 + throughput/total_tokens=370,944,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1933/500000] + train/ActionL1Loss=0.1153 + throughput/total_tokens=371,136,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1934/500000] + train/ActionL1Loss=0.1232 + throughput/total_tokens=371,328,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1935/500000] + train/ActionL1Loss=0.1134 + throughput/total_tokens=371,520,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1936/500000] + train/ActionL1Loss=0.1049 + throughput/total_tokens=371,712,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1937/500000] + train/ActionL1Loss=0.1179 + throughput/total_tokens=371,904,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1938/500000] + train/ActionL1Loss=0.1495 + throughput/total_tokens=372,096,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1939/500000] + train/ActionL1Loss=0.1532 + throughput/total_tokens=372,288,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1940/500000] + optim/total_grad_norm=6.180 + train/ActionL1Loss=0.1249 + throughput/total_tokens=372,480,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1941/500000] + train/ActionL1Loss=0.1362 + throughput/total_tokens=372,672,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1942/500000] + train/ActionL1Loss=0.1396 + throughput/total_tokens=372,864,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1943/500000] + train/ActionL1Loss=0.1124 + throughput/total_tokens=373,056,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1944/500000] + train/ActionL1Loss=0.1116 + throughput/total_tokens=373,248,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1945/500000] + train/ActionL1Loss=0.1276 + throughput/total_tokens=373,440,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1946/500000] + train/ActionL1Loss=0.1237 + throughput/total_tokens=373,632,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1947/500000] + train/ActionL1Loss=0.1396 + throughput/total_tokens=373,824,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1948/500000] + train/ActionL1Loss=0.1399 + throughput/total_tokens=374,016,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1949/500000] + train/ActionL1Loss=0.1296 + throughput/total_tokens=374,208,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=1950/500000] + train/ActionL1Loss=0.1423 + throughput/total_tokens=374,400,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1951/500000] + train/ActionL1Loss=0.1255 + throughput/total_tokens=374,592,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1952/500000] + train/ActionL1Loss=0.1033 + throughput/total_tokens=374,784,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1953/500000] + train/ActionL1Loss=0.1382 + throughput/total_tokens=374,976,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=1954/500000] + train/ActionL1Loss=0.1193 + throughput/total_tokens=375,168,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1955/500000] + train/ActionL1Loss=0.1232 + throughput/total_tokens=375,360,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1956/500000] + train/ActionL1Loss=0.1245 + throughput/total_tokens=375,552,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1957/500000] + train/ActionL1Loss=0.1127 + throughput/total_tokens=375,744,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1958/500000] + train/ActionL1Loss=0.1328 + throughput/total_tokens=375,936,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1959/500000] + train/ActionL1Loss=0.1079 + throughput/total_tokens=376,128,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1960/500000] + optim/total_grad_norm=7.108 + train/ActionL1Loss=0.1195 + throughput/total_tokens=376,320,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=1961/500000] + train/ActionL1Loss=0.1003 + throughput/total_tokens=376,512,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1962/500000] + train/ActionL1Loss=0.1216 + throughput/total_tokens=376,704,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1963/500000] + train/ActionL1Loss=0.1138 + throughput/total_tokens=376,896,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1964/500000] + train/ActionL1Loss=0.1309 + throughput/total_tokens=377,088,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1965/500000] + train/ActionL1Loss=0.1245 + throughput/total_tokens=377,280,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1966/500000] + train/ActionL1Loss=0.1212 + throughput/total_tokens=377,472,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=1967/500000] + train/ActionL1Loss=0.1089 + throughput/total_tokens=377,664,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1968/500000] + train/ActionL1Loss=0.1227 + throughput/total_tokens=377,856,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=1969/500000] + train/ActionL1Loss=0.1405 + throughput/total_tokens=378,048,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=1970/500000] + train/ActionL1Loss=0.1119 + throughput/total_tokens=378,240,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1971/500000] + train/ActionL1Loss=0.1200 + throughput/total_tokens=378,432,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1972/500000] + train/ActionL1Loss=0.1093 + throughput/total_tokens=378,624,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1973/500000] + train/ActionL1Loss=0.1162 + throughput/total_tokens=378,816,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1974/500000] + train/ActionL1Loss=0.1366 + throughput/total_tokens=379,008,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1975/500000] + train/ActionL1Loss=0.1092 + throughput/total_tokens=379,200,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1976/500000] + train/ActionL1Loss=0.1078 + throughput/total_tokens=379,392,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1977/500000] + train/ActionL1Loss=0.1275 + throughput/total_tokens=379,584,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1978/500000] + train/ActionL1Loss=0.1586 + throughput/total_tokens=379,776,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1979/500000] + train/ActionL1Loss=0.1303 + throughput/total_tokens=379,968,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1980/500000] + optim/total_grad_norm=6.323 + train/ActionL1Loss=0.1252 + throughput/total_tokens=380,160,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1981/500000] + train/ActionL1Loss=0.1182 + throughput/total_tokens=380,352,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1982/500000] + train/ActionL1Loss=0.1274 + throughput/total_tokens=380,544,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1983/500000] + train/ActionL1Loss=0.1006 + throughput/total_tokens=380,736,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1984/500000] + train/ActionL1Loss=0.1361 + throughput/total_tokens=380,928,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1985/500000] + train/ActionL1Loss=0.1267 + throughput/total_tokens=381,120,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1986/500000] + train/ActionL1Loss=0.1009 + throughput/total_tokens=381,312,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1987/500000] + train/ActionL1Loss=0.1113 + throughput/total_tokens=381,504,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1988/500000] + train/ActionL1Loss=0.1209 + throughput/total_tokens=381,696,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1989/500000] + train/ActionL1Loss=0.1052 + throughput/total_tokens=381,888,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1990/500000] + train/ActionL1Loss=0.1047 + throughput/total_tokens=382,080,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=1991/500000] + train/ActionL1Loss=0.1102 + throughput/total_tokens=382,272,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1992/500000] + train/ActionL1Loss=0.1346 + throughput/total_tokens=382,464,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1993/500000] + train/ActionL1Loss=0.1000 + throughput/total_tokens=382,656,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1994/500000] + train/ActionL1Loss=0.1122 + throughput/total_tokens=382,848,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1995/500000] + train/ActionL1Loss=0.1267 + throughput/total_tokens=383,040,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1996/500000] + train/ActionL1Loss=0.1080 + throughput/total_tokens=383,232,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=1997/500000] + train/ActionL1Loss=0.1361 + throughput/total_tokens=383,424,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1998/500000] + train/ActionL1Loss=0.1232 + throughput/total_tokens=383,616,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=1999/500000] + train/ActionL1Loss=0.1345 + throughput/total_tokens=383,808,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2000/500000] + optim/total_grad_norm=5.123 + train/ActionL1Loss=0.1279 + throughput/total_tokens=384,000,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +10/01 [00:59:45] INFO | >> Saving config... checkpoint.py:608 +10/01 [01:00:16] INFO | >> Saving model state... checkpoint.py:796 +10/01 [01:01:27] INFO | >> Saving optim state... checkpoint.py:811 +10/01 [01:02:54] INFO | >> Saving trainer state... checkpoint.py:670 + INFO | >> Saving config... checkpoint.py:608 +[step=2001/500000] + train/ActionL1Loss=0.1126 + throughput/total_tokens=384,192,000 + throughput/device/tokens_per_second=1,210 + throughput/device/batches_per_second=0.0504 +[step=2002/500000] + train/ActionL1Loss=0.1416 + throughput/total_tokens=384,384,000 + throughput/device/tokens_per_second=1,184 + throughput/device/batches_per_second=0.0494 +[step=2003/500000] + train/ActionL1Loss=0.1239 + throughput/total_tokens=384,576,000 + throughput/device/tokens_per_second=1,176 + throughput/device/batches_per_second=0.0490 +[step=2004/500000] + train/ActionL1Loss=0.1171 + throughput/total_tokens=384,768,000 + throughput/device/tokens_per_second=1,172 + throughput/device/batches_per_second=0.0489 +[step=2005/500000] + train/ActionL1Loss=0.1255 + throughput/total_tokens=384,960,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=2006/500000] + train/ActionL1Loss=0.1252 + throughput/total_tokens=385,152,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=2007/500000] + train/ActionL1Loss=0.1405 + throughput/total_tokens=385,344,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=2008/500000] + train/ActionL1Loss=0.1373 + throughput/total_tokens=385,536,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=2009/500000] + train/ActionL1Loss=0.1390 + throughput/total_tokens=385,728,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=2010/500000] + train/ActionL1Loss=0.1319 + throughput/total_tokens=385,920,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=2011/500000] + train/ActionL1Loss=0.1137 + throughput/total_tokens=386,112,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=2012/500000] + train/ActionL1Loss=0.1170 + throughput/total_tokens=386,304,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2013/500000] + train/ActionL1Loss=0.1200 + throughput/total_tokens=386,496,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2014/500000] + train/ActionL1Loss=0.1193 + throughput/total_tokens=386,688,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2015/500000] + train/ActionL1Loss=0.1278 + throughput/total_tokens=386,880,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2016/500000] + train/ActionL1Loss=0.1324 + throughput/total_tokens=387,072,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2017/500000] + train/ActionL1Loss=0.1225 + throughput/total_tokens=387,264,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2018/500000] + train/ActionL1Loss=0.1297 + throughput/total_tokens=387,456,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2019/500000] + train/ActionL1Loss=0.1146 + throughput/total_tokens=387,648,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2020/500000] + optim/total_grad_norm=8.507 + train/ActionL1Loss=0.1273 + throughput/total_tokens=387,840,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2021/500000] + train/ActionL1Loss=0.1200 + throughput/total_tokens=388,032,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2022/500000] + train/ActionL1Loss=0.1137 + throughput/total_tokens=388,224,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2023/500000] + train/ActionL1Loss=0.1347 + throughput/total_tokens=388,416,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2024/500000] + train/ActionL1Loss=0.1315 + throughput/total_tokens=388,608,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2025/500000] + train/ActionL1Loss=0.1423 + throughput/total_tokens=388,800,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2026/500000] + train/ActionL1Loss=0.1306 + throughput/total_tokens=388,992,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2027/500000] + train/ActionL1Loss=0.1244 + throughput/total_tokens=389,184,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2028/500000] + train/ActionL1Loss=0.1273 + throughput/total_tokens=389,376,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2029/500000] + train/ActionL1Loss=0.1303 + throughput/total_tokens=389,568,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2030/500000] + train/ActionL1Loss=0.1186 + throughput/total_tokens=389,760,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2031/500000] + train/ActionL1Loss=0.1081 + throughput/total_tokens=389,952,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2032/500000] + train/ActionL1Loss=0.1141 + throughput/total_tokens=390,144,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2033/500000] + train/ActionL1Loss=0.1242 + throughput/total_tokens=390,336,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2034/500000] + train/ActionL1Loss=0.1250 + throughput/total_tokens=390,528,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2035/500000] + train/ActionL1Loss=0.1074 + throughput/total_tokens=390,720,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2036/500000] + train/ActionL1Loss=0.1264 + throughput/total_tokens=390,912,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2037/500000] + train/ActionL1Loss=0.1130 + throughput/total_tokens=391,104,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2038/500000] + train/ActionL1Loss=0.0977 + throughput/total_tokens=391,296,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2039/500000] + train/ActionL1Loss=0.1270 + throughput/total_tokens=391,488,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2040/500000] + optim/total_grad_norm=8.681 + train/ActionL1Loss=0.1248 + throughput/total_tokens=391,680,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2041/500000] + train/ActionL1Loss=0.1165 + throughput/total_tokens=391,872,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2042/500000] + train/ActionL1Loss=0.1197 + throughput/total_tokens=392,064,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2043/500000] + train/ActionL1Loss=0.1158 + throughput/total_tokens=392,256,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2044/500000] + train/ActionL1Loss=0.1326 + throughput/total_tokens=392,448,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2045/500000] + train/ActionL1Loss=0.1247 + throughput/total_tokens=392,640,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2046/500000] + train/ActionL1Loss=0.1481 + throughput/total_tokens=392,832,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2047/500000] + train/ActionL1Loss=0.0987 + throughput/total_tokens=393,024,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2048/500000] + train/ActionL1Loss=0.0966 + throughput/total_tokens=393,216,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2049/500000] + train/ActionL1Loss=0.1278 + throughput/total_tokens=393,408,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2050/500000] + train/ActionL1Loss=0.1388 + throughput/total_tokens=393,600,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2051/500000] + train/ActionL1Loss=0.0925 + throughput/total_tokens=393,792,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2052/500000] + train/ActionL1Loss=0.1292 + throughput/total_tokens=393,984,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2053/500000] + train/ActionL1Loss=0.1029 + throughput/total_tokens=394,176,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2054/500000] + train/ActionL1Loss=0.1042 + throughput/total_tokens=394,368,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2055/500000] + train/ActionL1Loss=0.0995 + throughput/total_tokens=394,560,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2056/500000] + train/ActionL1Loss=0.1139 + throughput/total_tokens=394,752,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2057/500000] + train/ActionL1Loss=0.1063 + throughput/total_tokens=394,944,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2058/500000] + train/ActionL1Loss=0.1166 + throughput/total_tokens=395,136,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2059/500000] + train/ActionL1Loss=0.1084 + throughput/total_tokens=395,328,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2060/500000] + optim/total_grad_norm=7.364 + train/ActionL1Loss=0.1137 + throughput/total_tokens=395,520,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=2061/500000] + train/ActionL1Loss=0.1105 + throughput/total_tokens=395,712,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2062/500000] + train/ActionL1Loss=0.1443 + throughput/total_tokens=395,904,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2063/500000] + train/ActionL1Loss=0.1344 + throughput/total_tokens=396,096,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2064/500000] + train/ActionL1Loss=0.1158 + throughput/total_tokens=396,288,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2065/500000] + train/ActionL1Loss=0.1174 + throughput/total_tokens=396,480,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2066/500000] + train/ActionL1Loss=0.0993 + throughput/total_tokens=396,672,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2067/500000] + train/ActionL1Loss=0.1219 + throughput/total_tokens=396,864,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2068/500000] + train/ActionL1Loss=0.1265 + throughput/total_tokens=397,056,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2069/500000] + train/ActionL1Loss=0.1083 + throughput/total_tokens=397,248,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2070/500000] + train/ActionL1Loss=0.1065 + throughput/total_tokens=397,440,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2071/500000] + train/ActionL1Loss=0.1231 + throughput/total_tokens=397,632,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2072/500000] + train/ActionL1Loss=0.1113 + throughput/total_tokens=397,824,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2073/500000] + train/ActionL1Loss=0.1374 + throughput/total_tokens=398,016,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2074/500000] + train/ActionL1Loss=0.1076 + throughput/total_tokens=398,208,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2075/500000] + train/ActionL1Loss=0.1079 + throughput/total_tokens=398,400,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2076/500000] + train/ActionL1Loss=0.1060 + throughput/total_tokens=398,592,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2077/500000] + train/ActionL1Loss=0.1206 + throughput/total_tokens=398,784,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2078/500000] + train/ActionL1Loss=0.1213 + throughput/total_tokens=398,976,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2079/500000] + train/ActionL1Loss=0.1003 + throughput/total_tokens=399,168,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2080/500000] + optim/total_grad_norm=7.127 + train/ActionL1Loss=0.1095 + throughput/total_tokens=399,360,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2081/500000] + train/ActionL1Loss=0.1328 + throughput/total_tokens=399,552,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2082/500000] + train/ActionL1Loss=0.1384 + throughput/total_tokens=399,744,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2083/500000] + train/ActionL1Loss=0.1268 + throughput/total_tokens=399,936,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2084/500000] + train/ActionL1Loss=0.1300 + throughput/total_tokens=400,128,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2085/500000] + train/ActionL1Loss=0.1197 + throughput/total_tokens=400,320,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2086/500000] + train/ActionL1Loss=0.1227 + throughput/total_tokens=400,512,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2087/500000] + train/ActionL1Loss=0.1155 + throughput/total_tokens=400,704,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2088/500000] + train/ActionL1Loss=0.1248 + throughput/total_tokens=400,896,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2089/500000] + train/ActionL1Loss=0.1086 + throughput/total_tokens=401,088,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2090/500000] + train/ActionL1Loss=0.1067 + throughput/total_tokens=401,280,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2091/500000] + train/ActionL1Loss=0.1307 + throughput/total_tokens=401,472,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2092/500000] + train/ActionL1Loss=0.1300 + throughput/total_tokens=401,664,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2093/500000] + train/ActionL1Loss=0.1398 + throughput/total_tokens=401,856,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2094/500000] + train/ActionL1Loss=0.1088 + throughput/total_tokens=402,048,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2095/500000] + train/ActionL1Loss=0.1033 + throughput/total_tokens=402,240,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2096/500000] + train/ActionL1Loss=0.1029 + throughput/total_tokens=402,432,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2097/500000] + train/ActionL1Loss=0.1169 + throughput/total_tokens=402,624,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2098/500000] + train/ActionL1Loss=0.1268 + throughput/total_tokens=402,816,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2099/500000] + train/ActionL1Loss=0.1213 + throughput/total_tokens=403,008,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2100/500000] + optim/total_grad_norm=6.811 + train/ActionL1Loss=0.1087 + throughput/total_tokens=403,200,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2101/500000] + train/ActionL1Loss=0.1170 + throughput/total_tokens=403,392,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2102/500000] + train/ActionL1Loss=0.1057 + throughput/total_tokens=403,584,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2103/500000] + train/ActionL1Loss=0.1138 + throughput/total_tokens=403,776,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2104/500000] + train/ActionL1Loss=0.1301 + throughput/total_tokens=403,968,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2105/500000] + train/ActionL1Loss=0.1360 + throughput/total_tokens=404,160,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2106/500000] + train/ActionL1Loss=0.1105 + throughput/total_tokens=404,352,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2107/500000] + train/ActionL1Loss=0.1021 + throughput/total_tokens=404,544,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2108/500000] + train/ActionL1Loss=0.1057 + throughput/total_tokens=404,736,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2109/500000] + train/ActionL1Loss=0.1302 + throughput/total_tokens=404,928,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2110/500000] + train/ActionL1Loss=0.1044 + throughput/total_tokens=405,120,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=2111/500000] + train/ActionL1Loss=0.1208 + throughput/total_tokens=405,312,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2112/500000] + train/ActionL1Loss=0.1154 + throughput/total_tokens=405,504,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2113/500000] + train/ActionL1Loss=0.1292 + throughput/total_tokens=405,696,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2114/500000] + train/ActionL1Loss=0.1200 + throughput/total_tokens=405,888,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2115/500000] + train/ActionL1Loss=0.1020 + throughput/total_tokens=406,080,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2116/500000] + train/ActionL1Loss=0.1116 + throughput/total_tokens=406,272,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2117/500000] + train/ActionL1Loss=0.1168 + throughput/total_tokens=406,464,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2118/500000] + train/ActionL1Loss=0.1180 + throughput/total_tokens=406,656,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2119/500000] + train/ActionL1Loss=0.1193 + throughput/total_tokens=406,848,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2120/500000] + optim/total_grad_norm=5.559 + train/ActionL1Loss=0.1271 + throughput/total_tokens=407,040,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2121/500000] + train/ActionL1Loss=0.1273 + throughput/total_tokens=407,232,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2122/500000] + train/ActionL1Loss=0.1039 + throughput/total_tokens=407,424,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2123/500000] + train/ActionL1Loss=0.1258 + throughput/total_tokens=407,616,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2124/500000] + train/ActionL1Loss=0.1266 + throughput/total_tokens=407,808,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2125/500000] + train/ActionL1Loss=0.1074 + throughput/total_tokens=408,000,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2126/500000] + train/ActionL1Loss=0.1281 + throughput/total_tokens=408,192,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2127/500000] + train/ActionL1Loss=0.1110 + throughput/total_tokens=408,384,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2128/500000] + train/ActionL1Loss=0.1035 + throughput/total_tokens=408,576,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2129/500000] + train/ActionL1Loss=0.0833 + throughput/total_tokens=408,768,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2130/500000] + train/ActionL1Loss=0.1173 + throughput/total_tokens=408,960,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2131/500000] + train/ActionL1Loss=0.1103 + throughput/total_tokens=409,152,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2132/500000] + train/ActionL1Loss=0.1069 + throughput/total_tokens=409,344,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2133/500000] + train/ActionL1Loss=0.0904 + throughput/total_tokens=409,536,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2134/500000] + train/ActionL1Loss=0.1200 + throughput/total_tokens=409,728,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2135/500000] + train/ActionL1Loss=0.1165 + throughput/total_tokens=409,920,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2136/500000] + train/ActionL1Loss=0.1152 + throughput/total_tokens=410,112,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2137/500000] + train/ActionL1Loss=0.1095 + throughput/total_tokens=410,304,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2138/500000] + train/ActionL1Loss=0.1108 + throughput/total_tokens=410,496,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2139/500000] + train/ActionL1Loss=0.1037 + throughput/total_tokens=410,688,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2140/500000] + optim/total_grad_norm=9.173 + train/ActionL1Loss=0.1195 + throughput/total_tokens=410,880,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2141/500000] + train/ActionL1Loss=0.1140 + throughput/total_tokens=411,072,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2142/500000] + train/ActionL1Loss=0.1084 + throughput/total_tokens=411,264,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2143/500000] + train/ActionL1Loss=0.1050 + throughput/total_tokens=411,456,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2144/500000] + train/ActionL1Loss=0.1187 + throughput/total_tokens=411,648,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2145/500000] + train/ActionL1Loss=0.1260 + throughput/total_tokens=411,840,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2146/500000] + train/ActionL1Loss=0.1299 + throughput/total_tokens=412,032,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2147/500000] + train/ActionL1Loss=0.1002 + throughput/total_tokens=412,224,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2148/500000] + train/ActionL1Loss=0.1211 + throughput/total_tokens=412,416,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2149/500000] + train/ActionL1Loss=0.1178 + throughput/total_tokens=412,608,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2150/500000] + train/ActionL1Loss=0.1090 + throughput/total_tokens=412,800,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2151/500000] + train/ActionL1Loss=0.1030 + throughput/total_tokens=412,992,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=2152/500000] + train/ActionL1Loss=0.1280 + throughput/total_tokens=413,184,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=2153/500000] + train/ActionL1Loss=0.1352 + throughput/total_tokens=413,376,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=2154/500000] + train/ActionL1Loss=0.1112 + throughput/total_tokens=413,568,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=2155/500000] + train/ActionL1Loss=0.1044 + throughput/total_tokens=413,760,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=2156/500000] + train/ActionL1Loss=0.1278 + throughput/total_tokens=413,952,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=2157/500000] + train/ActionL1Loss=0.1019 + throughput/total_tokens=414,144,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=2158/500000] + train/ActionL1Loss=0.1163 + throughput/total_tokens=414,336,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=2159/500000] + train/ActionL1Loss=0.1344 + throughput/total_tokens=414,528,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=2160/500000] + optim/total_grad_norm=5.818 + train/ActionL1Loss=0.1238 + throughput/total_tokens=414,720,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=2161/500000] + train/ActionL1Loss=0.1060 + throughput/total_tokens=414,912,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=2162/500000] + train/ActionL1Loss=0.1064 + throughput/total_tokens=415,104,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=2163/500000] + train/ActionL1Loss=0.1144 + throughput/total_tokens=415,296,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2164/500000] + train/ActionL1Loss=0.1136 + throughput/total_tokens=415,488,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2165/500000] + train/ActionL1Loss=0.1121 + throughput/total_tokens=415,680,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2166/500000] + train/ActionL1Loss=0.1064 + throughput/total_tokens=415,872,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2167/500000] + train/ActionL1Loss=0.1015 + throughput/total_tokens=416,064,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2168/500000] + train/ActionL1Loss=0.1146 + throughput/total_tokens=416,256,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2169/500000] + train/ActionL1Loss=0.1101 + throughput/total_tokens=416,448,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2170/500000] + train/ActionL1Loss=0.1332 + throughput/total_tokens=416,640,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2171/500000] + train/ActionL1Loss=0.1035 + throughput/total_tokens=416,832,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2172/500000] + train/ActionL1Loss=0.1071 + throughput/total_tokens=417,024,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2173/500000] + train/ActionL1Loss=0.1235 + throughput/total_tokens=417,216,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2174/500000] + train/ActionL1Loss=0.1240 + throughput/total_tokens=417,408,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2175/500000] + train/ActionL1Loss=0.1226 + throughput/total_tokens=417,600,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2176/500000] + train/ActionL1Loss=0.1302 + throughput/total_tokens=417,792,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2177/500000] + train/ActionL1Loss=0.0980 + throughput/total_tokens=417,984,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2178/500000] + train/ActionL1Loss=0.1114 + throughput/total_tokens=418,176,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2179/500000] + train/ActionL1Loss=0.1105 + throughput/total_tokens=418,368,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2180/500000] + optim/total_grad_norm=6.645 + train/ActionL1Loss=0.1180 + throughput/total_tokens=418,560,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2181/500000] + train/ActionL1Loss=0.1100 + throughput/total_tokens=418,752,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2182/500000] + train/ActionL1Loss=0.1332 + throughput/total_tokens=418,944,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2183/500000] + train/ActionL1Loss=0.1236 + throughput/total_tokens=419,136,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2184/500000] + train/ActionL1Loss=0.1040 + throughput/total_tokens=419,328,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2185/500000] + train/ActionL1Loss=0.1000 + throughput/total_tokens=419,520,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2186/500000] + train/ActionL1Loss=0.1144 + throughput/total_tokens=419,712,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2187/500000] + train/ActionL1Loss=0.1090 + throughput/total_tokens=419,904,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2188/500000] + train/ActionL1Loss=0.1231 + throughput/total_tokens=420,096,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2189/500000] + train/ActionL1Loss=0.1337 + throughput/total_tokens=420,288,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2190/500000] + train/ActionL1Loss=0.1162 + throughput/total_tokens=420,480,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2191/500000] + train/ActionL1Loss=0.0983 + throughput/total_tokens=420,672,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2192/500000] + train/ActionL1Loss=0.1066 + throughput/total_tokens=420,864,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2193/500000] + train/ActionL1Loss=0.1106 + throughput/total_tokens=421,056,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2194/500000] + train/ActionL1Loss=0.1381 + throughput/total_tokens=421,248,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2195/500000] + train/ActionL1Loss=0.1220 + throughput/total_tokens=421,440,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2196/500000] + train/ActionL1Loss=0.1392 + throughput/total_tokens=421,632,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2197/500000] + train/ActionL1Loss=0.1123 + throughput/total_tokens=421,824,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2198/500000] + train/ActionL1Loss=0.1132 + throughput/total_tokens=422,016,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2199/500000] + train/ActionL1Loss=0.1164 + throughput/total_tokens=422,208,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2200/500000] + optim/total_grad_norm=7.288 + train/ActionL1Loss=0.1134 + throughput/total_tokens=422,400,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2201/500000] + train/ActionL1Loss=0.1197 + throughput/total_tokens=422,592,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2202/500000] + train/ActionL1Loss=0.1172 + throughput/total_tokens=422,784,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2203/500000] + train/ActionL1Loss=0.1292 + throughput/total_tokens=422,976,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2204/500000] + train/ActionL1Loss=0.1064 + throughput/total_tokens=423,168,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2205/500000] + train/ActionL1Loss=0.1012 + throughput/total_tokens=423,360,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2206/500000] + train/ActionL1Loss=0.1066 + throughput/total_tokens=423,552,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2207/500000] + train/ActionL1Loss=0.0971 + throughput/total_tokens=423,744,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2208/500000] + train/ActionL1Loss=0.1019 + throughput/total_tokens=423,936,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2209/500000] + train/ActionL1Loss=0.1026 + throughput/total_tokens=424,128,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2210/500000] + train/ActionL1Loss=0.1302 + throughput/total_tokens=424,320,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=2211/500000] + train/ActionL1Loss=0.1167 + throughput/total_tokens=424,512,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2212/500000] + train/ActionL1Loss=0.1142 + throughput/total_tokens=424,704,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2213/500000] + train/ActionL1Loss=0.1073 + throughput/total_tokens=424,896,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2214/500000] + train/ActionL1Loss=0.1126 + throughput/total_tokens=425,088,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2215/500000] + train/ActionL1Loss=0.1036 + throughput/total_tokens=425,280,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2216/500000] + train/ActionL1Loss=0.1097 + throughput/total_tokens=425,472,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2217/500000] + train/ActionL1Loss=0.0972 + throughput/total_tokens=425,664,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2218/500000] + train/ActionL1Loss=0.0881 + throughput/total_tokens=425,856,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2219/500000] + train/ActionL1Loss=0.0970 + throughput/total_tokens=426,048,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2220/500000] + optim/total_grad_norm=8.429 + train/ActionL1Loss=0.1286 + throughput/total_tokens=426,240,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2221/500000] + train/ActionL1Loss=0.0960 + throughput/total_tokens=426,432,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2222/500000] + train/ActionL1Loss=0.1021 + throughput/total_tokens=426,624,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2223/500000] + train/ActionL1Loss=0.1221 + throughput/total_tokens=426,816,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2224/500000] + train/ActionL1Loss=0.1257 + throughput/total_tokens=427,008,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2225/500000] + train/ActionL1Loss=0.1280 + throughput/total_tokens=427,200,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2226/500000] + train/ActionL1Loss=0.1394 + throughput/total_tokens=427,392,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2227/500000] + train/ActionL1Loss=0.1037 + throughput/total_tokens=427,584,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2228/500000] + train/ActionL1Loss=0.0984 + throughput/total_tokens=427,776,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2229/500000] + train/ActionL1Loss=0.1321 + throughput/total_tokens=427,968,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2230/500000] + train/ActionL1Loss=0.1135 + throughput/total_tokens=428,160,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2231/500000] + train/ActionL1Loss=0.1102 + throughput/total_tokens=428,352,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2232/500000] + train/ActionL1Loss=0.0905 + throughput/total_tokens=428,544,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2233/500000] + train/ActionL1Loss=0.1140 + throughput/total_tokens=428,736,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2234/500000] + train/ActionL1Loss=0.1259 + throughput/total_tokens=428,928,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2235/500000] + train/ActionL1Loss=0.0949 + throughput/total_tokens=429,120,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2236/500000] + train/ActionL1Loss=0.0882 + throughput/total_tokens=429,312,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2237/500000] + train/ActionL1Loss=0.1001 + throughput/total_tokens=429,504,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2238/500000] + train/ActionL1Loss=0.0964 + throughput/total_tokens=429,696,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2239/500000] + train/ActionL1Loss=0.1169 + throughput/total_tokens=429,888,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2240/500000] + optim/total_grad_norm=6.407 + train/ActionL1Loss=0.1079 + throughput/total_tokens=430,080,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2241/500000] + train/ActionL1Loss=0.1169 + throughput/total_tokens=430,272,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2242/500000] + train/ActionL1Loss=0.1021 + throughput/total_tokens=430,464,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2243/500000] + train/ActionL1Loss=0.1263 + throughput/total_tokens=430,656,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2244/500000] + train/ActionL1Loss=0.1069 + throughput/total_tokens=430,848,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2245/500000] + train/ActionL1Loss=0.1066 + throughput/total_tokens=431,040,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2246/500000] + train/ActionL1Loss=0.1089 + throughput/total_tokens=431,232,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2247/500000] + train/ActionL1Loss=0.1017 + throughput/total_tokens=431,424,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2248/500000] + train/ActionL1Loss=0.0995 + throughput/total_tokens=431,616,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2249/500000] + train/ActionL1Loss=0.1108 + throughput/total_tokens=431,808,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2250/500000] + train/ActionL1Loss=0.1099 + throughput/total_tokens=432,000,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2251/500000] + train/ActionL1Loss=0.1151 + throughput/total_tokens=432,192,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2252/500000] + train/ActionL1Loss=0.1163 + throughput/total_tokens=432,384,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2253/500000] + train/ActionL1Loss=0.0942 + throughput/total_tokens=432,576,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2254/500000] + train/ActionL1Loss=0.0762 + throughput/total_tokens=432,768,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2255/500000] + train/ActionL1Loss=0.1011 + throughput/total_tokens=432,960,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2256/500000] + train/ActionL1Loss=0.1031 + throughput/total_tokens=433,152,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2257/500000] + train/ActionL1Loss=0.0836 + throughput/total_tokens=433,344,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2258/500000] + train/ActionL1Loss=0.1107 + throughput/total_tokens=433,536,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2259/500000] + train/ActionL1Loss=0.1008 + throughput/total_tokens=433,728,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2260/500000] + optim/total_grad_norm=6.029 + train/ActionL1Loss=0.1150 + throughput/total_tokens=433,920,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=2261/500000] + train/ActionL1Loss=0.0936 + throughput/total_tokens=434,112,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2262/500000] + train/ActionL1Loss=0.1172 + throughput/total_tokens=434,304,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2263/500000] + train/ActionL1Loss=0.0974 + throughput/total_tokens=434,496,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2264/500000] + train/ActionL1Loss=0.1380 + throughput/total_tokens=434,688,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2265/500000] + train/ActionL1Loss=0.1024 + throughput/total_tokens=434,880,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2266/500000] + train/ActionL1Loss=0.1275 + throughput/total_tokens=435,072,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2267/500000] + train/ActionL1Loss=0.1257 + throughput/total_tokens=435,264,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2268/500000] + train/ActionL1Loss=0.1205 + throughput/total_tokens=435,456,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2269/500000] + train/ActionL1Loss=0.1303 + throughput/total_tokens=435,648,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2270/500000] + train/ActionL1Loss=0.1115 + throughput/total_tokens=435,840,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2271/500000] + train/ActionL1Loss=0.0958 + throughput/total_tokens=436,032,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2272/500000] + train/ActionL1Loss=0.1282 + throughput/total_tokens=436,224,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2273/500000] + train/ActionL1Loss=0.1170 + throughput/total_tokens=436,416,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2274/500000] + train/ActionL1Loss=0.1073 + throughput/total_tokens=436,608,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2275/500000] + train/ActionL1Loss=0.1055 + throughput/total_tokens=436,800,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2276/500000] + train/ActionL1Loss=0.0950 + throughput/total_tokens=436,992,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2277/500000] + train/ActionL1Loss=0.1310 + throughput/total_tokens=437,184,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2278/500000] + train/ActionL1Loss=0.1073 + throughput/total_tokens=437,376,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2279/500000] + train/ActionL1Loss=0.1295 + throughput/total_tokens=437,568,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2280/500000] + optim/total_grad_norm=5.443 + train/ActionL1Loss=0.1022 + throughput/total_tokens=437,760,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2281/500000] + train/ActionL1Loss=0.1113 + throughput/total_tokens=437,952,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2282/500000] + train/ActionL1Loss=0.1032 + throughput/total_tokens=438,144,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2283/500000] + train/ActionL1Loss=0.1211 + throughput/total_tokens=438,336,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2284/500000] + train/ActionL1Loss=0.1193 + throughput/total_tokens=438,528,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2285/500000] + train/ActionL1Loss=0.0965 + throughput/total_tokens=438,720,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2286/500000] + train/ActionL1Loss=0.1141 + throughput/total_tokens=438,912,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2287/500000] + train/ActionL1Loss=0.1031 + throughput/total_tokens=439,104,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2288/500000] + train/ActionL1Loss=0.1143 + throughput/total_tokens=439,296,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2289/500000] + train/ActionL1Loss=0.1082 + throughput/total_tokens=439,488,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2290/500000] + train/ActionL1Loss=0.1057 + throughput/total_tokens=439,680,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2291/500000] + train/ActionL1Loss=0.1216 + throughput/total_tokens=439,872,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2292/500000] + train/ActionL1Loss=0.1178 + throughput/total_tokens=440,064,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2293/500000] + train/ActionL1Loss=0.1019 + throughput/total_tokens=440,256,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2294/500000] + train/ActionL1Loss=0.1069 + throughput/total_tokens=440,448,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2295/500000] + train/ActionL1Loss=0.0960 + throughput/total_tokens=440,640,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2296/500000] + train/ActionL1Loss=0.0985 + throughput/total_tokens=440,832,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2297/500000] + train/ActionL1Loss=0.1211 + throughput/total_tokens=441,024,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2298/500000] + train/ActionL1Loss=0.1207 + throughput/total_tokens=441,216,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2299/500000] + train/ActionL1Loss=0.1099 + throughput/total_tokens=441,408,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2300/500000] + optim/total_grad_norm=6.286 + train/ActionL1Loss=0.1293 + throughput/total_tokens=441,600,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2301/500000] + train/ActionL1Loss=0.1115 + throughput/total_tokens=441,792,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2302/500000] + train/ActionL1Loss=0.1414 + throughput/total_tokens=441,984,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2303/500000] + train/ActionL1Loss=0.1174 + throughput/total_tokens=442,176,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2304/500000] + train/ActionL1Loss=0.1234 + throughput/total_tokens=442,368,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2305/500000] + train/ActionL1Loss=0.1226 + throughput/total_tokens=442,560,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2306/500000] + train/ActionL1Loss=0.1078 + throughput/total_tokens=442,752,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2307/500000] + train/ActionL1Loss=0.1063 + throughput/total_tokens=442,944,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2308/500000] + train/ActionL1Loss=0.1158 + throughput/total_tokens=443,136,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2309/500000] + train/ActionL1Loss=0.0962 + throughput/total_tokens=443,328,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2310/500000] + train/ActionL1Loss=0.1070 + throughput/total_tokens=443,520,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2311/500000] + train/ActionL1Loss=0.0831 + throughput/total_tokens=443,712,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2312/500000] + train/ActionL1Loss=0.1086 + throughput/total_tokens=443,904,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2313/500000] + train/ActionL1Loss=0.1055 + throughput/total_tokens=444,096,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2314/500000] + train/ActionL1Loss=0.0946 + throughput/total_tokens=444,288,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2315/500000] + train/ActionL1Loss=0.1000 + throughput/total_tokens=444,480,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2316/500000] + train/ActionL1Loss=0.1149 + throughput/total_tokens=444,672,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2317/500000] + train/ActionL1Loss=0.1165 + throughput/total_tokens=444,864,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2318/500000] + train/ActionL1Loss=0.1018 + throughput/total_tokens=445,056,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2319/500000] + train/ActionL1Loss=0.1115 + throughput/total_tokens=445,248,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2320/500000] + optim/total_grad_norm=7.541 + train/ActionL1Loss=0.1095 + throughput/total_tokens=445,440,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2321/500000] + train/ActionL1Loss=0.0949 + throughput/total_tokens=445,632,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2322/500000] + train/ActionL1Loss=0.1061 + throughput/total_tokens=445,824,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2323/500000] + train/ActionL1Loss=0.1117 + throughput/total_tokens=446,016,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2324/500000] + train/ActionL1Loss=0.0896 + throughput/total_tokens=446,208,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2325/500000] + train/ActionL1Loss=0.1155 + throughput/total_tokens=446,400,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2326/500000] + train/ActionL1Loss=0.1086 + throughput/total_tokens=446,592,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2327/500000] + train/ActionL1Loss=0.0953 + throughput/total_tokens=446,784,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2328/500000] + train/ActionL1Loss=0.1073 + throughput/total_tokens=446,976,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2329/500000] + train/ActionL1Loss=0.1252 + throughput/total_tokens=447,168,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2330/500000] + train/ActionL1Loss=0.1359 + throughput/total_tokens=447,360,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2331/500000] + train/ActionL1Loss=0.1357 + throughput/total_tokens=447,552,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2332/500000] + train/ActionL1Loss=0.1118 + throughput/total_tokens=447,744,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2333/500000] + train/ActionL1Loss=0.1239 + throughput/total_tokens=447,936,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2334/500000] + train/ActionL1Loss=0.1122 + throughput/total_tokens=448,128,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2335/500000] + train/ActionL1Loss=0.1067 + throughput/total_tokens=448,320,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2336/500000] + train/ActionL1Loss=0.1279 + throughput/total_tokens=448,512,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2337/500000] + train/ActionL1Loss=0.1076 + throughput/total_tokens=448,704,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2338/500000] + train/ActionL1Loss=0.1075 + throughput/total_tokens=448,896,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2339/500000] + train/ActionL1Loss=0.1165 + throughput/total_tokens=449,088,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2340/500000] + optim/total_grad_norm=5.573 + train/ActionL1Loss=0.0977 + throughput/total_tokens=449,280,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2341/500000] + train/ActionL1Loss=0.1218 + throughput/total_tokens=449,472,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2342/500000] + train/ActionL1Loss=0.1036 + throughput/total_tokens=449,664,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2343/500000] + train/ActionL1Loss=0.1061 + throughput/total_tokens=449,856,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2344/500000] + train/ActionL1Loss=0.1073 + throughput/total_tokens=450,048,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2345/500000] + train/ActionL1Loss=0.1169 + throughput/total_tokens=450,240,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2346/500000] + train/ActionL1Loss=0.0850 + throughput/total_tokens=450,432,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2347/500000] + train/ActionL1Loss=0.1031 + throughput/total_tokens=450,624,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2348/500000] + train/ActionL1Loss=0.1117 + throughput/total_tokens=450,816,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2349/500000] + train/ActionL1Loss=0.1055 + throughput/total_tokens=451,008,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2350/500000] + train/ActionL1Loss=0.1028 + throughput/total_tokens=451,200,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2351/500000] + train/ActionL1Loss=0.1166 + throughput/total_tokens=451,392,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2352/500000] + train/ActionL1Loss=0.1046 + throughput/total_tokens=451,584,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2353/500000] + train/ActionL1Loss=0.1107 + throughput/total_tokens=451,776,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2354/500000] + train/ActionL1Loss=0.1205 + throughput/total_tokens=451,968,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2355/500000] + train/ActionL1Loss=0.1177 + throughput/total_tokens=452,160,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2356/500000] + train/ActionL1Loss=0.1140 + throughput/total_tokens=452,352,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2357/500000] + train/ActionL1Loss=0.1151 + throughput/total_tokens=452,544,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2358/500000] + train/ActionL1Loss=0.1109 + throughput/total_tokens=452,736,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2359/500000] + train/ActionL1Loss=0.1124 + throughput/total_tokens=452,928,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2360/500000] + optim/total_grad_norm=6.147 + train/ActionL1Loss=0.1027 + throughput/total_tokens=453,120,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2361/500000] + train/ActionL1Loss=0.1064 + throughput/total_tokens=453,312,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2362/500000] + train/ActionL1Loss=0.1024 + throughput/total_tokens=453,504,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2363/500000] + train/ActionL1Loss=0.0980 + throughput/total_tokens=453,696,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2364/500000] + train/ActionL1Loss=0.1026 + throughput/total_tokens=453,888,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2365/500000] + train/ActionL1Loss=0.1258 + throughput/total_tokens=454,080,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2366/500000] + train/ActionL1Loss=0.1036 + throughput/total_tokens=454,272,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2367/500000] + train/ActionL1Loss=0.0990 + throughput/total_tokens=454,464,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2368/500000] + train/ActionL1Loss=0.0947 + throughput/total_tokens=454,656,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2369/500000] + train/ActionL1Loss=0.1022 + throughput/total_tokens=454,848,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2370/500000] + train/ActionL1Loss=0.0971 + throughput/total_tokens=455,040,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2371/500000] + train/ActionL1Loss=0.1266 + throughput/total_tokens=455,232,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2372/500000] + train/ActionL1Loss=0.1106 + throughput/total_tokens=455,424,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2373/500000] + train/ActionL1Loss=0.0990 + throughput/total_tokens=455,616,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2374/500000] + train/ActionL1Loss=0.1034 + throughput/total_tokens=455,808,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2375/500000] + train/ActionL1Loss=0.0962 + throughput/total_tokens=456,000,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2376/500000] + train/ActionL1Loss=0.1113 + throughput/total_tokens=456,192,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2377/500000] + train/ActionL1Loss=0.1308 + throughput/total_tokens=456,384,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2378/500000] + train/ActionL1Loss=0.1093 + throughput/total_tokens=456,576,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2379/500000] + train/ActionL1Loss=0.1110 + throughput/total_tokens=456,768,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2380/500000] + optim/total_grad_norm=8.316 + train/ActionL1Loss=0.0972 + throughput/total_tokens=456,960,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2381/500000] + train/ActionL1Loss=0.1046 + throughput/total_tokens=457,152,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2382/500000] + train/ActionL1Loss=0.1135 + throughput/total_tokens=457,344,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2383/500000] + train/ActionL1Loss=0.1194 + throughput/total_tokens=457,536,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2384/500000] + train/ActionL1Loss=0.1189 + throughput/total_tokens=457,728,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2385/500000] + train/ActionL1Loss=0.1028 + throughput/total_tokens=457,920,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2386/500000] + train/ActionL1Loss=0.1140 + throughput/total_tokens=458,112,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2387/500000] + train/ActionL1Loss=0.1108 + throughput/total_tokens=458,304,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2388/500000] + train/ActionL1Loss=0.0896 + throughput/total_tokens=458,496,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2389/500000] + train/ActionL1Loss=0.1285 + throughput/total_tokens=458,688,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2390/500000] + train/ActionL1Loss=0.1186 + throughput/total_tokens=458,880,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2391/500000] + train/ActionL1Loss=0.1077 + throughput/total_tokens=459,072,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2392/500000] + train/ActionL1Loss=0.1209 + throughput/total_tokens=459,264,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2393/500000] + train/ActionL1Loss=0.0998 + throughput/total_tokens=459,456,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2394/500000] + train/ActionL1Loss=0.1108 + throughput/total_tokens=459,648,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2395/500000] + train/ActionL1Loss=0.0888 + throughput/total_tokens=459,840,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2396/500000] + train/ActionL1Loss=0.1072 + throughput/total_tokens=460,032,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2397/500000] + train/ActionL1Loss=0.0990 + throughput/total_tokens=460,224,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2398/500000] + train/ActionL1Loss=0.0908 + throughput/total_tokens=460,416,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2399/500000] + train/ActionL1Loss=0.1042 + throughput/total_tokens=460,608,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2400/500000] + optim/total_grad_norm=6.326 + train/ActionL1Loss=0.0887 + throughput/total_tokens=460,800,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2401/500000] + train/ActionL1Loss=0.1030 + throughput/total_tokens=460,992,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2402/500000] + train/ActionL1Loss=0.1030 + throughput/total_tokens=461,184,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2403/500000] + train/ActionL1Loss=0.0990 + throughput/total_tokens=461,376,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2404/500000] + train/ActionL1Loss=0.0943 + throughput/total_tokens=461,568,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2405/500000] + train/ActionL1Loss=0.1019 + throughput/total_tokens=461,760,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2406/500000] + train/ActionL1Loss=0.0979 + throughput/total_tokens=461,952,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2407/500000] + train/ActionL1Loss=0.0925 + throughput/total_tokens=462,144,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2408/500000] + train/ActionL1Loss=0.0968 + throughput/total_tokens=462,336,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2409/500000] + train/ActionL1Loss=0.1226 + throughput/total_tokens=462,528,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2410/500000] + train/ActionL1Loss=0.1220 + throughput/total_tokens=462,720,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=2411/500000] + train/ActionL1Loss=0.0982 + throughput/total_tokens=462,912,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2412/500000] + train/ActionL1Loss=0.1175 + throughput/total_tokens=463,104,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2413/500000] + train/ActionL1Loss=0.1100 + throughput/total_tokens=463,296,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=2414/500000] + train/ActionL1Loss=0.1080 + throughput/total_tokens=463,488,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=2415/500000] + train/ActionL1Loss=0.1055 + throughput/total_tokens=463,680,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2416/500000] + train/ActionL1Loss=0.1012 + throughput/total_tokens=463,872,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2417/500000] + train/ActionL1Loss=0.0855 + throughput/total_tokens=464,064,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2418/500000] + train/ActionL1Loss=0.0918 + throughput/total_tokens=464,256,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2419/500000] + train/ActionL1Loss=0.1004 + throughput/total_tokens=464,448,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2420/500000] + optim/total_grad_norm=6.530 + train/ActionL1Loss=0.1097 + throughput/total_tokens=464,640,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2421/500000] + train/ActionL1Loss=0.1114 + throughput/total_tokens=464,832,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2422/500000] + train/ActionL1Loss=0.1184 + throughput/total_tokens=465,024,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2423/500000] + train/ActionL1Loss=0.1033 + throughput/total_tokens=465,216,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2424/500000] + train/ActionL1Loss=0.0931 + throughput/total_tokens=465,408,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2425/500000] + train/ActionL1Loss=0.1016 + throughput/total_tokens=465,600,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2426/500000] + train/ActionL1Loss=0.1162 + throughput/total_tokens=465,792,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2427/500000] + train/ActionL1Loss=0.1072 + throughput/total_tokens=465,984,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2428/500000] + train/ActionL1Loss=0.1177 + throughput/total_tokens=466,176,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2429/500000] + train/ActionL1Loss=0.0893 + throughput/total_tokens=466,368,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2430/500000] + train/ActionL1Loss=0.0996 + throughput/total_tokens=466,560,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2431/500000] + train/ActionL1Loss=0.1187 + throughput/total_tokens=466,752,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2432/500000] + train/ActionL1Loss=0.1015 + throughput/total_tokens=466,944,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=2433/500000] + train/ActionL1Loss=0.1003 + throughput/total_tokens=467,136,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2434/500000] + train/ActionL1Loss=0.0878 + throughput/total_tokens=467,328,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2435/500000] + train/ActionL1Loss=0.1247 + throughput/total_tokens=467,520,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2436/500000] + train/ActionL1Loss=0.0717 + throughput/total_tokens=467,712,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2437/500000] + train/ActionL1Loss=0.1040 + throughput/total_tokens=467,904,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2438/500000] + train/ActionL1Loss=0.1082 + throughput/total_tokens=468,096,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2439/500000] + train/ActionL1Loss=0.1084 + throughput/total_tokens=468,288,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2440/500000] + optim/total_grad_norm=6.589 + train/ActionL1Loss=0.1016 + throughput/total_tokens=468,480,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2441/500000] + train/ActionL1Loss=0.0961 + throughput/total_tokens=468,672,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2442/500000] + train/ActionL1Loss=0.0767 + throughput/total_tokens=468,864,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2443/500000] + train/ActionL1Loss=0.1010 + throughput/total_tokens=469,056,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2444/500000] + train/ActionL1Loss=0.0985 + throughput/total_tokens=469,248,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2445/500000] + train/ActionL1Loss=0.0968 + throughput/total_tokens=469,440,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2446/500000] + train/ActionL1Loss=0.0846 + throughput/total_tokens=469,632,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2447/500000] + train/ActionL1Loss=0.0872 + throughput/total_tokens=469,824,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2448/500000] + train/ActionL1Loss=0.0913 + throughput/total_tokens=470,016,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2449/500000] + train/ActionL1Loss=0.0941 + throughput/total_tokens=470,208,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2450/500000] + train/ActionL1Loss=0.1101 + throughput/total_tokens=470,400,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2451/500000] + train/ActionL1Loss=0.0970 + throughput/total_tokens=470,592,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2452/500000] + train/ActionL1Loss=0.1136 + throughput/total_tokens=470,784,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2453/500000] + train/ActionL1Loss=0.1182 + throughput/total_tokens=470,976,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2454/500000] + train/ActionL1Loss=0.1091 + throughput/total_tokens=471,168,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2455/500000] + train/ActionL1Loss=0.0966 + throughput/total_tokens=471,360,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2456/500000] + train/ActionL1Loss=0.0924 + throughput/total_tokens=471,552,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2457/500000] + train/ActionL1Loss=0.1061 + throughput/total_tokens=471,744,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2458/500000] + train/ActionL1Loss=0.1024 + throughput/total_tokens=471,936,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2459/500000] + train/ActionL1Loss=0.0948 + throughput/total_tokens=472,128,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2460/500000] + optim/total_grad_norm=6.485 + train/ActionL1Loss=0.1007 + throughput/total_tokens=472,320,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2461/500000] + train/ActionL1Loss=0.0877 + throughput/total_tokens=472,512,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2462/500000] + train/ActionL1Loss=0.1176 + throughput/total_tokens=472,704,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2463/500000] + train/ActionL1Loss=0.1131 + throughput/total_tokens=472,896,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2464/500000] + train/ActionL1Loss=0.1025 + throughput/total_tokens=473,088,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2465/500000] + train/ActionL1Loss=0.0940 + throughput/total_tokens=473,280,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2466/500000] + train/ActionL1Loss=0.0946 + throughput/total_tokens=473,472,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2467/500000] + train/ActionL1Loss=0.1206 + throughput/total_tokens=473,664,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2468/500000] + train/ActionL1Loss=0.1073 + throughput/total_tokens=473,856,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2469/500000] + train/ActionL1Loss=0.0946 + throughput/total_tokens=474,048,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2470/500000] + train/ActionL1Loss=0.0984 + throughput/total_tokens=474,240,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2471/500000] + train/ActionL1Loss=0.0903 + throughput/total_tokens=474,432,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2472/500000] + train/ActionL1Loss=0.0799 + throughput/total_tokens=474,624,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2473/500000] + train/ActionL1Loss=0.0914 + throughput/total_tokens=474,816,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2474/500000] + train/ActionL1Loss=0.1025 + throughput/total_tokens=475,008,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2475/500000] + train/ActionL1Loss=0.1013 + throughput/total_tokens=475,200,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2476/500000] + train/ActionL1Loss=0.0777 + throughput/total_tokens=475,392,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2477/500000] + train/ActionL1Loss=0.0902 + throughput/total_tokens=475,584,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2478/500000] + train/ActionL1Loss=0.0878 + throughput/total_tokens=475,776,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2479/500000] + train/ActionL1Loss=0.0887 + throughput/total_tokens=475,968,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2480/500000] + optim/total_grad_norm=7.005 + train/ActionL1Loss=0.0983 + throughput/total_tokens=476,160,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2481/500000] + train/ActionL1Loss=0.0899 + throughput/total_tokens=476,352,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2482/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=476,544,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2483/500000] + train/ActionL1Loss=0.0908 + throughput/total_tokens=476,736,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2484/500000] + train/ActionL1Loss=0.0935 + throughput/total_tokens=476,928,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2485/500000] + train/ActionL1Loss=0.0954 + throughput/total_tokens=477,120,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2486/500000] + train/ActionL1Loss=0.0707 + throughput/total_tokens=477,312,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2487/500000] + train/ActionL1Loss=0.0957 + throughput/total_tokens=477,504,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2488/500000] + train/ActionL1Loss=0.0943 + throughput/total_tokens=477,696,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2489/500000] + train/ActionL1Loss=0.1078 + throughput/total_tokens=477,888,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2490/500000] + train/ActionL1Loss=0.0853 + throughput/total_tokens=478,080,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2491/500000] + train/ActionL1Loss=0.1065 + throughput/total_tokens=478,272,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2492/500000] + train/ActionL1Loss=0.1010 + throughput/total_tokens=478,464,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2493/500000] + train/ActionL1Loss=0.1181 + throughput/total_tokens=478,656,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2494/500000] + train/ActionL1Loss=0.1035 + throughput/total_tokens=478,848,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2495/500000] + train/ActionL1Loss=0.0990 + throughput/total_tokens=479,040,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2496/500000] + train/ActionL1Loss=0.1041 + throughput/total_tokens=479,232,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2497/500000] + train/ActionL1Loss=0.0851 + throughput/total_tokens=479,424,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2498/500000] + train/ActionL1Loss=0.0765 + throughput/total_tokens=479,616,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2499/500000] + train/ActionL1Loss=0.1007 + throughput/total_tokens=479,808,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2500/500000] + optim/total_grad_norm=8.205 + train/ActionL1Loss=0.1049 + throughput/total_tokens=480,000,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +10/01 [03:56:32] INFO | >> Saving config... checkpoint.py:608 +10/01 [03:57:01] INFO | >> Saving model state... checkpoint.py:796 +10/01 [03:58:12] INFO | >> Saving optim state... checkpoint.py:811 +10/01 [03:59:44] INFO | >> Saving trainer state... checkpoint.py:670 + INFO | >> Saving config... checkpoint.py:608 +[step=2501/500000] + train/ActionL1Loss=0.1201 + throughput/total_tokens=480,192,000 + throughput/device/tokens_per_second=1,211 + throughput/device/batches_per_second=0.0505 +[step=2502/500000] + train/ActionL1Loss=0.0914 + throughput/total_tokens=480,384,000 + throughput/device/tokens_per_second=1,185 + throughput/device/batches_per_second=0.0494 +[step=2503/500000] + train/ActionL1Loss=0.0803 + throughput/total_tokens=480,576,000 + throughput/device/tokens_per_second=1,176 + throughput/device/batches_per_second=0.0490 +[step=2504/500000] + train/ActionL1Loss=0.1051 + throughput/total_tokens=480,768,000 + throughput/device/tokens_per_second=1,174 + throughput/device/batches_per_second=0.0489 +[step=2505/500000] + train/ActionL1Loss=0.1016 + throughput/total_tokens=480,960,000 + throughput/device/tokens_per_second=1,172 + throughput/device/batches_per_second=0.0488 +[step=2506/500000] + train/ActionL1Loss=0.0962 + throughput/total_tokens=481,152,000 + throughput/device/tokens_per_second=1,171 + throughput/device/batches_per_second=0.0488 +[step=2507/500000] + train/ActionL1Loss=0.1212 + throughput/total_tokens=481,344,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=2508/500000] + train/ActionL1Loss=0.0981 + throughput/total_tokens=481,536,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=2509/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=481,728,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=2510/500000] + train/ActionL1Loss=0.0939 + throughput/total_tokens=481,920,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=2511/500000] + train/ActionL1Loss=0.1039 + throughput/total_tokens=482,112,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=2512/500000] + train/ActionL1Loss=0.0933 + throughput/total_tokens=482,304,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=2513/500000] + train/ActionL1Loss=0.1000 + throughput/total_tokens=482,496,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=2514/500000] + train/ActionL1Loss=0.0883 + throughput/total_tokens=482,688,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=2515/500000] + train/ActionL1Loss=0.0836 + throughput/total_tokens=482,880,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=2516/500000] + train/ActionL1Loss=0.1051 + throughput/total_tokens=483,072,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=2517/500000] + train/ActionL1Loss=0.1021 + throughput/total_tokens=483,264,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=2518/500000] + train/ActionL1Loss=0.0957 + throughput/total_tokens=483,456,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=2519/500000] + train/ActionL1Loss=0.0945 + throughput/total_tokens=483,648,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2520/500000] + optim/total_grad_norm=7.614 + train/ActionL1Loss=0.0911 + throughput/total_tokens=483,840,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2521/500000] + train/ActionL1Loss=0.1188 + throughput/total_tokens=484,032,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2522/500000] + train/ActionL1Loss=0.1043 + throughput/total_tokens=484,224,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2523/500000] + train/ActionL1Loss=0.1001 + throughput/total_tokens=484,416,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2524/500000] + train/ActionL1Loss=0.0865 + throughput/total_tokens=484,608,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2525/500000] + train/ActionL1Loss=0.0885 + throughput/total_tokens=484,800,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2526/500000] + train/ActionL1Loss=0.1058 + throughput/total_tokens=484,992,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2527/500000] + train/ActionL1Loss=0.0834 + throughput/total_tokens=485,184,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2528/500000] + train/ActionL1Loss=0.0900 + throughput/total_tokens=485,376,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2529/500000] + train/ActionL1Loss=0.0840 + throughput/total_tokens=485,568,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2530/500000] + train/ActionL1Loss=0.1178 + throughput/total_tokens=485,760,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2531/500000] + train/ActionL1Loss=0.0901 + throughput/total_tokens=485,952,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2532/500000] + train/ActionL1Loss=0.0947 + throughput/total_tokens=486,144,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2533/500000] + train/ActionL1Loss=0.0914 + throughput/total_tokens=486,336,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2534/500000] + train/ActionL1Loss=0.0982 + throughput/total_tokens=486,528,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2535/500000] + train/ActionL1Loss=0.0991 + throughput/total_tokens=486,720,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2536/500000] + train/ActionL1Loss=0.1069 + throughput/total_tokens=486,912,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2537/500000] + train/ActionL1Loss=0.0942 + throughput/total_tokens=487,104,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2538/500000] + train/ActionL1Loss=0.0987 + throughput/total_tokens=487,296,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2539/500000] + train/ActionL1Loss=0.0983 + throughput/total_tokens=487,488,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2540/500000] + optim/total_grad_norm=5.796 + train/ActionL1Loss=0.1192 + throughput/total_tokens=487,680,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2541/500000] + train/ActionL1Loss=0.0881 + throughput/total_tokens=487,872,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2542/500000] + train/ActionL1Loss=0.0937 + throughput/total_tokens=488,064,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2543/500000] + train/ActionL1Loss=0.0909 + throughput/total_tokens=488,256,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2544/500000] + train/ActionL1Loss=0.0927 + throughput/total_tokens=488,448,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2545/500000] + train/ActionL1Loss=0.1167 + throughput/total_tokens=488,640,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2546/500000] + train/ActionL1Loss=0.1306 + throughput/total_tokens=488,832,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2547/500000] + train/ActionL1Loss=0.1027 + throughput/total_tokens=489,024,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2548/500000] + train/ActionL1Loss=0.0964 + throughput/total_tokens=489,216,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2549/500000] + train/ActionL1Loss=0.1026 + throughput/total_tokens=489,408,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2550/500000] + train/ActionL1Loss=0.0849 + throughput/total_tokens=489,600,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2551/500000] + train/ActionL1Loss=0.1130 + throughput/total_tokens=489,792,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2552/500000] + train/ActionL1Loss=0.0979 + throughput/total_tokens=489,984,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2553/500000] + train/ActionL1Loss=0.0847 + throughput/total_tokens=490,176,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2554/500000] + train/ActionL1Loss=0.0983 + throughput/total_tokens=490,368,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2555/500000] + train/ActionL1Loss=0.0960 + throughput/total_tokens=490,560,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2556/500000] + train/ActionL1Loss=0.0929 + throughput/total_tokens=490,752,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2557/500000] + train/ActionL1Loss=0.1013 + throughput/total_tokens=490,944,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2558/500000] + train/ActionL1Loss=0.0995 + throughput/total_tokens=491,136,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2559/500000] + train/ActionL1Loss=0.1014 + throughput/total_tokens=491,328,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2560/500000] + optim/total_grad_norm=5.827 + train/ActionL1Loss=0.0981 + throughput/total_tokens=491,520,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=2561/500000] + train/ActionL1Loss=0.1049 + throughput/total_tokens=491,712,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2562/500000] + train/ActionL1Loss=0.1025 + throughput/total_tokens=491,904,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2563/500000] + train/ActionL1Loss=0.0896 + throughput/total_tokens=492,096,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2564/500000] + train/ActionL1Loss=0.0966 + throughput/total_tokens=492,288,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2565/500000] + train/ActionL1Loss=0.1020 + throughput/total_tokens=492,480,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2566/500000] + train/ActionL1Loss=0.0952 + throughput/total_tokens=492,672,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2567/500000] + train/ActionL1Loss=0.1109 + throughput/total_tokens=492,864,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2568/500000] + train/ActionL1Loss=0.1063 + throughput/total_tokens=493,056,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2569/500000] + train/ActionL1Loss=0.0921 + throughput/total_tokens=493,248,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2570/500000] + train/ActionL1Loss=0.1112 + throughput/total_tokens=493,440,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2571/500000] + train/ActionL1Loss=0.0975 + throughput/total_tokens=493,632,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2572/500000] + train/ActionL1Loss=0.0917 + throughput/total_tokens=493,824,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2573/500000] + train/ActionL1Loss=0.0954 + throughput/total_tokens=494,016,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2574/500000] + train/ActionL1Loss=0.0951 + throughput/total_tokens=494,208,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2575/500000] + train/ActionL1Loss=0.1098 + throughput/total_tokens=494,400,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2576/500000] + train/ActionL1Loss=0.1221 + throughput/total_tokens=494,592,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2577/500000] + train/ActionL1Loss=0.1220 + throughput/total_tokens=494,784,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2578/500000] + train/ActionL1Loss=0.0988 + throughput/total_tokens=494,976,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2579/500000] + train/ActionL1Loss=0.0991 + throughput/total_tokens=495,168,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2580/500000] + optim/total_grad_norm=6.464 + train/ActionL1Loss=0.1159 + throughput/total_tokens=495,360,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2581/500000] + train/ActionL1Loss=0.1135 + throughput/total_tokens=495,552,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2582/500000] + train/ActionL1Loss=0.1261 + throughput/total_tokens=495,744,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=2583/500000] + train/ActionL1Loss=0.1086 + throughput/total_tokens=495,936,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=2584/500000] + train/ActionL1Loss=0.0994 + throughput/total_tokens=496,128,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=2585/500000] + train/ActionL1Loss=0.1112 + throughput/total_tokens=496,320,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2586/500000] + train/ActionL1Loss=0.1048 + throughput/total_tokens=496,512,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2587/500000] + train/ActionL1Loss=0.1071 + throughput/total_tokens=496,704,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2588/500000] + train/ActionL1Loss=0.1180 + throughput/total_tokens=496,896,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2589/500000] + train/ActionL1Loss=0.1240 + throughput/total_tokens=497,088,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2590/500000] + train/ActionL1Loss=0.0947 + throughput/total_tokens=497,280,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2591/500000] + train/ActionL1Loss=0.1091 + throughput/total_tokens=497,472,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2592/500000] + train/ActionL1Loss=0.1080 + throughput/total_tokens=497,664,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2593/500000] + train/ActionL1Loss=0.1034 + throughput/total_tokens=497,856,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2594/500000] + train/ActionL1Loss=0.1084 + throughput/total_tokens=498,048,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2595/500000] + train/ActionL1Loss=0.1156 + throughput/total_tokens=498,240,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2596/500000] + train/ActionL1Loss=0.0962 + throughput/total_tokens=498,432,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2597/500000] + train/ActionL1Loss=0.0843 + throughput/total_tokens=498,624,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2598/500000] + train/ActionL1Loss=0.0997 + throughput/total_tokens=498,816,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2599/500000] + train/ActionL1Loss=0.1186 + throughput/total_tokens=499,008,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2600/500000] + optim/total_grad_norm=7.044 + train/ActionL1Loss=0.1001 + throughput/total_tokens=499,200,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2601/500000] + train/ActionL1Loss=0.0997 + throughput/total_tokens=499,392,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2602/500000] + train/ActionL1Loss=0.1016 + throughput/total_tokens=499,584,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2603/500000] + train/ActionL1Loss=0.0924 + throughput/total_tokens=499,776,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2604/500000] + train/ActionL1Loss=0.1050 + throughput/total_tokens=499,968,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2605/500000] + train/ActionL1Loss=0.0962 + throughput/total_tokens=500,160,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2606/500000] + train/ActionL1Loss=0.1064 + throughput/total_tokens=500,352,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2607/500000] + train/ActionL1Loss=0.0954 + throughput/total_tokens=500,544,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2608/500000] + train/ActionL1Loss=0.1082 + throughput/total_tokens=500,736,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2609/500000] + train/ActionL1Loss=0.1149 + throughput/total_tokens=500,928,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2610/500000] + train/ActionL1Loss=0.0880 + throughput/total_tokens=501,120,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=2611/500000] + train/ActionL1Loss=0.1078 + throughput/total_tokens=501,312,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2612/500000] + train/ActionL1Loss=0.1072 + throughput/total_tokens=501,504,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2613/500000] + train/ActionL1Loss=0.0937 + throughput/total_tokens=501,696,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2614/500000] + train/ActionL1Loss=0.0969 + throughput/total_tokens=501,888,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2615/500000] + train/ActionL1Loss=0.0844 + throughput/total_tokens=502,080,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2616/500000] + train/ActionL1Loss=0.0970 + throughput/total_tokens=502,272,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2617/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=502,464,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2618/500000] + train/ActionL1Loss=0.0881 + throughput/total_tokens=502,656,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2619/500000] + train/ActionL1Loss=0.0949 + throughput/total_tokens=502,848,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2620/500000] + optim/total_grad_norm=6.395 + train/ActionL1Loss=0.0865 + throughput/total_tokens=503,040,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2621/500000] + train/ActionL1Loss=0.0912 + throughput/total_tokens=503,232,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2622/500000] + train/ActionL1Loss=0.1093 + throughput/total_tokens=503,424,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2623/500000] + train/ActionL1Loss=0.1223 + throughput/total_tokens=503,616,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2624/500000] + train/ActionL1Loss=0.0943 + throughput/total_tokens=503,808,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2625/500000] + train/ActionL1Loss=0.1006 + throughput/total_tokens=504,000,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2626/500000] + train/ActionL1Loss=0.1088 + throughput/total_tokens=504,192,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2627/500000] + train/ActionL1Loss=0.0882 + throughput/total_tokens=504,384,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2628/500000] + train/ActionL1Loss=0.1073 + throughput/total_tokens=504,576,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2629/500000] + train/ActionL1Loss=0.1070 + throughput/total_tokens=504,768,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2630/500000] + train/ActionL1Loss=0.1036 + throughput/total_tokens=504,960,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2631/500000] + train/ActionL1Loss=0.1194 + throughput/total_tokens=505,152,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2632/500000] + train/ActionL1Loss=0.1146 + throughput/total_tokens=505,344,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2633/500000] + train/ActionL1Loss=0.1045 + throughput/total_tokens=505,536,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2634/500000] + train/ActionL1Loss=0.0901 + throughput/total_tokens=505,728,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2635/500000] + train/ActionL1Loss=0.1002 + throughput/total_tokens=505,920,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2636/500000] + train/ActionL1Loss=0.1082 + throughput/total_tokens=506,112,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2637/500000] + train/ActionL1Loss=0.0852 + throughput/total_tokens=506,304,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2638/500000] + train/ActionL1Loss=0.1094 + throughput/total_tokens=506,496,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2639/500000] + train/ActionL1Loss=0.0751 + throughput/total_tokens=506,688,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2640/500000] + optim/total_grad_norm=8.300 + train/ActionL1Loss=0.0902 + throughput/total_tokens=506,880,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2641/500000] + train/ActionL1Loss=0.0955 + throughput/total_tokens=507,072,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2642/500000] + train/ActionL1Loss=0.1047 + throughput/total_tokens=507,264,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2643/500000] + train/ActionL1Loss=0.0911 + throughput/total_tokens=507,456,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2644/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=507,648,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2645/500000] + train/ActionL1Loss=0.0930 + throughput/total_tokens=507,840,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2646/500000] + train/ActionL1Loss=0.1043 + throughput/total_tokens=508,032,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2647/500000] + train/ActionL1Loss=0.0944 + throughput/total_tokens=508,224,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2648/500000] + train/ActionL1Loss=0.1059 + throughput/total_tokens=508,416,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2649/500000] + train/ActionL1Loss=0.1057 + throughput/total_tokens=508,608,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2650/500000] + train/ActionL1Loss=0.0989 + throughput/total_tokens=508,800,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2651/500000] + train/ActionL1Loss=0.1196 + throughput/total_tokens=508,992,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2652/500000] + train/ActionL1Loss=0.1181 + throughput/total_tokens=509,184,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2653/500000] + train/ActionL1Loss=0.1110 + throughput/total_tokens=509,376,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2654/500000] + train/ActionL1Loss=0.1014 + throughput/total_tokens=509,568,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2655/500000] + train/ActionL1Loss=0.0885 + throughput/total_tokens=509,760,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2656/500000] + train/ActionL1Loss=0.1028 + throughput/total_tokens=509,952,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2657/500000] + train/ActionL1Loss=0.0866 + throughput/total_tokens=510,144,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2658/500000] + train/ActionL1Loss=0.0896 + throughput/total_tokens=510,336,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2659/500000] + train/ActionL1Loss=0.0873 + throughput/total_tokens=510,528,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2660/500000] + optim/total_grad_norm=7.744 + train/ActionL1Loss=0.0933 + throughput/total_tokens=510,720,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=2661/500000] + train/ActionL1Loss=0.0964 + throughput/total_tokens=510,912,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2662/500000] + train/ActionL1Loss=0.1042 + throughput/total_tokens=511,104,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2663/500000] + train/ActionL1Loss=0.1023 + throughput/total_tokens=511,296,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2664/500000] + train/ActionL1Loss=0.0898 + throughput/total_tokens=511,488,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2665/500000] + train/ActionL1Loss=0.0898 + throughput/total_tokens=511,680,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2666/500000] + train/ActionL1Loss=0.0966 + throughput/total_tokens=511,872,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2667/500000] + train/ActionL1Loss=0.1078 + throughput/total_tokens=512,064,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2668/500000] + train/ActionL1Loss=0.1070 + throughput/total_tokens=512,256,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2669/500000] + train/ActionL1Loss=0.1081 + throughput/total_tokens=512,448,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2670/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=512,640,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2671/500000] + train/ActionL1Loss=0.0853 + throughput/total_tokens=512,832,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2672/500000] + train/ActionL1Loss=0.0836 + throughput/total_tokens=513,024,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2673/500000] + train/ActionL1Loss=0.0934 + throughput/total_tokens=513,216,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2674/500000] + train/ActionL1Loss=0.1059 + throughput/total_tokens=513,408,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2675/500000] + train/ActionL1Loss=0.1044 + throughput/total_tokens=513,600,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2676/500000] + train/ActionL1Loss=0.0914 + throughput/total_tokens=513,792,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2677/500000] + train/ActionL1Loss=0.0894 + throughput/total_tokens=513,984,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2678/500000] + train/ActionL1Loss=0.1066 + throughput/total_tokens=514,176,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2679/500000] + train/ActionL1Loss=0.0962 + throughput/total_tokens=514,368,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2680/500000] + optim/total_grad_norm=5.150 + train/ActionL1Loss=0.0920 + throughput/total_tokens=514,560,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2681/500000] + train/ActionL1Loss=0.1164 + throughput/total_tokens=514,752,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2682/500000] + train/ActionL1Loss=0.1027 + throughput/total_tokens=514,944,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2683/500000] + train/ActionL1Loss=0.1025 + throughput/total_tokens=515,136,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2684/500000] + train/ActionL1Loss=0.1134 + throughput/total_tokens=515,328,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2685/500000] + train/ActionL1Loss=0.0863 + throughput/total_tokens=515,520,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2686/500000] + train/ActionL1Loss=0.0894 + throughput/total_tokens=515,712,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2687/500000] + train/ActionL1Loss=0.1142 + throughput/total_tokens=515,904,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2688/500000] + train/ActionL1Loss=0.0971 + throughput/total_tokens=516,096,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2689/500000] + train/ActionL1Loss=0.1011 + throughput/total_tokens=516,288,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2690/500000] + train/ActionL1Loss=0.0908 + throughput/total_tokens=516,480,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2691/500000] + train/ActionL1Loss=0.0975 + throughput/total_tokens=516,672,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2692/500000] + train/ActionL1Loss=0.0865 + throughput/total_tokens=516,864,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2693/500000] + train/ActionL1Loss=0.0927 + throughput/total_tokens=517,056,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2694/500000] + train/ActionL1Loss=0.0916 + throughput/total_tokens=517,248,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2695/500000] + train/ActionL1Loss=0.1090 + throughput/total_tokens=517,440,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2696/500000] + train/ActionL1Loss=0.0955 + throughput/total_tokens=517,632,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2697/500000] + train/ActionL1Loss=0.0824 + throughput/total_tokens=517,824,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2698/500000] + train/ActionL1Loss=0.0856 + throughput/total_tokens=518,016,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2699/500000] + train/ActionL1Loss=0.0779 + throughput/total_tokens=518,208,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2700/500000] + optim/total_grad_norm=5.884 + train/ActionL1Loss=0.0948 + throughput/total_tokens=518,400,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2701/500000] + train/ActionL1Loss=0.0746 + throughput/total_tokens=518,592,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2702/500000] + train/ActionL1Loss=0.1015 + throughput/total_tokens=518,784,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2703/500000] + train/ActionL1Loss=0.0969 + throughput/total_tokens=518,976,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2704/500000] + train/ActionL1Loss=0.0901 + throughput/total_tokens=519,168,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2705/500000] + train/ActionL1Loss=0.0886 + throughput/total_tokens=519,360,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2706/500000] + train/ActionL1Loss=0.1047 + throughput/total_tokens=519,552,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2707/500000] + train/ActionL1Loss=0.0872 + throughput/total_tokens=519,744,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2708/500000] + train/ActionL1Loss=0.1208 + throughput/total_tokens=519,936,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2709/500000] + train/ActionL1Loss=0.0832 + throughput/total_tokens=520,128,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2710/500000] + train/ActionL1Loss=0.0967 + throughput/total_tokens=520,320,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2711/500000] + train/ActionL1Loss=0.0797 + throughput/total_tokens=520,512,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2712/500000] + train/ActionL1Loss=0.1157 + throughput/total_tokens=520,704,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2713/500000] + train/ActionL1Loss=0.1024 + throughput/total_tokens=520,896,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2714/500000] + train/ActionL1Loss=0.0906 + throughput/total_tokens=521,088,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2715/500000] + train/ActionL1Loss=0.1082 + throughput/total_tokens=521,280,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2716/500000] + train/ActionL1Loss=0.1143 + throughput/total_tokens=521,472,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2717/500000] + train/ActionL1Loss=0.0994 + throughput/total_tokens=521,664,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2718/500000] + train/ActionL1Loss=0.0863 + throughput/total_tokens=521,856,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2719/500000] + train/ActionL1Loss=0.0893 + throughput/total_tokens=522,048,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2720/500000] + optim/total_grad_norm=6.687 + train/ActionL1Loss=0.1164 + throughput/total_tokens=522,240,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2721/500000] + train/ActionL1Loss=0.1004 + throughput/total_tokens=522,432,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2722/500000] + train/ActionL1Loss=0.1066 + throughput/total_tokens=522,624,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2723/500000] + train/ActionL1Loss=0.0862 + throughput/total_tokens=522,816,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2724/500000] + train/ActionL1Loss=0.1049 + throughput/total_tokens=523,008,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2725/500000] + train/ActionL1Loss=0.1102 + throughput/total_tokens=523,200,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2726/500000] + train/ActionL1Loss=0.0971 + throughput/total_tokens=523,392,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2727/500000] + train/ActionL1Loss=0.1034 + throughput/total_tokens=523,584,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2728/500000] + train/ActionL1Loss=0.0827 + throughput/total_tokens=523,776,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2729/500000] + train/ActionL1Loss=0.1072 + throughput/total_tokens=523,968,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2730/500000] + train/ActionL1Loss=0.0939 + throughput/total_tokens=524,160,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2731/500000] + train/ActionL1Loss=0.0913 + throughput/total_tokens=524,352,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2732/500000] + train/ActionL1Loss=0.0879 + throughput/total_tokens=524,544,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2733/500000] + train/ActionL1Loss=0.0851 + throughput/total_tokens=524,736,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2734/500000] + train/ActionL1Loss=0.1041 + throughput/total_tokens=524,928,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2735/500000] + train/ActionL1Loss=0.0747 + throughput/total_tokens=525,120,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2736/500000] + train/ActionL1Loss=0.0831 + throughput/total_tokens=525,312,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2737/500000] + train/ActionL1Loss=0.0916 + throughput/total_tokens=525,504,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2738/500000] + train/ActionL1Loss=0.0997 + throughput/total_tokens=525,696,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2739/500000] + train/ActionL1Loss=0.1090 + throughput/total_tokens=525,888,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2740/500000] + optim/total_grad_norm=7.723 + train/ActionL1Loss=0.0961 + throughput/total_tokens=526,080,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2741/500000] + train/ActionL1Loss=0.1123 + throughput/total_tokens=526,272,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2742/500000] + train/ActionL1Loss=0.1049 + throughput/total_tokens=526,464,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2743/500000] + train/ActionL1Loss=0.0965 + throughput/total_tokens=526,656,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2744/500000] + train/ActionL1Loss=0.1089 + throughput/total_tokens=526,848,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2745/500000] + train/ActionL1Loss=0.0976 + throughput/total_tokens=527,040,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2746/500000] + train/ActionL1Loss=0.0918 + throughput/total_tokens=527,232,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2747/500000] + train/ActionL1Loss=0.0804 + throughput/total_tokens=527,424,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2748/500000] + train/ActionL1Loss=0.0716 + throughput/total_tokens=527,616,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2749/500000] + train/ActionL1Loss=0.1046 + throughput/total_tokens=527,808,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2750/500000] + train/ActionL1Loss=0.0952 + throughput/total_tokens=528,000,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2751/500000] + train/ActionL1Loss=0.0831 + throughput/total_tokens=528,192,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2752/500000] + train/ActionL1Loss=0.0891 + throughput/total_tokens=528,384,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2753/500000] + train/ActionL1Loss=0.0930 + throughput/total_tokens=528,576,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2754/500000] + train/ActionL1Loss=0.0857 + throughput/total_tokens=528,768,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2755/500000] + train/ActionL1Loss=0.0988 + throughput/total_tokens=528,960,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2756/500000] + train/ActionL1Loss=0.1265 + throughput/total_tokens=529,152,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2757/500000] + train/ActionL1Loss=0.1178 + throughput/total_tokens=529,344,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2758/500000] + train/ActionL1Loss=0.0868 + throughput/total_tokens=529,536,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2759/500000] + train/ActionL1Loss=0.1059 + throughput/total_tokens=529,728,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2760/500000] + optim/total_grad_norm=6.502 + train/ActionL1Loss=0.0906 + throughput/total_tokens=529,920,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2761/500000] + train/ActionL1Loss=0.1011 + throughput/total_tokens=530,112,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2762/500000] + train/ActionL1Loss=0.1167 + throughput/total_tokens=530,304,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2763/500000] + train/ActionL1Loss=0.1030 + throughput/total_tokens=530,496,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2764/500000] + train/ActionL1Loss=0.1038 + throughput/total_tokens=530,688,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2765/500000] + train/ActionL1Loss=0.0861 + throughput/total_tokens=530,880,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2766/500000] + train/ActionL1Loss=0.0785 + throughput/total_tokens=531,072,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2767/500000] + train/ActionL1Loss=0.0856 + throughput/total_tokens=531,264,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2768/500000] + train/ActionL1Loss=0.0929 + throughput/total_tokens=531,456,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2769/500000] + train/ActionL1Loss=0.0807 + throughput/total_tokens=531,648,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2770/500000] + train/ActionL1Loss=0.1127 + throughput/total_tokens=531,840,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2771/500000] + train/ActionL1Loss=0.0943 + throughput/total_tokens=532,032,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2772/500000] + train/ActionL1Loss=0.0920 + throughput/total_tokens=532,224,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2773/500000] + train/ActionL1Loss=0.0985 + throughput/total_tokens=532,416,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2774/500000] + train/ActionL1Loss=0.0921 + throughput/total_tokens=532,608,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2775/500000] + train/ActionL1Loss=0.0851 + throughput/total_tokens=532,800,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2776/500000] + train/ActionL1Loss=0.1236 + throughput/total_tokens=532,992,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2777/500000] + train/ActionL1Loss=0.1065 + throughput/total_tokens=533,184,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2778/500000] + train/ActionL1Loss=0.1056 + throughput/total_tokens=533,376,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2779/500000] + train/ActionL1Loss=0.1076 + throughput/total_tokens=533,568,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2780/500000] + optim/total_grad_norm=5.569 + train/ActionL1Loss=0.1082 + throughput/total_tokens=533,760,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2781/500000] + train/ActionL1Loss=0.1184 + throughput/total_tokens=533,952,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2782/500000] + train/ActionL1Loss=0.0781 + throughput/total_tokens=534,144,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2783/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=534,336,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2784/500000] + train/ActionL1Loss=0.0930 + throughput/total_tokens=534,528,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2785/500000] + train/ActionL1Loss=0.1108 + throughput/total_tokens=534,720,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2786/500000] + train/ActionL1Loss=0.1116 + throughput/total_tokens=534,912,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2787/500000] + train/ActionL1Loss=0.1175 + throughput/total_tokens=535,104,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2788/500000] + train/ActionL1Loss=0.1126 + throughput/total_tokens=535,296,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2789/500000] + train/ActionL1Loss=0.1034 + throughput/total_tokens=535,488,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2790/500000] + train/ActionL1Loss=0.0919 + throughput/total_tokens=535,680,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2791/500000] + train/ActionL1Loss=0.0846 + throughput/total_tokens=535,872,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2792/500000] + train/ActionL1Loss=0.0930 + throughput/total_tokens=536,064,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2793/500000] + train/ActionL1Loss=0.0880 + throughput/total_tokens=536,256,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2794/500000] + train/ActionL1Loss=0.0824 + throughput/total_tokens=536,448,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2795/500000] + train/ActionL1Loss=0.0870 + throughput/total_tokens=536,640,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2796/500000] + train/ActionL1Loss=0.0970 + throughput/total_tokens=536,832,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2797/500000] + train/ActionL1Loss=0.0834 + throughput/total_tokens=537,024,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2798/500000] + train/ActionL1Loss=0.0975 + throughput/total_tokens=537,216,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2799/500000] + train/ActionL1Loss=0.1079 + throughput/total_tokens=537,408,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2800/500000] + optim/total_grad_norm=5.507 + train/ActionL1Loss=0.1072 + throughput/total_tokens=537,600,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2801/500000] + train/ActionL1Loss=0.0805 + throughput/total_tokens=537,792,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2802/500000] + train/ActionL1Loss=0.0833 + throughput/total_tokens=537,984,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2803/500000] + train/ActionL1Loss=0.0856 + throughput/total_tokens=538,176,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2804/500000] + train/ActionL1Loss=0.0773 + throughput/total_tokens=538,368,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2805/500000] + train/ActionL1Loss=0.0870 + throughput/total_tokens=538,560,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2806/500000] + train/ActionL1Loss=0.0794 + throughput/total_tokens=538,752,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2807/500000] + train/ActionL1Loss=0.0880 + throughput/total_tokens=538,944,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2808/500000] + train/ActionL1Loss=0.1193 + throughput/total_tokens=539,136,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2809/500000] + train/ActionL1Loss=0.0910 + throughput/total_tokens=539,328,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2810/500000] + train/ActionL1Loss=0.0973 + throughput/total_tokens=539,520,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2811/500000] + train/ActionL1Loss=0.1021 + throughput/total_tokens=539,712,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2812/500000] + train/ActionL1Loss=0.0988 + throughput/total_tokens=539,904,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2813/500000] + train/ActionL1Loss=0.0874 + throughput/total_tokens=540,096,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2814/500000] + train/ActionL1Loss=0.1079 + throughput/total_tokens=540,288,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2815/500000] + train/ActionL1Loss=0.1048 + throughput/total_tokens=540,480,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2816/500000] + train/ActionL1Loss=0.1028 + throughput/total_tokens=540,672,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2817/500000] + train/ActionL1Loss=0.0858 + throughput/total_tokens=540,864,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2818/500000] + train/ActionL1Loss=0.1047 + throughput/total_tokens=541,056,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2819/500000] + train/ActionL1Loss=0.1063 + throughput/total_tokens=541,248,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2820/500000] + optim/total_grad_norm=6.334 + train/ActionL1Loss=0.0893 + throughput/total_tokens=541,440,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2821/500000] + train/ActionL1Loss=0.1068 + throughput/total_tokens=541,632,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2822/500000] + train/ActionL1Loss=0.1026 + throughput/total_tokens=541,824,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2823/500000] + train/ActionL1Loss=0.0918 + throughput/total_tokens=542,016,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2824/500000] + train/ActionL1Loss=0.0965 + throughput/total_tokens=542,208,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2825/500000] + train/ActionL1Loss=0.0863 + throughput/total_tokens=542,400,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2826/500000] + train/ActionL1Loss=0.0949 + throughput/total_tokens=542,592,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2827/500000] + train/ActionL1Loss=0.0929 + throughput/total_tokens=542,784,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2828/500000] + train/ActionL1Loss=0.1132 + throughput/total_tokens=542,976,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2829/500000] + train/ActionL1Loss=0.0828 + throughput/total_tokens=543,168,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2830/500000] + train/ActionL1Loss=0.0891 + throughput/total_tokens=543,360,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2831/500000] + train/ActionL1Loss=0.0859 + throughput/total_tokens=543,552,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2832/500000] + train/ActionL1Loss=0.0998 + throughput/total_tokens=543,744,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2833/500000] + train/ActionL1Loss=0.1090 + throughput/total_tokens=543,936,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2834/500000] + train/ActionL1Loss=0.1069 + throughput/total_tokens=544,128,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2835/500000] + train/ActionL1Loss=0.0819 + throughput/total_tokens=544,320,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2836/500000] + train/ActionL1Loss=0.1127 + throughput/total_tokens=544,512,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2837/500000] + train/ActionL1Loss=0.0971 + throughput/total_tokens=544,704,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2838/500000] + train/ActionL1Loss=0.1126 + throughput/total_tokens=544,896,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2839/500000] + train/ActionL1Loss=0.0753 + throughput/total_tokens=545,088,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2840/500000] + optim/total_grad_norm=6.413 + train/ActionL1Loss=0.1156 + throughput/total_tokens=545,280,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2841/500000] + train/ActionL1Loss=0.1051 + throughput/total_tokens=545,472,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2842/500000] + train/ActionL1Loss=0.1093 + throughput/total_tokens=545,664,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2843/500000] + train/ActionL1Loss=0.0846 + throughput/total_tokens=545,856,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2844/500000] + train/ActionL1Loss=0.0830 + throughput/total_tokens=546,048,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2845/500000] + train/ActionL1Loss=0.0871 + throughput/total_tokens=546,240,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2846/500000] + train/ActionL1Loss=0.0897 + throughput/total_tokens=546,432,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2847/500000] + train/ActionL1Loss=0.0785 + throughput/total_tokens=546,624,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2848/500000] + train/ActionL1Loss=0.0843 + throughput/total_tokens=546,816,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2849/500000] + train/ActionL1Loss=0.1164 + throughput/total_tokens=547,008,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2850/500000] + train/ActionL1Loss=0.1141 + throughput/total_tokens=547,200,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2851/500000] + train/ActionL1Loss=0.0896 + throughput/total_tokens=547,392,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2852/500000] + train/ActionL1Loss=0.0914 + throughput/total_tokens=547,584,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2853/500000] + train/ActionL1Loss=0.0843 + throughput/total_tokens=547,776,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2854/500000] + train/ActionL1Loss=0.1119 + throughput/total_tokens=547,968,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2855/500000] + train/ActionL1Loss=0.0939 + throughput/total_tokens=548,160,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2856/500000] + train/ActionL1Loss=0.0931 + throughput/total_tokens=548,352,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2857/500000] + train/ActionL1Loss=0.1053 + throughput/total_tokens=548,544,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2858/500000] + train/ActionL1Loss=0.0958 + throughput/total_tokens=548,736,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2859/500000] + train/ActionL1Loss=0.0891 + throughput/total_tokens=548,928,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2860/500000] + optim/total_grad_norm=6.253 + train/ActionL1Loss=0.0865 + throughput/total_tokens=549,120,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2861/500000] + train/ActionL1Loss=0.1045 + throughput/total_tokens=549,312,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2862/500000] + train/ActionL1Loss=0.0950 + throughput/total_tokens=549,504,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2863/500000] + train/ActionL1Loss=0.1023 + throughput/total_tokens=549,696,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2864/500000] + train/ActionL1Loss=0.0977 + throughput/total_tokens=549,888,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2865/500000] + train/ActionL1Loss=0.0868 + throughput/total_tokens=550,080,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2866/500000] + train/ActionL1Loss=0.1061 + throughput/total_tokens=550,272,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2867/500000] + train/ActionL1Loss=0.0954 + throughput/total_tokens=550,464,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2868/500000] + train/ActionL1Loss=0.1011 + throughput/total_tokens=550,656,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2869/500000] + train/ActionL1Loss=0.1053 + throughput/total_tokens=550,848,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2870/500000] + train/ActionL1Loss=0.1120 + throughput/total_tokens=551,040,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2871/500000] + train/ActionL1Loss=0.0967 + throughput/total_tokens=551,232,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2872/500000] + train/ActionL1Loss=0.1009 + throughput/total_tokens=551,424,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2873/500000] + train/ActionL1Loss=0.0997 + throughput/total_tokens=551,616,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2874/500000] + train/ActionL1Loss=0.1017 + throughput/total_tokens=551,808,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2875/500000] + train/ActionL1Loss=0.1092 + throughput/total_tokens=552,000,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2876/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=552,192,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2877/500000] + train/ActionL1Loss=0.1031 + throughput/total_tokens=552,384,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2878/500000] + train/ActionL1Loss=0.1015 + throughput/total_tokens=552,576,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2879/500000] + train/ActionL1Loss=0.0882 + throughput/total_tokens=552,768,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2880/500000] + optim/total_grad_norm=5.214 + train/ActionL1Loss=0.0913 + throughput/total_tokens=552,960,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2881/500000] + train/ActionL1Loss=0.0849 + throughput/total_tokens=553,152,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2882/500000] + train/ActionL1Loss=0.0875 + throughput/total_tokens=553,344,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2883/500000] + train/ActionL1Loss=0.0709 + throughput/total_tokens=553,536,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2884/500000] + train/ActionL1Loss=0.0809 + throughput/total_tokens=553,728,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2885/500000] + train/ActionL1Loss=0.1020 + throughput/total_tokens=553,920,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2886/500000] + train/ActionL1Loss=0.0982 + throughput/total_tokens=554,112,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2887/500000] + train/ActionL1Loss=0.0871 + throughput/total_tokens=554,304,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2888/500000] + train/ActionL1Loss=0.0944 + throughput/total_tokens=554,496,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2889/500000] + train/ActionL1Loss=0.0791 + throughput/total_tokens=554,688,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2890/500000] + train/ActionL1Loss=0.0754 + throughput/total_tokens=554,880,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2891/500000] + train/ActionL1Loss=0.0998 + throughput/total_tokens=555,072,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2892/500000] + train/ActionL1Loss=0.1019 + throughput/total_tokens=555,264,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2893/500000] + train/ActionL1Loss=0.0890 + throughput/total_tokens=555,456,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2894/500000] + train/ActionL1Loss=0.0879 + throughput/total_tokens=555,648,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2895/500000] + train/ActionL1Loss=0.0849 + throughput/total_tokens=555,840,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2896/500000] + train/ActionL1Loss=0.0860 + throughput/total_tokens=556,032,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2897/500000] + train/ActionL1Loss=0.0845 + throughput/total_tokens=556,224,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2898/500000] + train/ActionL1Loss=0.0853 + throughput/total_tokens=556,416,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2899/500000] + train/ActionL1Loss=0.0791 + throughput/total_tokens=556,608,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2900/500000] + optim/total_grad_norm=5.524 + train/ActionL1Loss=0.1058 + throughput/total_tokens=556,800,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2901/500000] + train/ActionL1Loss=0.1136 + throughput/total_tokens=556,992,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2902/500000] + train/ActionL1Loss=0.0902 + throughput/total_tokens=557,184,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=2903/500000] + train/ActionL1Loss=0.0728 + throughput/total_tokens=557,376,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=2904/500000] + train/ActionL1Loss=0.1016 + throughput/total_tokens=557,568,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=2905/500000] + train/ActionL1Loss=0.0830 + throughput/total_tokens=557,760,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=2906/500000] + train/ActionL1Loss=0.0798 + throughput/total_tokens=557,952,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=2907/500000] + train/ActionL1Loss=0.0752 + throughput/total_tokens=558,144,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2908/500000] + train/ActionL1Loss=0.0911 + throughput/total_tokens=558,336,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2909/500000] + train/ActionL1Loss=0.0787 + throughput/total_tokens=558,528,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2910/500000] + train/ActionL1Loss=0.1168 + throughput/total_tokens=558,720,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2911/500000] + train/ActionL1Loss=0.0937 + throughput/total_tokens=558,912,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2912/500000] + train/ActionL1Loss=0.0847 + throughput/total_tokens=559,104,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2913/500000] + train/ActionL1Loss=0.0907 + throughput/total_tokens=559,296,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2914/500000] + train/ActionL1Loss=0.1011 + throughput/total_tokens=559,488,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2915/500000] + train/ActionL1Loss=0.0841 + throughput/total_tokens=559,680,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2916/500000] + train/ActionL1Loss=0.1126 + throughput/total_tokens=559,872,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2917/500000] + train/ActionL1Loss=0.0888 + throughput/total_tokens=560,064,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2918/500000] + train/ActionL1Loss=0.0959 + throughput/total_tokens=560,256,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2919/500000] + train/ActionL1Loss=0.1017 + throughput/total_tokens=560,448,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=2920/500000] + optim/total_grad_norm=6.317 + train/ActionL1Loss=0.0952 + throughput/total_tokens=560,640,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2921/500000] + train/ActionL1Loss=0.0968 + throughput/total_tokens=560,832,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2922/500000] + train/ActionL1Loss=0.1018 + throughput/total_tokens=561,024,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2923/500000] + train/ActionL1Loss=0.0907 + throughput/total_tokens=561,216,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2924/500000] + train/ActionL1Loss=0.0952 + throughput/total_tokens=561,408,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2925/500000] + train/ActionL1Loss=0.1070 + throughput/total_tokens=561,600,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2926/500000] + train/ActionL1Loss=0.0921 + throughput/total_tokens=561,792,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2927/500000] + train/ActionL1Loss=0.0788 + throughput/total_tokens=561,984,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2928/500000] + train/ActionL1Loss=0.0927 + throughput/total_tokens=562,176,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2929/500000] + train/ActionL1Loss=0.0948 + throughput/total_tokens=562,368,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2930/500000] + train/ActionL1Loss=0.0879 + throughput/total_tokens=562,560,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2931/500000] + train/ActionL1Loss=0.0857 + throughput/total_tokens=562,752,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2932/500000] + train/ActionL1Loss=0.1134 + throughput/total_tokens=562,944,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2933/500000] + train/ActionL1Loss=0.1144 + throughput/total_tokens=563,136,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2934/500000] + train/ActionL1Loss=0.0846 + throughput/total_tokens=563,328,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2935/500000] + train/ActionL1Loss=0.1066 + throughput/total_tokens=563,520,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2936/500000] + train/ActionL1Loss=0.1043 + throughput/total_tokens=563,712,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2937/500000] + train/ActionL1Loss=0.0920 + throughput/total_tokens=563,904,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2938/500000] + train/ActionL1Loss=0.0993 + throughput/total_tokens=564,096,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2939/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=564,288,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2940/500000] + optim/total_grad_norm=6.445 + train/ActionL1Loss=0.0889 + throughput/total_tokens=564,480,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2941/500000] + train/ActionL1Loss=0.1018 + throughput/total_tokens=564,672,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2942/500000] + train/ActionL1Loss=0.0934 + throughput/total_tokens=564,864,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2943/500000] + train/ActionL1Loss=0.0933 + throughput/total_tokens=565,056,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2944/500000] + train/ActionL1Loss=0.0927 + throughput/total_tokens=565,248,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2945/500000] + train/ActionL1Loss=0.0885 + throughput/total_tokens=565,440,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2946/500000] + train/ActionL1Loss=0.1027 + throughput/total_tokens=565,632,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2947/500000] + train/ActionL1Loss=0.0929 + throughput/total_tokens=565,824,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2948/500000] + train/ActionL1Loss=0.0907 + throughput/total_tokens=566,016,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2949/500000] + train/ActionL1Loss=0.1019 + throughput/total_tokens=566,208,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2950/500000] + train/ActionL1Loss=0.1171 + throughput/total_tokens=566,400,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2951/500000] + train/ActionL1Loss=0.1013 + throughput/total_tokens=566,592,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2952/500000] + train/ActionL1Loss=0.1041 + throughput/total_tokens=566,784,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2953/500000] + train/ActionL1Loss=0.0951 + throughput/total_tokens=566,976,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2954/500000] + train/ActionL1Loss=0.1083 + throughput/total_tokens=567,168,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2955/500000] + train/ActionL1Loss=0.1138 + throughput/total_tokens=567,360,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=2956/500000] + train/ActionL1Loss=0.0862 + throughput/total_tokens=567,552,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=2957/500000] + train/ActionL1Loss=0.0845 + throughput/total_tokens=567,744,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2958/500000] + train/ActionL1Loss=0.0873 + throughput/total_tokens=567,936,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2959/500000] + train/ActionL1Loss=0.0913 + throughput/total_tokens=568,128,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2960/500000] + optim/total_grad_norm=7.066 + train/ActionL1Loss=0.0813 + throughput/total_tokens=568,320,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2961/500000] + train/ActionL1Loss=0.0908 + throughput/total_tokens=568,512,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2962/500000] + train/ActionL1Loss=0.0913 + throughput/total_tokens=568,704,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2963/500000] + train/ActionL1Loss=0.0929 + throughput/total_tokens=568,896,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2964/500000] + train/ActionL1Loss=0.0966 + throughput/total_tokens=569,088,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2965/500000] + train/ActionL1Loss=0.1012 + throughput/total_tokens=569,280,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2966/500000] + train/ActionL1Loss=0.1047 + throughput/total_tokens=569,472,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2967/500000] + train/ActionL1Loss=0.0840 + throughput/total_tokens=569,664,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2968/500000] + train/ActionL1Loss=0.0942 + throughput/total_tokens=569,856,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2969/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=570,048,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2970/500000] + train/ActionL1Loss=0.0889 + throughput/total_tokens=570,240,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=2971/500000] + train/ActionL1Loss=0.0862 + throughput/total_tokens=570,432,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2972/500000] + train/ActionL1Loss=0.1150 + throughput/total_tokens=570,624,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2973/500000] + train/ActionL1Loss=0.0971 + throughput/total_tokens=570,816,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2974/500000] + train/ActionL1Loss=0.0980 + throughput/total_tokens=571,008,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=2975/500000] + train/ActionL1Loss=0.0826 + throughput/total_tokens=571,200,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2976/500000] + train/ActionL1Loss=0.0940 + throughput/total_tokens=571,392,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2977/500000] + train/ActionL1Loss=0.1044 + throughput/total_tokens=571,584,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2978/500000] + train/ActionL1Loss=0.0910 + throughput/total_tokens=571,776,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2979/500000] + train/ActionL1Loss=0.0946 + throughput/total_tokens=571,968,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2980/500000] + optim/total_grad_norm=6.865 + train/ActionL1Loss=0.0900 + throughput/total_tokens=572,160,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2981/500000] + train/ActionL1Loss=0.0845 + throughput/total_tokens=572,352,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2982/500000] + train/ActionL1Loss=0.0798 + throughput/total_tokens=572,544,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2983/500000] + train/ActionL1Loss=0.0956 + throughput/total_tokens=572,736,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2984/500000] + train/ActionL1Loss=0.0924 + throughput/total_tokens=572,928,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=2985/500000] + train/ActionL1Loss=0.1043 + throughput/total_tokens=573,120,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2986/500000] + train/ActionL1Loss=0.0940 + throughput/total_tokens=573,312,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2987/500000] + train/ActionL1Loss=0.0891 + throughput/total_tokens=573,504,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2988/500000] + train/ActionL1Loss=0.1009 + throughput/total_tokens=573,696,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2989/500000] + train/ActionL1Loss=0.1136 + throughput/total_tokens=573,888,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2990/500000] + train/ActionL1Loss=0.0880 + throughput/total_tokens=574,080,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=2991/500000] + train/ActionL1Loss=0.0986 + throughput/total_tokens=574,272,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2992/500000] + train/ActionL1Loss=0.1004 + throughput/total_tokens=574,464,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=2993/500000] + train/ActionL1Loss=0.0906 + throughput/total_tokens=574,656,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2994/500000] + train/ActionL1Loss=0.0824 + throughput/total_tokens=574,848,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=2995/500000] + train/ActionL1Loss=0.0908 + throughput/total_tokens=575,040,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2996/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=575,232,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2997/500000] + train/ActionL1Loss=0.0867 + throughput/total_tokens=575,424,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2998/500000] + train/ActionL1Loss=0.0794 + throughput/total_tokens=575,616,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=2999/500000] + train/ActionL1Loss=0.0873 + throughput/total_tokens=575,808,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3000/500000] + optim/total_grad_norm=4.844 + train/ActionL1Loss=0.0911 + throughput/total_tokens=576,000,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +10/01 [06:53:16] INFO | >> Saving config... checkpoint.py:608 +10/01 [06:53:57] INFO | >> Saving model state... checkpoint.py:796 +10/01 [06:55:05] INFO | >> Saving optim state... checkpoint.py:811 +10/01 [06:56:37] INFO | >> Saving trainer state... checkpoint.py:670 + INFO | >> Saving config... checkpoint.py:608 +[step=3001/500000] + train/ActionL1Loss=0.0752 + throughput/total_tokens=576,192,000 + throughput/device/tokens_per_second=1,208 + throughput/device/batches_per_second=0.0504 +[step=3002/500000] + train/ActionL1Loss=0.0892 + throughput/total_tokens=576,384,000 + throughput/device/tokens_per_second=1,175 + throughput/device/batches_per_second=0.0490 +[step=3003/500000] + train/ActionL1Loss=0.1075 + throughput/total_tokens=576,576,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=3004/500000] + train/ActionL1Loss=0.0805 + throughput/total_tokens=576,768,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=3005/500000] + train/ActionL1Loss=0.0759 + throughput/total_tokens=576,960,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=3006/500000] + train/ActionL1Loss=0.0850 + throughput/total_tokens=577,152,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3007/500000] + train/ActionL1Loss=0.0873 + throughput/total_tokens=577,344,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3008/500000] + train/ActionL1Loss=0.0683 + throughput/total_tokens=577,536,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3009/500000] + train/ActionL1Loss=0.0990 + throughput/total_tokens=577,728,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3010/500000] + train/ActionL1Loss=0.0930 + throughput/total_tokens=577,920,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3011/500000] + train/ActionL1Loss=0.0932 + throughput/total_tokens=578,112,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3012/500000] + train/ActionL1Loss=0.0989 + throughput/total_tokens=578,304,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3013/500000] + train/ActionL1Loss=0.0931 + throughput/total_tokens=578,496,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3014/500000] + train/ActionL1Loss=0.0885 + throughput/total_tokens=578,688,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3015/500000] + train/ActionL1Loss=0.0963 + throughput/total_tokens=578,880,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3016/500000] + train/ActionL1Loss=0.1036 + throughput/total_tokens=579,072,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3017/500000] + train/ActionL1Loss=0.0998 + throughput/total_tokens=579,264,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3018/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=579,456,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3019/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=579,648,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3020/500000] + optim/total_grad_norm=8.622 + train/ActionL1Loss=0.0863 + throughput/total_tokens=579,840,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3021/500000] + train/ActionL1Loss=0.0939 + throughput/total_tokens=580,032,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3022/500000] + train/ActionL1Loss=0.0996 + throughput/total_tokens=580,224,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3023/500000] + train/ActionL1Loss=0.1064 + throughput/total_tokens=580,416,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3024/500000] + train/ActionL1Loss=0.0753 + throughput/total_tokens=580,608,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3025/500000] + train/ActionL1Loss=0.0908 + throughput/total_tokens=580,800,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3026/500000] + train/ActionL1Loss=0.0965 + throughput/total_tokens=580,992,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3027/500000] + train/ActionL1Loss=0.0875 + throughput/total_tokens=581,184,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3028/500000] + train/ActionL1Loss=0.0796 + throughput/total_tokens=581,376,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3029/500000] + train/ActionL1Loss=0.1088 + throughput/total_tokens=581,568,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3030/500000] + train/ActionL1Loss=0.1012 + throughput/total_tokens=581,760,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3031/500000] + train/ActionL1Loss=0.1038 + throughput/total_tokens=581,952,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3032/500000] + train/ActionL1Loss=0.1070 + throughput/total_tokens=582,144,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3033/500000] + train/ActionL1Loss=0.0946 + throughput/total_tokens=582,336,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3034/500000] + train/ActionL1Loss=0.0924 + throughput/total_tokens=582,528,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3035/500000] + train/ActionL1Loss=0.0918 + throughput/total_tokens=582,720,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3036/500000] + train/ActionL1Loss=0.1041 + throughput/total_tokens=582,912,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3037/500000] + train/ActionL1Loss=0.0762 + throughput/total_tokens=583,104,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3038/500000] + train/ActionL1Loss=0.0863 + throughput/total_tokens=583,296,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3039/500000] + train/ActionL1Loss=0.0795 + throughput/total_tokens=583,488,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3040/500000] + optim/total_grad_norm=6.695 + train/ActionL1Loss=0.0985 + throughput/total_tokens=583,680,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3041/500000] + train/ActionL1Loss=0.0967 + throughput/total_tokens=583,872,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3042/500000] + train/ActionL1Loss=0.0862 + throughput/total_tokens=584,064,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3043/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=584,256,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3044/500000] + train/ActionL1Loss=0.0924 + throughput/total_tokens=584,448,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3045/500000] + train/ActionL1Loss=0.1226 + throughput/total_tokens=584,640,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3046/500000] + train/ActionL1Loss=0.1036 + throughput/total_tokens=584,832,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3047/500000] + train/ActionL1Loss=0.0841 + throughput/total_tokens=585,024,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3048/500000] + train/ActionL1Loss=0.0837 + throughput/total_tokens=585,216,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3049/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=585,408,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3050/500000] + train/ActionL1Loss=0.1088 + throughput/total_tokens=585,600,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3051/500000] + train/ActionL1Loss=0.0832 + throughput/total_tokens=585,792,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3052/500000] + train/ActionL1Loss=0.0891 + throughput/total_tokens=585,984,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3053/500000] + train/ActionL1Loss=0.0899 + throughput/total_tokens=586,176,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3054/500000] + train/ActionL1Loss=0.1111 + throughput/total_tokens=586,368,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3055/500000] + train/ActionL1Loss=0.1103 + throughput/total_tokens=586,560,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3056/500000] + train/ActionL1Loss=0.0937 + throughput/total_tokens=586,752,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3057/500000] + train/ActionL1Loss=0.0995 + throughput/total_tokens=586,944,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3058/500000] + train/ActionL1Loss=0.0957 + throughput/total_tokens=587,136,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3059/500000] + train/ActionL1Loss=0.0944 + throughput/total_tokens=587,328,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3060/500000] + optim/total_grad_norm=7.434 + train/ActionL1Loss=0.0871 + throughput/total_tokens=587,520,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3061/500000] + train/ActionL1Loss=0.0846 + throughput/total_tokens=587,712,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=3062/500000] + train/ActionL1Loss=0.0941 + throughput/total_tokens=587,904,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=3063/500000] + train/ActionL1Loss=0.1033 + throughput/total_tokens=588,096,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=3064/500000] + train/ActionL1Loss=0.0779 + throughput/total_tokens=588,288,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3065/500000] + train/ActionL1Loss=0.0958 + throughput/total_tokens=588,480,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3066/500000] + train/ActionL1Loss=0.0792 + throughput/total_tokens=588,672,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3067/500000] + train/ActionL1Loss=0.0988 + throughput/total_tokens=588,864,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3068/500000] + train/ActionL1Loss=0.0954 + throughput/total_tokens=589,056,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3069/500000] + train/ActionL1Loss=0.0939 + throughput/total_tokens=589,248,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3070/500000] + train/ActionL1Loss=0.1027 + throughput/total_tokens=589,440,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3071/500000] + train/ActionL1Loss=0.0990 + throughput/total_tokens=589,632,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3072/500000] + train/ActionL1Loss=0.0834 + throughput/total_tokens=589,824,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3073/500000] + train/ActionL1Loss=0.0845 + throughput/total_tokens=590,016,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3074/500000] + train/ActionL1Loss=0.0920 + throughput/total_tokens=590,208,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3075/500000] + train/ActionL1Loss=0.0948 + throughput/total_tokens=590,400,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3076/500000] + train/ActionL1Loss=0.1000 + throughput/total_tokens=590,592,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3077/500000] + train/ActionL1Loss=0.0856 + throughput/total_tokens=590,784,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3078/500000] + train/ActionL1Loss=0.0949 + throughput/total_tokens=590,976,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3079/500000] + train/ActionL1Loss=0.0969 + throughput/total_tokens=591,168,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3080/500000] + optim/total_grad_norm=6.495 + train/ActionL1Loss=0.0797 + throughput/total_tokens=591,360,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3081/500000] + train/ActionL1Loss=0.0987 + throughput/total_tokens=591,552,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3082/500000] + train/ActionL1Loss=0.0861 + throughput/total_tokens=591,744,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3083/500000] + train/ActionL1Loss=0.0785 + throughput/total_tokens=591,936,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3084/500000] + train/ActionL1Loss=0.0968 + throughput/total_tokens=592,128,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3085/500000] + train/ActionL1Loss=0.1021 + throughput/total_tokens=592,320,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3086/500000] + train/ActionL1Loss=0.0851 + throughput/total_tokens=592,512,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3087/500000] + train/ActionL1Loss=0.0886 + throughput/total_tokens=592,704,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3088/500000] + train/ActionL1Loss=0.0875 + throughput/total_tokens=592,896,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3089/500000] + train/ActionL1Loss=0.0955 + throughput/total_tokens=593,088,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3090/500000] + train/ActionL1Loss=0.0773 + throughput/total_tokens=593,280,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3091/500000] + train/ActionL1Loss=0.0858 + throughput/total_tokens=593,472,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3092/500000] + train/ActionL1Loss=0.0889 + throughput/total_tokens=593,664,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3093/500000] + train/ActionL1Loss=0.1149 + throughput/total_tokens=593,856,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3094/500000] + train/ActionL1Loss=0.1027 + throughput/total_tokens=594,048,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3095/500000] + train/ActionL1Loss=0.0864 + throughput/total_tokens=594,240,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3096/500000] + train/ActionL1Loss=0.0812 + throughput/total_tokens=594,432,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3097/500000] + train/ActionL1Loss=0.0963 + throughput/total_tokens=594,624,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3098/500000] + train/ActionL1Loss=0.0962 + throughput/total_tokens=594,816,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3099/500000] + train/ActionL1Loss=0.1020 + throughput/total_tokens=595,008,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3100/500000] + optim/total_grad_norm=5.678 + train/ActionL1Loss=0.0918 + throughput/total_tokens=595,200,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3101/500000] + train/ActionL1Loss=0.1046 + throughput/total_tokens=595,392,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3102/500000] + train/ActionL1Loss=0.0953 + throughput/total_tokens=595,584,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3103/500000] + train/ActionL1Loss=0.0931 + throughput/total_tokens=595,776,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3104/500000] + train/ActionL1Loss=0.0914 + throughput/total_tokens=595,968,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3105/500000] + train/ActionL1Loss=0.0849 + throughput/total_tokens=596,160,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3106/500000] + train/ActionL1Loss=0.1061 + throughput/total_tokens=596,352,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3107/500000] + train/ActionL1Loss=0.1051 + throughput/total_tokens=596,544,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3108/500000] + train/ActionL1Loss=0.0958 + throughput/total_tokens=596,736,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3109/500000] + train/ActionL1Loss=0.0875 + throughput/total_tokens=596,928,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3110/500000] + train/ActionL1Loss=0.0894 + throughput/total_tokens=597,120,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3111/500000] + train/ActionL1Loss=0.1040 + throughput/total_tokens=597,312,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3112/500000] + train/ActionL1Loss=0.1031 + throughput/total_tokens=597,504,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3113/500000] + train/ActionL1Loss=0.0979 + throughput/total_tokens=597,696,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3114/500000] + train/ActionL1Loss=0.0891 + throughput/total_tokens=597,888,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3115/500000] + train/ActionL1Loss=0.0953 + throughput/total_tokens=598,080,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3116/500000] + train/ActionL1Loss=0.0951 + throughput/total_tokens=598,272,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3117/500000] + train/ActionL1Loss=0.1024 + throughput/total_tokens=598,464,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3118/500000] + train/ActionL1Loss=0.1149 + throughput/total_tokens=598,656,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3119/500000] + train/ActionL1Loss=0.1025 + throughput/total_tokens=598,848,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3120/500000] + optim/total_grad_norm=7.251 + train/ActionL1Loss=0.1032 + throughput/total_tokens=599,040,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3121/500000] + train/ActionL1Loss=0.0804 + throughput/total_tokens=599,232,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3122/500000] + train/ActionL1Loss=0.0917 + throughput/total_tokens=599,424,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3123/500000] + train/ActionL1Loss=0.1108 + throughput/total_tokens=599,616,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3124/500000] + train/ActionL1Loss=0.0911 + throughput/total_tokens=599,808,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3125/500000] + train/ActionL1Loss=0.0765 + throughput/total_tokens=600,000,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3126/500000] + train/ActionL1Loss=0.0784 + throughput/total_tokens=600,192,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3127/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=600,384,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3128/500000] + train/ActionL1Loss=0.1035 + throughput/total_tokens=600,576,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3129/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=600,768,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3130/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=600,960,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3131/500000] + train/ActionL1Loss=0.0786 + throughput/total_tokens=601,152,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3132/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=601,344,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3133/500000] + train/ActionL1Loss=0.0835 + throughput/total_tokens=601,536,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3134/500000] + train/ActionL1Loss=0.0913 + throughput/total_tokens=601,728,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3135/500000] + train/ActionL1Loss=0.0959 + throughput/total_tokens=601,920,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3136/500000] + train/ActionL1Loss=0.0883 + throughput/total_tokens=602,112,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3137/500000] + train/ActionL1Loss=0.0893 + throughput/total_tokens=602,304,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3138/500000] + train/ActionL1Loss=0.0777 + throughput/total_tokens=602,496,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3139/500000] + train/ActionL1Loss=0.0825 + throughput/total_tokens=602,688,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3140/500000] + optim/total_grad_norm=6.995 + train/ActionL1Loss=0.0752 + throughput/total_tokens=602,880,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3141/500000] + train/ActionL1Loss=0.0988 + throughput/total_tokens=603,072,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3142/500000] + train/ActionL1Loss=0.0768 + throughput/total_tokens=603,264,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3143/500000] + train/ActionL1Loss=0.0916 + throughput/total_tokens=603,456,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3144/500000] + train/ActionL1Loss=0.1123 + throughput/total_tokens=603,648,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3145/500000] + train/ActionL1Loss=0.1004 + throughput/total_tokens=603,840,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3146/500000] + train/ActionL1Loss=0.1072 + throughput/total_tokens=604,032,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3147/500000] + train/ActionL1Loss=0.0807 + throughput/total_tokens=604,224,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3148/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=604,416,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3149/500000] + train/ActionL1Loss=0.1021 + throughput/total_tokens=604,608,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3150/500000] + train/ActionL1Loss=0.0968 + throughput/total_tokens=604,800,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3151/500000] + train/ActionL1Loss=0.0949 + throughput/total_tokens=604,992,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3152/500000] + train/ActionL1Loss=0.0922 + throughput/total_tokens=605,184,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3153/500000] + train/ActionL1Loss=0.0752 + throughput/total_tokens=605,376,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3154/500000] + train/ActionL1Loss=0.1064 + throughput/total_tokens=605,568,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3155/500000] + train/ActionL1Loss=0.0965 + throughput/total_tokens=605,760,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3156/500000] + train/ActionL1Loss=0.0866 + throughput/total_tokens=605,952,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3157/500000] + train/ActionL1Loss=0.0876 + throughput/total_tokens=606,144,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3158/500000] + train/ActionL1Loss=0.0966 + throughput/total_tokens=606,336,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3159/500000] + train/ActionL1Loss=0.0824 + throughput/total_tokens=606,528,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3160/500000] + optim/total_grad_norm=6.536 + train/ActionL1Loss=0.0909 + throughput/total_tokens=606,720,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3161/500000] + train/ActionL1Loss=0.0678 + throughput/total_tokens=606,912,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3162/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=607,104,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3163/500000] + train/ActionL1Loss=0.0955 + throughput/total_tokens=607,296,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3164/500000] + train/ActionL1Loss=0.0981 + throughput/total_tokens=607,488,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3165/500000] + train/ActionL1Loss=0.0996 + throughput/total_tokens=607,680,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3166/500000] + train/ActionL1Loss=0.0895 + throughput/total_tokens=607,872,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3167/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=608,064,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3168/500000] + train/ActionL1Loss=0.0779 + throughput/total_tokens=608,256,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3169/500000] + train/ActionL1Loss=0.0776 + throughput/total_tokens=608,448,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3170/500000] + train/ActionL1Loss=0.0950 + throughput/total_tokens=608,640,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3171/500000] + train/ActionL1Loss=0.1058 + throughput/total_tokens=608,832,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3172/500000] + train/ActionL1Loss=0.0867 + throughput/total_tokens=609,024,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3173/500000] + train/ActionL1Loss=0.0980 + throughput/total_tokens=609,216,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3174/500000] + train/ActionL1Loss=0.0944 + throughput/total_tokens=609,408,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3175/500000] + train/ActionL1Loss=0.1002 + throughput/total_tokens=609,600,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3176/500000] + train/ActionL1Loss=0.0846 + throughput/total_tokens=609,792,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3177/500000] + train/ActionL1Loss=0.0865 + throughput/total_tokens=609,984,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3178/500000] + train/ActionL1Loss=0.0808 + throughput/total_tokens=610,176,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3179/500000] + train/ActionL1Loss=0.0915 + throughput/total_tokens=610,368,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3180/500000] + optim/total_grad_norm=6.877 + train/ActionL1Loss=0.0867 + throughput/total_tokens=610,560,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3181/500000] + train/ActionL1Loss=0.0832 + throughput/total_tokens=610,752,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3182/500000] + train/ActionL1Loss=0.0751 + throughput/total_tokens=610,944,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3183/500000] + train/ActionL1Loss=0.0865 + throughput/total_tokens=611,136,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3184/500000] + train/ActionL1Loss=0.1037 + throughput/total_tokens=611,328,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3185/500000] + train/ActionL1Loss=0.1057 + throughput/total_tokens=611,520,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3186/500000] + train/ActionL1Loss=0.1165 + throughput/total_tokens=611,712,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3187/500000] + train/ActionL1Loss=0.0850 + throughput/total_tokens=611,904,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3188/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=612,096,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3189/500000] + train/ActionL1Loss=0.0840 + throughput/total_tokens=612,288,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3190/500000] + train/ActionL1Loss=0.0948 + throughput/total_tokens=612,480,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3191/500000] + train/ActionL1Loss=0.0909 + throughput/total_tokens=612,672,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3192/500000] + train/ActionL1Loss=0.0966 + throughput/total_tokens=612,864,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3193/500000] + train/ActionL1Loss=0.0956 + throughput/total_tokens=613,056,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3194/500000] + train/ActionL1Loss=0.0882 + throughput/total_tokens=613,248,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3195/500000] + train/ActionL1Loss=0.1132 + throughput/total_tokens=613,440,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3196/500000] + train/ActionL1Loss=0.0881 + throughput/total_tokens=613,632,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3197/500000] + train/ActionL1Loss=0.0917 + throughput/total_tokens=613,824,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3198/500000] + train/ActionL1Loss=0.0861 + throughput/total_tokens=614,016,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3199/500000] + train/ActionL1Loss=0.1062 + throughput/total_tokens=614,208,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3200/500000] + optim/total_grad_norm=6.297 + train/ActionL1Loss=0.1058 + throughput/total_tokens=614,400,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3201/500000] + train/ActionL1Loss=0.1042 + throughput/total_tokens=614,592,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3202/500000] + train/ActionL1Loss=0.0994 + throughput/total_tokens=614,784,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3203/500000] + train/ActionL1Loss=0.0914 + throughput/total_tokens=614,976,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3204/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=615,168,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3205/500000] + train/ActionL1Loss=0.0921 + throughput/total_tokens=615,360,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3206/500000] + train/ActionL1Loss=0.0878 + throughput/total_tokens=615,552,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3207/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=615,744,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3208/500000] + train/ActionL1Loss=0.0794 + throughput/total_tokens=615,936,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3209/500000] + train/ActionL1Loss=0.1043 + throughput/total_tokens=616,128,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3210/500000] + train/ActionL1Loss=0.1002 + throughput/total_tokens=616,320,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3211/500000] + train/ActionL1Loss=0.0885 + throughput/total_tokens=616,512,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3212/500000] + train/ActionL1Loss=0.0853 + throughput/total_tokens=616,704,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3213/500000] + train/ActionL1Loss=0.0786 + throughput/total_tokens=616,896,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3214/500000] + train/ActionL1Loss=0.0935 + throughput/total_tokens=617,088,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3215/500000] + train/ActionL1Loss=0.0878 + throughput/total_tokens=617,280,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3216/500000] + train/ActionL1Loss=0.0840 + throughput/total_tokens=617,472,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3217/500000] + train/ActionL1Loss=0.1143 + throughput/total_tokens=617,664,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3218/500000] + train/ActionL1Loss=0.1225 + throughput/total_tokens=617,856,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3219/500000] + train/ActionL1Loss=0.0801 + throughput/total_tokens=618,048,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3220/500000] + optim/total_grad_norm=5.921 + train/ActionL1Loss=0.0841 + throughput/total_tokens=618,240,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3221/500000] + train/ActionL1Loss=0.1078 + throughput/total_tokens=618,432,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3222/500000] + train/ActionL1Loss=0.0973 + throughput/total_tokens=618,624,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3223/500000] + train/ActionL1Loss=0.1124 + throughput/total_tokens=618,816,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3224/500000] + train/ActionL1Loss=0.1095 + throughput/total_tokens=619,008,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3225/500000] + train/ActionL1Loss=0.0800 + throughput/total_tokens=619,200,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3226/500000] + train/ActionL1Loss=0.0807 + throughput/total_tokens=619,392,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3227/500000] + train/ActionL1Loss=0.0925 + throughput/total_tokens=619,584,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3228/500000] + train/ActionL1Loss=0.0927 + throughput/total_tokens=619,776,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3229/500000] + train/ActionL1Loss=0.0926 + throughput/total_tokens=619,968,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3230/500000] + train/ActionL1Loss=0.0831 + throughput/total_tokens=620,160,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=3231/500000] + train/ActionL1Loss=0.0778 + throughput/total_tokens=620,352,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3232/500000] + train/ActionL1Loss=0.1024 + throughput/total_tokens=620,544,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3233/500000] + train/ActionL1Loss=0.0897 + throughput/total_tokens=620,736,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3234/500000] + train/ActionL1Loss=0.0865 + throughput/total_tokens=620,928,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3235/500000] + train/ActionL1Loss=0.0854 + throughput/total_tokens=621,120,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3236/500000] + train/ActionL1Loss=0.0765 + throughput/total_tokens=621,312,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3237/500000] + train/ActionL1Loss=0.0800 + throughput/total_tokens=621,504,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3238/500000] + train/ActionL1Loss=0.0780 + throughput/total_tokens=621,696,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3239/500000] + train/ActionL1Loss=0.0822 + throughput/total_tokens=621,888,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3240/500000] + optim/total_grad_norm=6.464 + train/ActionL1Loss=0.0746 + throughput/total_tokens=622,080,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3241/500000] + train/ActionL1Loss=0.0795 + throughput/total_tokens=622,272,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3242/500000] + train/ActionL1Loss=0.0881 + throughput/total_tokens=622,464,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3243/500000] + train/ActionL1Loss=0.0701 + throughput/total_tokens=622,656,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3244/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=622,848,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3245/500000] + train/ActionL1Loss=0.0839 + throughput/total_tokens=623,040,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3246/500000] + train/ActionL1Loss=0.0760 + throughput/total_tokens=623,232,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3247/500000] + train/ActionL1Loss=0.0816 + throughput/total_tokens=623,424,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3248/500000] + train/ActionL1Loss=0.0909 + throughput/total_tokens=623,616,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3249/500000] + train/ActionL1Loss=0.0949 + throughput/total_tokens=623,808,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3250/500000] + train/ActionL1Loss=0.0627 + throughput/total_tokens=624,000,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3251/500000] + train/ActionL1Loss=0.0731 + throughput/total_tokens=624,192,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3252/500000] + train/ActionL1Loss=0.0875 + throughput/total_tokens=624,384,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3253/500000] + train/ActionL1Loss=0.1027 + throughput/total_tokens=624,576,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3254/500000] + train/ActionL1Loss=0.0942 + throughput/total_tokens=624,768,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3255/500000] + train/ActionL1Loss=0.0965 + throughput/total_tokens=624,960,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3256/500000] + train/ActionL1Loss=0.0849 + throughput/total_tokens=625,152,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3257/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=625,344,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3258/500000] + train/ActionL1Loss=0.0827 + throughput/total_tokens=625,536,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3259/500000] + train/ActionL1Loss=0.0874 + throughput/total_tokens=625,728,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3260/500000] + optim/total_grad_norm=6.841 + train/ActionL1Loss=0.0862 + throughput/total_tokens=625,920,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3261/500000] + train/ActionL1Loss=0.0869 + throughput/total_tokens=626,112,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3262/500000] + train/ActionL1Loss=0.0804 + throughput/total_tokens=626,304,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3263/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=626,496,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3264/500000] + train/ActionL1Loss=0.0868 + throughput/total_tokens=626,688,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3265/500000] + train/ActionL1Loss=0.0961 + throughput/total_tokens=626,880,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3266/500000] + train/ActionL1Loss=0.0889 + throughput/total_tokens=627,072,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3267/500000] + train/ActionL1Loss=0.0820 + throughput/total_tokens=627,264,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3268/500000] + train/ActionL1Loss=0.0811 + throughput/total_tokens=627,456,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3269/500000] + train/ActionL1Loss=0.0871 + throughput/total_tokens=627,648,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3270/500000] + train/ActionL1Loss=0.0876 + throughput/total_tokens=627,840,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3271/500000] + train/ActionL1Loss=0.0993 + throughput/total_tokens=628,032,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3272/500000] + train/ActionL1Loss=0.0933 + throughput/total_tokens=628,224,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3273/500000] + train/ActionL1Loss=0.0996 + throughput/total_tokens=628,416,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3274/500000] + train/ActionL1Loss=0.0939 + throughput/total_tokens=628,608,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3275/500000] + train/ActionL1Loss=0.0702 + throughput/total_tokens=628,800,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3276/500000] + train/ActionL1Loss=0.0899 + throughput/total_tokens=628,992,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3277/500000] + train/ActionL1Loss=0.0890 + throughput/total_tokens=629,184,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3278/500000] + train/ActionL1Loss=0.0911 + throughput/total_tokens=629,376,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3279/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=629,568,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3280/500000] + optim/total_grad_norm=7.209 + train/ActionL1Loss=0.0961 + throughput/total_tokens=629,760,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3281/500000] + train/ActionL1Loss=0.0833 + throughput/total_tokens=629,952,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3282/500000] + train/ActionL1Loss=0.0804 + throughput/total_tokens=630,144,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3283/500000] + train/ActionL1Loss=0.0944 + throughput/total_tokens=630,336,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3284/500000] + train/ActionL1Loss=0.0987 + throughput/total_tokens=630,528,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3285/500000] + train/ActionL1Loss=0.0925 + throughput/total_tokens=630,720,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3286/500000] + train/ActionL1Loss=0.1090 + throughput/total_tokens=630,912,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3287/500000] + train/ActionL1Loss=0.0986 + throughput/total_tokens=631,104,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3288/500000] + train/ActionL1Loss=0.1086 + throughput/total_tokens=631,296,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3289/500000] + train/ActionL1Loss=0.1020 + throughput/total_tokens=631,488,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3290/500000] + train/ActionL1Loss=0.0987 + throughput/total_tokens=631,680,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3291/500000] + train/ActionL1Loss=0.0946 + throughput/total_tokens=631,872,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3292/500000] + train/ActionL1Loss=0.1039 + throughput/total_tokens=632,064,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3293/500000] + train/ActionL1Loss=0.0997 + throughput/total_tokens=632,256,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3294/500000] + train/ActionL1Loss=0.0942 + throughput/total_tokens=632,448,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3295/500000] + train/ActionL1Loss=0.0790 + throughput/total_tokens=632,640,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3296/500000] + train/ActionL1Loss=0.0812 + throughput/total_tokens=632,832,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3297/500000] + train/ActionL1Loss=0.0863 + throughput/total_tokens=633,024,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3298/500000] + train/ActionL1Loss=0.0968 + throughput/total_tokens=633,216,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3299/500000] + train/ActionL1Loss=0.0787 + throughput/total_tokens=633,408,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3300/500000] + optim/total_grad_norm=7.805 + train/ActionL1Loss=0.0920 + throughput/total_tokens=633,600,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3301/500000] + train/ActionL1Loss=0.0865 + throughput/total_tokens=633,792,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=3302/500000] + train/ActionL1Loss=0.1010 + throughput/total_tokens=633,984,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=3303/500000] + train/ActionL1Loss=0.0966 + throughput/total_tokens=634,176,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=3304/500000] + train/ActionL1Loss=0.0956 + throughput/total_tokens=634,368,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=3305/500000] + train/ActionL1Loss=0.0896 + throughput/total_tokens=634,560,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=3306/500000] + train/ActionL1Loss=0.1117 + throughput/total_tokens=634,752,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=3307/500000] + train/ActionL1Loss=0.0934 + throughput/total_tokens=634,944,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3308/500000] + train/ActionL1Loss=0.0906 + throughput/total_tokens=635,136,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3309/500000] + train/ActionL1Loss=0.0733 + throughput/total_tokens=635,328,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3310/500000] + train/ActionL1Loss=0.1004 + throughput/total_tokens=635,520,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3311/500000] + train/ActionL1Loss=0.1128 + throughput/total_tokens=635,712,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3312/500000] + train/ActionL1Loss=0.1058 + throughput/total_tokens=635,904,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3313/500000] + train/ActionL1Loss=0.1069 + throughput/total_tokens=636,096,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3314/500000] + train/ActionL1Loss=0.0869 + throughput/total_tokens=636,288,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3315/500000] + train/ActionL1Loss=0.0903 + throughput/total_tokens=636,480,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3316/500000] + train/ActionL1Loss=0.0898 + throughput/total_tokens=636,672,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3317/500000] + train/ActionL1Loss=0.0886 + throughput/total_tokens=636,864,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3318/500000] + train/ActionL1Loss=0.0813 + throughput/total_tokens=637,056,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3319/500000] + train/ActionL1Loss=0.0940 + throughput/total_tokens=637,248,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3320/500000] + optim/total_grad_norm=7.647 + train/ActionL1Loss=0.0802 + throughput/total_tokens=637,440,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3321/500000] + train/ActionL1Loss=0.0794 + throughput/total_tokens=637,632,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3322/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=637,824,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3323/500000] + train/ActionL1Loss=0.1041 + throughput/total_tokens=638,016,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3324/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=638,208,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3325/500000] + train/ActionL1Loss=0.0884 + throughput/total_tokens=638,400,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3326/500000] + train/ActionL1Loss=0.0792 + throughput/total_tokens=638,592,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3327/500000] + train/ActionL1Loss=0.0752 + throughput/total_tokens=638,784,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3328/500000] + train/ActionL1Loss=0.0801 + throughput/total_tokens=638,976,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3329/500000] + train/ActionL1Loss=0.0852 + throughput/total_tokens=639,168,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3330/500000] + train/ActionL1Loss=0.0879 + throughput/total_tokens=639,360,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3331/500000] + train/ActionL1Loss=0.0935 + throughput/total_tokens=639,552,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3332/500000] + train/ActionL1Loss=0.0855 + throughput/total_tokens=639,744,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3333/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=639,936,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3334/500000] + train/ActionL1Loss=0.0744 + throughput/total_tokens=640,128,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3335/500000] + train/ActionL1Loss=0.0768 + throughput/total_tokens=640,320,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3336/500000] + train/ActionL1Loss=0.0990 + throughput/total_tokens=640,512,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3337/500000] + train/ActionL1Loss=0.0933 + throughput/total_tokens=640,704,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3338/500000] + train/ActionL1Loss=0.0918 + throughput/total_tokens=640,896,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3339/500000] + train/ActionL1Loss=0.0956 + throughput/total_tokens=641,088,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3340/500000] + optim/total_grad_norm=6.765 + train/ActionL1Loss=0.0832 + throughput/total_tokens=641,280,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3341/500000] + train/ActionL1Loss=0.0834 + throughput/total_tokens=641,472,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3342/500000] + train/ActionL1Loss=0.0771 + throughput/total_tokens=641,664,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3343/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=641,856,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3344/500000] + train/ActionL1Loss=0.0841 + throughput/total_tokens=642,048,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3345/500000] + train/ActionL1Loss=0.0882 + throughput/total_tokens=642,240,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3346/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=642,432,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3347/500000] + train/ActionL1Loss=0.0888 + throughput/total_tokens=642,624,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3348/500000] + train/ActionL1Loss=0.0870 + throughput/total_tokens=642,816,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3349/500000] + train/ActionL1Loss=0.0746 + throughput/total_tokens=643,008,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3350/500000] + train/ActionL1Loss=0.0721 + throughput/total_tokens=643,200,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3351/500000] + train/ActionL1Loss=0.0813 + throughput/total_tokens=643,392,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3352/500000] + train/ActionL1Loss=0.0804 + throughput/total_tokens=643,584,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3353/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=643,776,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3354/500000] + train/ActionL1Loss=0.0849 + throughput/total_tokens=643,968,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3355/500000] + train/ActionL1Loss=0.0821 + throughput/total_tokens=644,160,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3356/500000] + train/ActionL1Loss=0.0872 + throughput/total_tokens=644,352,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3357/500000] + train/ActionL1Loss=0.0713 + throughput/total_tokens=644,544,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3358/500000] + train/ActionL1Loss=0.0799 + throughput/total_tokens=644,736,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3359/500000] + train/ActionL1Loss=0.0931 + throughput/total_tokens=644,928,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3360/500000] + optim/total_grad_norm=5.602 + train/ActionL1Loss=0.0894 + throughput/total_tokens=645,120,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3361/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=645,312,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3362/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=645,504,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3363/500000] + train/ActionL1Loss=0.0894 + throughput/total_tokens=645,696,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3364/500000] + train/ActionL1Loss=0.0974 + throughput/total_tokens=645,888,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3365/500000] + train/ActionL1Loss=0.0863 + throughput/total_tokens=646,080,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3366/500000] + train/ActionL1Loss=0.0932 + throughput/total_tokens=646,272,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3367/500000] + train/ActionL1Loss=0.1008 + throughput/total_tokens=646,464,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3368/500000] + train/ActionL1Loss=0.0823 + throughput/total_tokens=646,656,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3369/500000] + train/ActionL1Loss=0.1020 + throughput/total_tokens=646,848,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3370/500000] + train/ActionL1Loss=0.1034 + throughput/total_tokens=647,040,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3371/500000] + train/ActionL1Loss=0.0704 + throughput/total_tokens=647,232,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3372/500000] + train/ActionL1Loss=0.0755 + throughput/total_tokens=647,424,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3373/500000] + train/ActionL1Loss=0.0913 + throughput/total_tokens=647,616,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3374/500000] + train/ActionL1Loss=0.1160 + throughput/total_tokens=647,808,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3375/500000] + train/ActionL1Loss=0.0913 + throughput/total_tokens=648,000,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3376/500000] + train/ActionL1Loss=0.0909 + throughput/total_tokens=648,192,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3377/500000] + train/ActionL1Loss=0.0806 + throughput/total_tokens=648,384,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3378/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=648,576,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3379/500000] + train/ActionL1Loss=0.0888 + throughput/total_tokens=648,768,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3380/500000] + optim/total_grad_norm=7.642 + train/ActionL1Loss=0.0878 + throughput/total_tokens=648,960,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3381/500000] + train/ActionL1Loss=0.0912 + throughput/total_tokens=649,152,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3382/500000] + train/ActionL1Loss=0.1114 + throughput/total_tokens=649,344,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3383/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=649,536,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3384/500000] + train/ActionL1Loss=0.0934 + throughput/total_tokens=649,728,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3385/500000] + train/ActionL1Loss=0.1054 + throughput/total_tokens=649,920,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3386/500000] + train/ActionL1Loss=0.1006 + throughput/total_tokens=650,112,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3387/500000] + train/ActionL1Loss=0.0860 + throughput/total_tokens=650,304,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3388/500000] + train/ActionL1Loss=0.1041 + throughput/total_tokens=650,496,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3389/500000] + train/ActionL1Loss=0.0855 + throughput/total_tokens=650,688,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3390/500000] + train/ActionL1Loss=0.0905 + throughput/total_tokens=650,880,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3391/500000] + train/ActionL1Loss=0.1013 + throughput/total_tokens=651,072,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3392/500000] + train/ActionL1Loss=0.0797 + throughput/total_tokens=651,264,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3393/500000] + train/ActionL1Loss=0.0856 + throughput/total_tokens=651,456,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3394/500000] + train/ActionL1Loss=0.0899 + throughput/total_tokens=651,648,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3395/500000] + train/ActionL1Loss=0.0882 + throughput/total_tokens=651,840,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3396/500000] + train/ActionL1Loss=0.0953 + throughput/total_tokens=652,032,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3397/500000] + train/ActionL1Loss=0.0799 + throughput/total_tokens=652,224,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3398/500000] + train/ActionL1Loss=0.0984 + throughput/total_tokens=652,416,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3399/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=652,608,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3400/500000] + optim/total_grad_norm=6.328 + train/ActionL1Loss=0.0779 + throughput/total_tokens=652,800,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3401/500000] + train/ActionL1Loss=0.0840 + throughput/total_tokens=652,992,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3402/500000] + train/ActionL1Loss=0.0895 + throughput/total_tokens=653,184,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3403/500000] + train/ActionL1Loss=0.0835 + throughput/total_tokens=653,376,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3404/500000] + train/ActionL1Loss=0.1114 + throughput/total_tokens=653,568,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3405/500000] + train/ActionL1Loss=0.0796 + throughput/total_tokens=653,760,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3406/500000] + train/ActionL1Loss=0.0952 + throughput/total_tokens=653,952,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3407/500000] + train/ActionL1Loss=0.1108 + throughput/total_tokens=654,144,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3408/500000] + train/ActionL1Loss=0.1030 + throughput/total_tokens=654,336,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3409/500000] + train/ActionL1Loss=0.0818 + throughput/total_tokens=654,528,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3410/500000] + train/ActionL1Loss=0.0907 + throughput/total_tokens=654,720,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3411/500000] + train/ActionL1Loss=0.0751 + throughput/total_tokens=654,912,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3412/500000] + train/ActionL1Loss=0.0848 + throughput/total_tokens=655,104,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3413/500000] + train/ActionL1Loss=0.1231 + throughput/total_tokens=655,296,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3414/500000] + train/ActionL1Loss=0.0963 + throughput/total_tokens=655,488,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3415/500000] + train/ActionL1Loss=0.0860 + throughput/total_tokens=655,680,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3416/500000] + train/ActionL1Loss=0.0936 + throughput/total_tokens=655,872,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3417/500000] + train/ActionL1Loss=0.0768 + throughput/total_tokens=656,064,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3418/500000] + train/ActionL1Loss=0.0925 + throughput/total_tokens=656,256,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3419/500000] + train/ActionL1Loss=0.0734 + throughput/total_tokens=656,448,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3420/500000] + optim/total_grad_norm=6.612 + train/ActionL1Loss=0.1071 + throughput/total_tokens=656,640,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3421/500000] + train/ActionL1Loss=0.0809 + throughput/total_tokens=656,832,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3422/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=657,024,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3423/500000] + train/ActionL1Loss=0.0726 + throughput/total_tokens=657,216,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3424/500000] + train/ActionL1Loss=0.0874 + throughput/total_tokens=657,408,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3425/500000] + train/ActionL1Loss=0.0829 + throughput/total_tokens=657,600,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3426/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=657,792,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3427/500000] + train/ActionL1Loss=0.0788 + throughput/total_tokens=657,984,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3428/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=658,176,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3429/500000] + train/ActionL1Loss=0.1060 + throughput/total_tokens=658,368,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3430/500000] + train/ActionL1Loss=0.0806 + throughput/total_tokens=658,560,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3431/500000] + train/ActionL1Loss=0.0935 + throughput/total_tokens=658,752,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3432/500000] + train/ActionL1Loss=0.0798 + throughput/total_tokens=658,944,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3433/500000] + train/ActionL1Loss=0.0827 + throughput/total_tokens=659,136,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3434/500000] + train/ActionL1Loss=0.0841 + throughput/total_tokens=659,328,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3435/500000] + train/ActionL1Loss=0.0914 + throughput/total_tokens=659,520,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3436/500000] + train/ActionL1Loss=0.0931 + throughput/total_tokens=659,712,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3437/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=659,904,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3438/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=660,096,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3439/500000] + train/ActionL1Loss=0.0843 + throughput/total_tokens=660,288,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3440/500000] + optim/total_grad_norm=9.000 + train/ActionL1Loss=0.0997 + throughput/total_tokens=660,480,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3441/500000] + train/ActionL1Loss=0.0832 + throughput/total_tokens=660,672,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3442/500000] + train/ActionL1Loss=0.0670 + throughput/total_tokens=660,864,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3443/500000] + train/ActionL1Loss=0.0657 + throughput/total_tokens=661,056,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3444/500000] + train/ActionL1Loss=0.0994 + throughput/total_tokens=661,248,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3445/500000] + train/ActionL1Loss=0.0744 + throughput/total_tokens=661,440,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3446/500000] + train/ActionL1Loss=0.0837 + throughput/total_tokens=661,632,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3447/500000] + train/ActionL1Loss=0.0821 + throughput/total_tokens=661,824,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3448/500000] + train/ActionL1Loss=0.0890 + throughput/total_tokens=662,016,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3449/500000] + train/ActionL1Loss=0.0777 + throughput/total_tokens=662,208,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3450/500000] + train/ActionL1Loss=0.0684 + throughput/total_tokens=662,400,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3451/500000] + train/ActionL1Loss=0.0777 + throughput/total_tokens=662,592,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3452/500000] + train/ActionL1Loss=0.0749 + throughput/total_tokens=662,784,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3453/500000] + train/ActionL1Loss=0.0775 + throughput/total_tokens=662,976,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3454/500000] + train/ActionL1Loss=0.0937 + throughput/total_tokens=663,168,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3455/500000] + train/ActionL1Loss=0.0801 + throughput/total_tokens=663,360,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3456/500000] + train/ActionL1Loss=0.0945 + throughput/total_tokens=663,552,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3457/500000] + train/ActionL1Loss=0.0907 + throughput/total_tokens=663,744,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3458/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=663,936,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3459/500000] + train/ActionL1Loss=0.0818 + throughput/total_tokens=664,128,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3460/500000] + optim/total_grad_norm=6.926 + train/ActionL1Loss=0.0979 + throughput/total_tokens=664,320,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3461/500000] + train/ActionL1Loss=0.0994 + throughput/total_tokens=664,512,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3462/500000] + train/ActionL1Loss=0.0833 + throughput/total_tokens=664,704,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3463/500000] + train/ActionL1Loss=0.0891 + throughput/total_tokens=664,896,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3464/500000] + train/ActionL1Loss=0.0748 + throughput/total_tokens=665,088,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3465/500000] + train/ActionL1Loss=0.0747 + throughput/total_tokens=665,280,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3466/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=665,472,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3467/500000] + train/ActionL1Loss=0.0703 + throughput/total_tokens=665,664,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3468/500000] + train/ActionL1Loss=0.0884 + throughput/total_tokens=665,856,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3469/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=666,048,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3470/500000] + train/ActionL1Loss=0.0775 + throughput/total_tokens=666,240,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3471/500000] + train/ActionL1Loss=0.0928 + throughput/total_tokens=666,432,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3472/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=666,624,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3473/500000] + train/ActionL1Loss=0.0801 + throughput/total_tokens=666,816,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3474/500000] + train/ActionL1Loss=0.0869 + throughput/total_tokens=667,008,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3475/500000] + train/ActionL1Loss=0.0887 + throughput/total_tokens=667,200,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3476/500000] + train/ActionL1Loss=0.0919 + throughput/total_tokens=667,392,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3477/500000] + train/ActionL1Loss=0.0676 + throughput/total_tokens=667,584,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3478/500000] + train/ActionL1Loss=0.0749 + throughput/total_tokens=667,776,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3479/500000] + train/ActionL1Loss=0.0849 + throughput/total_tokens=667,968,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3480/500000] + optim/total_grad_norm=6.323 + train/ActionL1Loss=0.0709 + throughput/total_tokens=668,160,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3481/500000] + train/ActionL1Loss=0.0885 + throughput/total_tokens=668,352,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3482/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=668,544,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3483/500000] + train/ActionL1Loss=0.0916 + throughput/total_tokens=668,736,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3484/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=668,928,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3485/500000] + train/ActionL1Loss=0.1107 + throughput/total_tokens=669,120,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3486/500000] + train/ActionL1Loss=0.0822 + throughput/total_tokens=669,312,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3487/500000] + train/ActionL1Loss=0.0729 + throughput/total_tokens=669,504,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3488/500000] + train/ActionL1Loss=0.0903 + throughput/total_tokens=669,696,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3489/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=669,888,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3490/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=670,080,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3491/500000] + train/ActionL1Loss=0.0966 + throughput/total_tokens=670,272,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3492/500000] + train/ActionL1Loss=0.0743 + throughput/total_tokens=670,464,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3493/500000] + train/ActionL1Loss=0.0822 + throughput/total_tokens=670,656,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3494/500000] + train/ActionL1Loss=0.0724 + throughput/total_tokens=670,848,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3495/500000] + train/ActionL1Loss=0.0903 + throughput/total_tokens=671,040,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3496/500000] + train/ActionL1Loss=0.0816 + throughput/total_tokens=671,232,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3497/500000] + train/ActionL1Loss=0.0926 + throughput/total_tokens=671,424,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3498/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=671,616,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3499/500000] + train/ActionL1Loss=0.0846 + throughput/total_tokens=671,808,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3500/500000] + optim/total_grad_norm=5.764 + train/ActionL1Loss=0.0940 + throughput/total_tokens=672,000,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +10/01 [09:49:59] INFO | >> Saving config... checkpoint.py:608 +10/01 [09:50:43] INFO | >> Saving model state... checkpoint.py:796 +10/01 [09:51:56] INFO | >> Saving optim state... checkpoint.py:811 +10/01 [09:53:29] INFO | >> Saving trainer state... checkpoint.py:670 + INFO | >> Saving config... checkpoint.py:608 +[step=3501/500000] + train/ActionL1Loss=0.0730 + throughput/total_tokens=672,192,000 + throughput/device/tokens_per_second=1,209 + throughput/device/batches_per_second=0.0504 +[step=3502/500000] + train/ActionL1Loss=0.1006 + throughput/total_tokens=672,384,000 + throughput/device/tokens_per_second=1,173 + throughput/device/batches_per_second=0.0489 +[step=3503/500000] + train/ActionL1Loss=0.0747 + throughput/total_tokens=672,576,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=3504/500000] + train/ActionL1Loss=0.0911 + throughput/total_tokens=672,768,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=3505/500000] + train/ActionL1Loss=0.0762 + throughput/total_tokens=672,960,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=3506/500000] + train/ActionL1Loss=0.0664 + throughput/total_tokens=673,152,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=3507/500000] + train/ActionL1Loss=0.0890 + throughput/total_tokens=673,344,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=3508/500000] + train/ActionL1Loss=0.0869 + throughput/total_tokens=673,536,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=3509/500000] + train/ActionL1Loss=0.0812 + throughput/total_tokens=673,728,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3510/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=673,920,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3511/500000] + train/ActionL1Loss=0.0744 + throughput/total_tokens=674,112,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3512/500000] + train/ActionL1Loss=0.0814 + throughput/total_tokens=674,304,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3513/500000] + train/ActionL1Loss=0.0593 + throughput/total_tokens=674,496,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3514/500000] + train/ActionL1Loss=0.0820 + throughput/total_tokens=674,688,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3515/500000] + train/ActionL1Loss=0.0740 + throughput/total_tokens=674,880,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3516/500000] + train/ActionL1Loss=0.1107 + throughput/total_tokens=675,072,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3517/500000] + train/ActionL1Loss=0.0795 + throughput/total_tokens=675,264,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3518/500000] + train/ActionL1Loss=0.0944 + throughput/total_tokens=675,456,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3519/500000] + train/ActionL1Loss=0.0803 + throughput/total_tokens=675,648,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3520/500000] + optim/total_grad_norm=5.974 + train/ActionL1Loss=0.0665 + throughput/total_tokens=675,840,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3521/500000] + train/ActionL1Loss=0.0782 + throughput/total_tokens=676,032,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3522/500000] + train/ActionL1Loss=0.0792 + throughput/total_tokens=676,224,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3523/500000] + train/ActionL1Loss=0.0879 + throughput/total_tokens=676,416,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3524/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=676,608,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3525/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=676,800,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3526/500000] + train/ActionL1Loss=0.0887 + throughput/total_tokens=676,992,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3527/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=677,184,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3528/500000] + train/ActionL1Loss=0.0818 + throughput/total_tokens=677,376,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3529/500000] + train/ActionL1Loss=0.0739 + throughput/total_tokens=677,568,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3530/500000] + train/ActionL1Loss=0.0840 + throughput/total_tokens=677,760,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3531/500000] + train/ActionL1Loss=0.0860 + throughput/total_tokens=677,952,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3532/500000] + train/ActionL1Loss=0.0878 + throughput/total_tokens=678,144,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3533/500000] + train/ActionL1Loss=0.0844 + throughput/total_tokens=678,336,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3534/500000] + train/ActionL1Loss=0.0929 + throughput/total_tokens=678,528,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3535/500000] + train/ActionL1Loss=0.0988 + throughput/total_tokens=678,720,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3536/500000] + train/ActionL1Loss=0.0870 + throughput/total_tokens=678,912,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3537/500000] + train/ActionL1Loss=0.0974 + throughput/total_tokens=679,104,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3538/500000] + train/ActionL1Loss=0.0885 + throughput/total_tokens=679,296,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3539/500000] + train/ActionL1Loss=0.0670 + throughput/total_tokens=679,488,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3540/500000] + optim/total_grad_norm=6.701 + train/ActionL1Loss=0.1028 + throughput/total_tokens=679,680,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3541/500000] + train/ActionL1Loss=0.1062 + throughput/total_tokens=679,872,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3542/500000] + train/ActionL1Loss=0.0970 + throughput/total_tokens=680,064,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3543/500000] + train/ActionL1Loss=0.0930 + throughput/total_tokens=680,256,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3544/500000] + train/ActionL1Loss=0.0900 + throughput/total_tokens=680,448,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3545/500000] + train/ActionL1Loss=0.0775 + throughput/total_tokens=680,640,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3546/500000] + train/ActionL1Loss=0.0829 + throughput/total_tokens=680,832,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3547/500000] + train/ActionL1Loss=0.0910 + throughput/total_tokens=681,024,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3548/500000] + train/ActionL1Loss=0.0823 + throughput/total_tokens=681,216,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3549/500000] + train/ActionL1Loss=0.0950 + throughput/total_tokens=681,408,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3550/500000] + train/ActionL1Loss=0.0735 + throughput/total_tokens=681,600,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3551/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=681,792,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3552/500000] + train/ActionL1Loss=0.0766 + throughput/total_tokens=681,984,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=3553/500000] + train/ActionL1Loss=0.0954 + throughput/total_tokens=682,176,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=3554/500000] + train/ActionL1Loss=0.0864 + throughput/total_tokens=682,368,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=3555/500000] + train/ActionL1Loss=0.0851 + throughput/total_tokens=682,560,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=3556/500000] + train/ActionL1Loss=0.0858 + throughput/total_tokens=682,752,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=3557/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=682,944,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=3558/500000] + train/ActionL1Loss=0.0892 + throughput/total_tokens=683,136,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3559/500000] + train/ActionL1Loss=0.0992 + throughput/total_tokens=683,328,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3560/500000] + optim/total_grad_norm=7.310 + train/ActionL1Loss=0.0791 + throughput/total_tokens=683,520,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3561/500000] + train/ActionL1Loss=0.0925 + throughput/total_tokens=683,712,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3562/500000] + train/ActionL1Loss=0.0758 + throughput/total_tokens=683,904,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3563/500000] + train/ActionL1Loss=0.0845 + throughput/total_tokens=684,096,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3564/500000] + train/ActionL1Loss=0.0855 + throughput/total_tokens=684,288,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3565/500000] + train/ActionL1Loss=0.0547 + throughput/total_tokens=684,480,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3566/500000] + train/ActionL1Loss=0.0796 + throughput/total_tokens=684,672,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3567/500000] + train/ActionL1Loss=0.0906 + throughput/total_tokens=684,864,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3568/500000] + train/ActionL1Loss=0.0882 + throughput/total_tokens=685,056,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3569/500000] + train/ActionL1Loss=0.0750 + throughput/total_tokens=685,248,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3570/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=685,440,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3571/500000] + train/ActionL1Loss=0.0898 + throughput/total_tokens=685,632,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3572/500000] + train/ActionL1Loss=0.0762 + throughput/total_tokens=685,824,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3573/500000] + train/ActionL1Loss=0.0795 + throughput/total_tokens=686,016,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3574/500000] + train/ActionL1Loss=0.0796 + throughput/total_tokens=686,208,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3575/500000] + train/ActionL1Loss=0.0946 + throughput/total_tokens=686,400,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3576/500000] + train/ActionL1Loss=0.0784 + throughput/total_tokens=686,592,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3577/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=686,784,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3578/500000] + train/ActionL1Loss=0.0869 + throughput/total_tokens=686,976,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3579/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=687,168,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3580/500000] + optim/total_grad_norm=5.210 + train/ActionL1Loss=0.0883 + throughput/total_tokens=687,360,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3581/500000] + train/ActionL1Loss=0.0849 + throughput/total_tokens=687,552,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3582/500000] + train/ActionL1Loss=0.0789 + throughput/total_tokens=687,744,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3583/500000] + train/ActionL1Loss=0.0878 + throughput/total_tokens=687,936,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3584/500000] + train/ActionL1Loss=0.0849 + throughput/total_tokens=688,128,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3585/500000] + train/ActionL1Loss=0.0853 + throughput/total_tokens=688,320,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3586/500000] + train/ActionL1Loss=0.0850 + throughput/total_tokens=688,512,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3587/500000] + train/ActionL1Loss=0.0739 + throughput/total_tokens=688,704,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3588/500000] + train/ActionL1Loss=0.0830 + throughput/total_tokens=688,896,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3589/500000] + train/ActionL1Loss=0.1024 + throughput/total_tokens=689,088,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3590/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=689,280,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3591/500000] + train/ActionL1Loss=0.0829 + throughput/total_tokens=689,472,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3592/500000] + train/ActionL1Loss=0.0630 + throughput/total_tokens=689,664,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3593/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=689,856,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3594/500000] + train/ActionL1Loss=0.0751 + throughput/total_tokens=690,048,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3595/500000] + train/ActionL1Loss=0.0620 + throughput/total_tokens=690,240,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3596/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=690,432,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3597/500000] + train/ActionL1Loss=0.0853 + throughput/total_tokens=690,624,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3598/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=690,816,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3599/500000] + train/ActionL1Loss=0.0684 + throughput/total_tokens=691,008,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3600/500000] + optim/total_grad_norm=5.721 + train/ActionL1Loss=0.1162 + throughput/total_tokens=691,200,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3601/500000] + train/ActionL1Loss=0.0970 + throughput/total_tokens=691,392,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3602/500000] + train/ActionL1Loss=0.0720 + throughput/total_tokens=691,584,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3603/500000] + train/ActionL1Loss=0.0856 + throughput/total_tokens=691,776,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3604/500000] + train/ActionL1Loss=0.0787 + throughput/total_tokens=691,968,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3605/500000] + train/ActionL1Loss=0.0838 + throughput/total_tokens=692,160,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3606/500000] + train/ActionL1Loss=0.0856 + throughput/total_tokens=692,352,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3607/500000] + train/ActionL1Loss=0.0948 + throughput/total_tokens=692,544,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3608/500000] + train/ActionL1Loss=0.0548 + throughput/total_tokens=692,736,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3609/500000] + train/ActionL1Loss=0.0798 + throughput/total_tokens=692,928,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3610/500000] + train/ActionL1Loss=0.0888 + throughput/total_tokens=693,120,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3611/500000] + train/ActionL1Loss=0.0869 + throughput/total_tokens=693,312,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3612/500000] + train/ActionL1Loss=0.0809 + throughput/total_tokens=693,504,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3613/500000] + train/ActionL1Loss=0.0928 + throughput/total_tokens=693,696,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3614/500000] + train/ActionL1Loss=0.0827 + throughput/total_tokens=693,888,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3615/500000] + train/ActionL1Loss=0.0954 + throughput/total_tokens=694,080,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3616/500000] + train/ActionL1Loss=0.0759 + throughput/total_tokens=694,272,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3617/500000] + train/ActionL1Loss=0.0862 + throughput/total_tokens=694,464,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3618/500000] + train/ActionL1Loss=0.0624 + throughput/total_tokens=694,656,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3619/500000] + train/ActionL1Loss=0.0829 + throughput/total_tokens=694,848,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3620/500000] + optim/total_grad_norm=6.442 + train/ActionL1Loss=0.0703 + throughput/total_tokens=695,040,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=3621/500000] + train/ActionL1Loss=0.0677 + throughput/total_tokens=695,232,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3622/500000] + train/ActionL1Loss=0.0805 + throughput/total_tokens=695,424,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3623/500000] + train/ActionL1Loss=0.0865 + throughput/total_tokens=695,616,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3624/500000] + train/ActionL1Loss=0.0818 + throughput/total_tokens=695,808,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3625/500000] + train/ActionL1Loss=0.0879 + throughput/total_tokens=696,000,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3626/500000] + train/ActionL1Loss=0.0770 + throughput/total_tokens=696,192,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3627/500000] + train/ActionL1Loss=0.1019 + throughput/total_tokens=696,384,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3628/500000] + train/ActionL1Loss=0.0729 + throughput/total_tokens=696,576,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3629/500000] + train/ActionL1Loss=0.0680 + throughput/total_tokens=696,768,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3630/500000] + train/ActionL1Loss=0.0884 + throughput/total_tokens=696,960,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3631/500000] + train/ActionL1Loss=0.0857 + throughput/total_tokens=697,152,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3632/500000] + train/ActionL1Loss=0.0674 + throughput/total_tokens=697,344,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3633/500000] + train/ActionL1Loss=0.0893 + throughput/total_tokens=697,536,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3634/500000] + train/ActionL1Loss=0.0699 + throughput/total_tokens=697,728,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3635/500000] + train/ActionL1Loss=0.0829 + throughput/total_tokens=697,920,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3636/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=698,112,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3637/500000] + train/ActionL1Loss=0.0669 + throughput/total_tokens=698,304,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3638/500000] + train/ActionL1Loss=0.0751 + throughput/total_tokens=698,496,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3639/500000] + train/ActionL1Loss=0.0740 + throughput/total_tokens=698,688,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3640/500000] + optim/total_grad_norm=7.595 + train/ActionL1Loss=0.0914 + throughput/total_tokens=698,880,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3641/500000] + train/ActionL1Loss=0.0986 + throughput/total_tokens=699,072,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3642/500000] + train/ActionL1Loss=0.0788 + throughput/total_tokens=699,264,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3643/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=699,456,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3644/500000] + train/ActionL1Loss=0.0761 + throughput/total_tokens=699,648,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3645/500000] + train/ActionL1Loss=0.0960 + throughput/total_tokens=699,840,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3646/500000] + train/ActionL1Loss=0.1049 + throughput/total_tokens=700,032,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3647/500000] + train/ActionL1Loss=0.0989 + throughput/total_tokens=700,224,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3648/500000] + train/ActionL1Loss=0.0708 + throughput/total_tokens=700,416,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3649/500000] + train/ActionL1Loss=0.0836 + throughput/total_tokens=700,608,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3650/500000] + train/ActionL1Loss=0.0785 + throughput/total_tokens=700,800,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=3651/500000] + train/ActionL1Loss=0.0860 + throughput/total_tokens=700,992,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3652/500000] + train/ActionL1Loss=0.0894 + throughput/total_tokens=701,184,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3653/500000] + train/ActionL1Loss=0.0973 + throughput/total_tokens=701,376,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3654/500000] + train/ActionL1Loss=0.0820 + throughput/total_tokens=701,568,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3655/500000] + train/ActionL1Loss=0.1097 + throughput/total_tokens=701,760,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3656/500000] + train/ActionL1Loss=0.0885 + throughput/total_tokens=701,952,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3657/500000] + train/ActionL1Loss=0.0905 + throughput/total_tokens=702,144,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3658/500000] + train/ActionL1Loss=0.0828 + throughput/total_tokens=702,336,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3659/500000] + train/ActionL1Loss=0.0788 + throughput/total_tokens=702,528,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3660/500000] + optim/total_grad_norm=6.200 + train/ActionL1Loss=0.0884 + throughput/total_tokens=702,720,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3661/500000] + train/ActionL1Loss=0.0871 + throughput/total_tokens=702,912,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3662/500000] + train/ActionL1Loss=0.0811 + throughput/total_tokens=703,104,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3663/500000] + train/ActionL1Loss=0.0938 + throughput/total_tokens=703,296,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3664/500000] + train/ActionL1Loss=0.0872 + throughput/total_tokens=703,488,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3665/500000] + train/ActionL1Loss=0.0977 + throughput/total_tokens=703,680,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3666/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=703,872,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3667/500000] + train/ActionL1Loss=0.0795 + throughput/total_tokens=704,064,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3668/500000] + train/ActionL1Loss=0.0821 + throughput/total_tokens=704,256,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3669/500000] + train/ActionL1Loss=0.0831 + throughput/total_tokens=704,448,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3670/500000] + train/ActionL1Loss=0.0940 + throughput/total_tokens=704,640,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3671/500000] + train/ActionL1Loss=0.0677 + throughput/total_tokens=704,832,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3672/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=705,024,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3673/500000] + train/ActionL1Loss=0.0707 + throughput/total_tokens=705,216,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3674/500000] + train/ActionL1Loss=0.0875 + throughput/total_tokens=705,408,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3675/500000] + train/ActionL1Loss=0.0858 + throughput/total_tokens=705,600,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3676/500000] + train/ActionL1Loss=0.0847 + throughput/total_tokens=705,792,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3677/500000] + train/ActionL1Loss=0.0812 + throughput/total_tokens=705,984,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3678/500000] + train/ActionL1Loss=0.0850 + throughput/total_tokens=706,176,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3679/500000] + train/ActionL1Loss=0.0993 + throughput/total_tokens=706,368,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3680/500000] + optim/total_grad_norm=5.651 + train/ActionL1Loss=0.0902 + throughput/total_tokens=706,560,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3681/500000] + train/ActionL1Loss=0.0863 + throughput/total_tokens=706,752,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3682/500000] + train/ActionL1Loss=0.1086 + throughput/total_tokens=706,944,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3683/500000] + train/ActionL1Loss=0.0925 + throughput/total_tokens=707,136,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3684/500000] + train/ActionL1Loss=0.0899 + throughput/total_tokens=707,328,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3685/500000] + train/ActionL1Loss=0.0885 + throughput/total_tokens=707,520,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3686/500000] + train/ActionL1Loss=0.0703 + throughput/total_tokens=707,712,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3687/500000] + train/ActionL1Loss=0.0671 + throughput/total_tokens=707,904,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3688/500000] + train/ActionL1Loss=0.0759 + throughput/total_tokens=708,096,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3689/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=708,288,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3690/500000] + train/ActionL1Loss=0.0834 + throughput/total_tokens=708,480,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=3691/500000] + train/ActionL1Loss=0.0793 + throughput/total_tokens=708,672,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3692/500000] + train/ActionL1Loss=0.0782 + throughput/total_tokens=708,864,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3693/500000] + train/ActionL1Loss=0.0813 + throughput/total_tokens=709,056,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3694/500000] + train/ActionL1Loss=0.0800 + throughput/total_tokens=709,248,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3695/500000] + train/ActionL1Loss=0.1031 + throughput/total_tokens=709,440,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3696/500000] + train/ActionL1Loss=0.0757 + throughput/total_tokens=709,632,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3697/500000] + train/ActionL1Loss=0.0963 + throughput/total_tokens=709,824,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3698/500000] + train/ActionL1Loss=0.0595 + throughput/total_tokens=710,016,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3699/500000] + train/ActionL1Loss=0.0826 + throughput/total_tokens=710,208,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3700/500000] + optim/total_grad_norm=6.060 + train/ActionL1Loss=0.0807 + throughput/total_tokens=710,400,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=3701/500000] + train/ActionL1Loss=0.0739 + throughput/total_tokens=710,592,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3702/500000] + train/ActionL1Loss=0.0788 + throughput/total_tokens=710,784,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3703/500000] + train/ActionL1Loss=0.0833 + throughput/total_tokens=710,976,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3704/500000] + train/ActionL1Loss=0.0755 + throughput/total_tokens=711,168,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3705/500000] + train/ActionL1Loss=0.0893 + throughput/total_tokens=711,360,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3706/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=711,552,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3707/500000] + train/ActionL1Loss=0.0837 + throughput/total_tokens=711,744,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3708/500000] + train/ActionL1Loss=0.0974 + throughput/total_tokens=711,936,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3709/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=712,128,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3710/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=712,320,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3711/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=712,512,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3712/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=712,704,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3713/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=712,896,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3714/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=713,088,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3715/500000] + train/ActionL1Loss=0.1078 + throughput/total_tokens=713,280,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3716/500000] + train/ActionL1Loss=0.1098 + throughput/total_tokens=713,472,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3717/500000] + train/ActionL1Loss=0.0819 + throughput/total_tokens=713,664,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3718/500000] + train/ActionL1Loss=0.0655 + throughput/total_tokens=713,856,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3719/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=714,048,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3720/500000] + optim/total_grad_norm=7.046 + train/ActionL1Loss=0.1008 + throughput/total_tokens=714,240,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=3721/500000] + train/ActionL1Loss=0.0829 + throughput/total_tokens=714,432,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3722/500000] + train/ActionL1Loss=0.0963 + throughput/total_tokens=714,624,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3723/500000] + train/ActionL1Loss=0.0802 + throughput/total_tokens=714,816,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3724/500000] + train/ActionL1Loss=0.0976 + throughput/total_tokens=715,008,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3725/500000] + train/ActionL1Loss=0.0804 + throughput/total_tokens=715,200,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3726/500000] + train/ActionL1Loss=0.0784 + throughput/total_tokens=715,392,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3727/500000] + train/ActionL1Loss=0.0907 + throughput/total_tokens=715,584,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3728/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=715,776,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3729/500000] + train/ActionL1Loss=0.0938 + throughput/total_tokens=715,968,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3730/500000] + train/ActionL1Loss=0.0782 + throughput/total_tokens=716,160,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3731/500000] + train/ActionL1Loss=0.0893 + throughput/total_tokens=716,352,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3732/500000] + train/ActionL1Loss=0.0923 + throughput/total_tokens=716,544,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3733/500000] + train/ActionL1Loss=0.0942 + throughput/total_tokens=716,736,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3734/500000] + train/ActionL1Loss=0.0931 + throughput/total_tokens=716,928,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3735/500000] + train/ActionL1Loss=0.1134 + throughput/total_tokens=717,120,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3736/500000] + train/ActionL1Loss=0.0967 + throughput/total_tokens=717,312,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3737/500000] + train/ActionL1Loss=0.1065 + throughput/total_tokens=717,504,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3738/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=717,696,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3739/500000] + train/ActionL1Loss=0.0537 + throughput/total_tokens=717,888,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3740/500000] + optim/total_grad_norm=5.588 + train/ActionL1Loss=0.0800 + throughput/total_tokens=718,080,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3741/500000] + train/ActionL1Loss=0.0697 + throughput/total_tokens=718,272,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3742/500000] + train/ActionL1Loss=0.0819 + throughput/total_tokens=718,464,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3743/500000] + train/ActionL1Loss=0.0883 + throughput/total_tokens=718,656,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3744/500000] + train/ActionL1Loss=0.1197 + throughput/total_tokens=718,848,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3745/500000] + train/ActionL1Loss=0.0662 + throughput/total_tokens=719,040,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3746/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=719,232,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3747/500000] + train/ActionL1Loss=0.1249 + throughput/total_tokens=719,424,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3748/500000] + train/ActionL1Loss=0.0958 + throughput/total_tokens=719,616,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3749/500000] + train/ActionL1Loss=0.0907 + throughput/total_tokens=719,808,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3750/500000] + train/ActionL1Loss=0.0889 + throughput/total_tokens=720,000,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=3751/500000] + train/ActionL1Loss=0.0912 + throughput/total_tokens=720,192,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3752/500000] + train/ActionL1Loss=0.0800 + throughput/total_tokens=720,384,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3753/500000] + train/ActionL1Loss=0.0884 + throughput/total_tokens=720,576,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3754/500000] + train/ActionL1Loss=0.0784 + throughput/total_tokens=720,768,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3755/500000] + train/ActionL1Loss=0.0662 + throughput/total_tokens=720,960,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3756/500000] + train/ActionL1Loss=0.0802 + throughput/total_tokens=721,152,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3757/500000] + train/ActionL1Loss=0.0913 + throughput/total_tokens=721,344,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3758/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=721,536,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3759/500000] + train/ActionL1Loss=0.0727 + throughput/total_tokens=721,728,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3760/500000] + optim/total_grad_norm=6.713 + train/ActionL1Loss=0.0858 + throughput/total_tokens=721,920,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3761/500000] + train/ActionL1Loss=0.0859 + throughput/total_tokens=722,112,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3762/500000] + train/ActionL1Loss=0.0833 + throughput/total_tokens=722,304,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3763/500000] + train/ActionL1Loss=0.0898 + throughput/total_tokens=722,496,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3764/500000] + train/ActionL1Loss=0.0931 + throughput/total_tokens=722,688,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3765/500000] + train/ActionL1Loss=0.0563 + throughput/total_tokens=722,880,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3766/500000] + train/ActionL1Loss=0.0846 + throughput/total_tokens=723,072,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3767/500000] + train/ActionL1Loss=0.0835 + throughput/total_tokens=723,264,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3768/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=723,456,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3769/500000] + train/ActionL1Loss=0.0802 + throughput/total_tokens=723,648,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3770/500000] + train/ActionL1Loss=0.0648 + throughput/total_tokens=723,840,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3771/500000] + train/ActionL1Loss=0.0803 + throughput/total_tokens=724,032,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3772/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=724,224,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3773/500000] + train/ActionL1Loss=0.0973 + throughput/total_tokens=724,416,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3774/500000] + train/ActionL1Loss=0.0869 + throughput/total_tokens=724,608,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3775/500000] + train/ActionL1Loss=0.0798 + throughput/total_tokens=724,800,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3776/500000] + train/ActionL1Loss=0.0971 + throughput/total_tokens=724,992,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3777/500000] + train/ActionL1Loss=0.0773 + throughput/total_tokens=725,184,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3778/500000] + train/ActionL1Loss=0.0901 + throughput/total_tokens=725,376,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3779/500000] + train/ActionL1Loss=0.0894 + throughput/total_tokens=725,568,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3780/500000] + optim/total_grad_norm=7.539 + train/ActionL1Loss=0.0910 + throughput/total_tokens=725,760,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3781/500000] + train/ActionL1Loss=0.1076 + throughput/total_tokens=725,952,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3782/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=726,144,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3783/500000] + train/ActionL1Loss=0.0750 + throughput/total_tokens=726,336,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3784/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=726,528,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3785/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=726,720,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3786/500000] + train/ActionL1Loss=0.0730 + throughput/total_tokens=726,912,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3787/500000] + train/ActionL1Loss=0.0989 + throughput/total_tokens=727,104,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3788/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=727,296,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3789/500000] + train/ActionL1Loss=0.0780 + throughput/total_tokens=727,488,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3790/500000] + train/ActionL1Loss=0.0802 + throughput/total_tokens=727,680,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3791/500000] + train/ActionL1Loss=0.0826 + throughput/total_tokens=727,872,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3792/500000] + train/ActionL1Loss=0.0718 + throughput/total_tokens=728,064,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3793/500000] + train/ActionL1Loss=0.0794 + throughput/total_tokens=728,256,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3794/500000] + train/ActionL1Loss=0.0739 + throughput/total_tokens=728,448,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3795/500000] + train/ActionL1Loss=0.0870 + throughput/total_tokens=728,640,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3796/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=728,832,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3797/500000] + train/ActionL1Loss=0.0806 + throughput/total_tokens=729,024,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3798/500000] + train/ActionL1Loss=0.0922 + throughput/total_tokens=729,216,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3799/500000] + train/ActionL1Loss=0.0681 + throughput/total_tokens=729,408,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3800/500000] + optim/total_grad_norm=5.345 + train/ActionL1Loss=0.0934 + throughput/total_tokens=729,600,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=3801/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=729,792,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=3802/500000] + train/ActionL1Loss=0.0950 + throughput/total_tokens=729,984,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3803/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=730,176,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3804/500000] + train/ActionL1Loss=0.0732 + throughput/total_tokens=730,368,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3805/500000] + train/ActionL1Loss=0.0931 + throughput/total_tokens=730,560,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3806/500000] + train/ActionL1Loss=0.0902 + throughput/total_tokens=730,752,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3807/500000] + train/ActionL1Loss=0.0842 + throughput/total_tokens=730,944,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3808/500000] + train/ActionL1Loss=0.0744 + throughput/total_tokens=731,136,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3809/500000] + train/ActionL1Loss=0.0747 + throughput/total_tokens=731,328,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3810/500000] + train/ActionL1Loss=0.0864 + throughput/total_tokens=731,520,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3811/500000] + train/ActionL1Loss=0.0790 + throughput/total_tokens=731,712,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3812/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=731,904,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3813/500000] + train/ActionL1Loss=0.0743 + throughput/total_tokens=732,096,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=3814/500000] + train/ActionL1Loss=0.0614 + throughput/total_tokens=732,288,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=3815/500000] + train/ActionL1Loss=0.0858 + throughput/total_tokens=732,480,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=3816/500000] + train/ActionL1Loss=0.0777 + throughput/total_tokens=732,672,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=3817/500000] + train/ActionL1Loss=0.0724 + throughput/total_tokens=732,864,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=3818/500000] + train/ActionL1Loss=0.0847 + throughput/total_tokens=733,056,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=3819/500000] + train/ActionL1Loss=0.0614 + throughput/total_tokens=733,248,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3820/500000] + optim/total_grad_norm=5.372 + train/ActionL1Loss=0.0684 + throughput/total_tokens=733,440,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3821/500000] + train/ActionL1Loss=0.0710 + throughput/total_tokens=733,632,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3822/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=733,824,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3823/500000] + train/ActionL1Loss=0.0686 + throughput/total_tokens=734,016,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3824/500000] + train/ActionL1Loss=0.0978 + throughput/total_tokens=734,208,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3825/500000] + train/ActionL1Loss=0.0867 + throughput/total_tokens=734,400,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3826/500000] + train/ActionL1Loss=0.0788 + throughput/total_tokens=734,592,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3827/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=734,784,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3828/500000] + train/ActionL1Loss=0.0772 + throughput/total_tokens=734,976,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3829/500000] + train/ActionL1Loss=0.0853 + throughput/total_tokens=735,168,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3830/500000] + train/ActionL1Loss=0.1005 + throughput/total_tokens=735,360,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3831/500000] + train/ActionL1Loss=0.0785 + throughput/total_tokens=735,552,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3832/500000] + train/ActionL1Loss=0.0792 + throughput/total_tokens=735,744,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3833/500000] + train/ActionL1Loss=0.0813 + throughput/total_tokens=735,936,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3834/500000] + train/ActionL1Loss=0.0843 + throughput/total_tokens=736,128,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3835/500000] + train/ActionL1Loss=0.0890 + throughput/total_tokens=736,320,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3836/500000] + train/ActionL1Loss=0.0923 + throughput/total_tokens=736,512,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3837/500000] + train/ActionL1Loss=0.0842 + throughput/total_tokens=736,704,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3838/500000] + train/ActionL1Loss=0.0799 + throughput/total_tokens=736,896,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3839/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=737,088,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3840/500000] + optim/total_grad_norm=5.516 + train/ActionL1Loss=0.0779 + throughput/total_tokens=737,280,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=3841/500000] + train/ActionL1Loss=0.0775 + throughput/total_tokens=737,472,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3842/500000] + train/ActionL1Loss=0.0895 + throughput/total_tokens=737,664,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3843/500000] + train/ActionL1Loss=0.0820 + throughput/total_tokens=737,856,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3844/500000] + train/ActionL1Loss=0.1159 + throughput/total_tokens=738,048,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3845/500000] + train/ActionL1Loss=0.0889 + throughput/total_tokens=738,240,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3846/500000] + train/ActionL1Loss=0.0801 + throughput/total_tokens=738,432,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3847/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=738,624,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3848/500000] + train/ActionL1Loss=0.1014 + throughput/total_tokens=738,816,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3849/500000] + train/ActionL1Loss=0.0936 + throughput/total_tokens=739,008,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3850/500000] + train/ActionL1Loss=0.0776 + throughput/total_tokens=739,200,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=3851/500000] + train/ActionL1Loss=0.0791 + throughput/total_tokens=739,392,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3852/500000] + train/ActionL1Loss=0.0876 + throughput/total_tokens=739,584,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3853/500000] + train/ActionL1Loss=0.0723 + throughput/total_tokens=739,776,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3854/500000] + train/ActionL1Loss=0.0746 + throughput/total_tokens=739,968,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3855/500000] + train/ActionL1Loss=0.0799 + throughput/total_tokens=740,160,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3856/500000] + train/ActionL1Loss=0.0808 + throughput/total_tokens=740,352,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3857/500000] + train/ActionL1Loss=0.0806 + throughput/total_tokens=740,544,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3858/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=740,736,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3859/500000] + train/ActionL1Loss=0.0698 + throughput/total_tokens=740,928,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3860/500000] + optim/total_grad_norm=6.403 + train/ActionL1Loss=0.0867 + throughput/total_tokens=741,120,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3861/500000] + train/ActionL1Loss=0.0800 + throughput/total_tokens=741,312,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3862/500000] + train/ActionL1Loss=0.0779 + throughput/total_tokens=741,504,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3863/500000] + train/ActionL1Loss=0.0893 + throughput/total_tokens=741,696,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3864/500000] + train/ActionL1Loss=0.0822 + throughput/total_tokens=741,888,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3865/500000] + train/ActionL1Loss=0.0670 + throughput/total_tokens=742,080,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3866/500000] + train/ActionL1Loss=0.0599 + throughput/total_tokens=742,272,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3867/500000] + train/ActionL1Loss=0.0704 + throughput/total_tokens=742,464,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3868/500000] + train/ActionL1Loss=0.0860 + throughput/total_tokens=742,656,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3869/500000] + train/ActionL1Loss=0.0686 + throughput/total_tokens=742,848,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3870/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=743,040,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=3871/500000] + train/ActionL1Loss=0.0963 + throughput/total_tokens=743,232,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3872/500000] + train/ActionL1Loss=0.0821 + throughput/total_tokens=743,424,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3873/500000] + train/ActionL1Loss=0.0955 + throughput/total_tokens=743,616,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3874/500000] + train/ActionL1Loss=0.0827 + throughput/total_tokens=743,808,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3875/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=744,000,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3876/500000] + train/ActionL1Loss=0.0777 + throughput/total_tokens=744,192,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3877/500000] + train/ActionL1Loss=0.0902 + throughput/total_tokens=744,384,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3878/500000] + train/ActionL1Loss=0.0865 + throughput/total_tokens=744,576,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3879/500000] + train/ActionL1Loss=0.0790 + throughput/total_tokens=744,768,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3880/500000] + optim/total_grad_norm=8.045 + train/ActionL1Loss=0.0802 + throughput/total_tokens=744,960,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=3881/500000] + train/ActionL1Loss=0.0883 + throughput/total_tokens=745,152,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3882/500000] + train/ActionL1Loss=0.0935 + throughput/total_tokens=745,344,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3883/500000] + train/ActionL1Loss=0.0636 + throughput/total_tokens=745,536,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3884/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=745,728,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3885/500000] + train/ActionL1Loss=0.1072 + throughput/total_tokens=745,920,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3886/500000] + train/ActionL1Loss=0.1003 + throughput/total_tokens=746,112,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3887/500000] + train/ActionL1Loss=0.0842 + throughput/total_tokens=746,304,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3888/500000] + train/ActionL1Loss=0.0790 + throughput/total_tokens=746,496,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3889/500000] + train/ActionL1Loss=0.0740 + throughput/total_tokens=746,688,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3890/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=746,880,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3891/500000] + train/ActionL1Loss=0.0980 + throughput/total_tokens=747,072,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3892/500000] + train/ActionL1Loss=0.0848 + throughput/total_tokens=747,264,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3893/500000] + train/ActionL1Loss=0.0828 + throughput/total_tokens=747,456,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3894/500000] + train/ActionL1Loss=0.0790 + throughput/total_tokens=747,648,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3895/500000] + train/ActionL1Loss=0.0838 + throughput/total_tokens=747,840,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3896/500000] + train/ActionL1Loss=0.0850 + throughput/total_tokens=748,032,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3897/500000] + train/ActionL1Loss=0.0790 + throughput/total_tokens=748,224,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3898/500000] + train/ActionL1Loss=0.0751 + throughput/total_tokens=748,416,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3899/500000] + train/ActionL1Loss=0.1014 + throughput/total_tokens=748,608,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3900/500000] + optim/total_grad_norm=6.681 + train/ActionL1Loss=0.0973 + throughput/total_tokens=748,800,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=3901/500000] + train/ActionL1Loss=0.0791 + throughput/total_tokens=748,992,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3902/500000] + train/ActionL1Loss=0.0804 + throughput/total_tokens=749,184,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3903/500000] + train/ActionL1Loss=0.0787 + throughput/total_tokens=749,376,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3904/500000] + train/ActionL1Loss=0.1133 + throughput/total_tokens=749,568,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3905/500000] + train/ActionL1Loss=0.0855 + throughput/total_tokens=749,760,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3906/500000] + train/ActionL1Loss=0.0678 + throughput/total_tokens=749,952,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3907/500000] + train/ActionL1Loss=0.0818 + throughput/total_tokens=750,144,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3908/500000] + train/ActionL1Loss=0.0925 + throughput/total_tokens=750,336,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3909/500000] + train/ActionL1Loss=0.0997 + throughput/total_tokens=750,528,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=3910/500000] + train/ActionL1Loss=0.0871 + throughput/total_tokens=750,720,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=3911/500000] + train/ActionL1Loss=0.0860 + throughput/total_tokens=750,912,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3912/500000] + train/ActionL1Loss=0.0882 + throughput/total_tokens=751,104,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=3913/500000] + train/ActionL1Loss=0.0813 + throughput/total_tokens=751,296,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3914/500000] + train/ActionL1Loss=0.0854 + throughput/total_tokens=751,488,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3915/500000] + train/ActionL1Loss=0.0804 + throughput/total_tokens=751,680,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3916/500000] + train/ActionL1Loss=0.0752 + throughput/total_tokens=751,872,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3917/500000] + train/ActionL1Loss=0.0863 + throughput/total_tokens=752,064,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3918/500000] + train/ActionL1Loss=0.0845 + throughput/total_tokens=752,256,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3919/500000] + train/ActionL1Loss=0.0847 + throughput/total_tokens=752,448,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=3920/500000] + optim/total_grad_norm=6.155 + train/ActionL1Loss=0.0766 + throughput/total_tokens=752,640,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3921/500000] + train/ActionL1Loss=0.0816 + throughput/total_tokens=752,832,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3922/500000] + train/ActionL1Loss=0.0828 + throughput/total_tokens=753,024,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3923/500000] + train/ActionL1Loss=0.0851 + throughput/total_tokens=753,216,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3924/500000] + train/ActionL1Loss=0.0919 + throughput/total_tokens=753,408,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3925/500000] + train/ActionL1Loss=0.0796 + throughput/total_tokens=753,600,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3926/500000] + train/ActionL1Loss=0.0878 + throughput/total_tokens=753,792,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3927/500000] + train/ActionL1Loss=0.0749 + throughput/total_tokens=753,984,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3928/500000] + train/ActionL1Loss=0.0775 + throughput/total_tokens=754,176,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3929/500000] + train/ActionL1Loss=0.0750 + throughput/total_tokens=754,368,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3930/500000] + train/ActionL1Loss=0.0875 + throughput/total_tokens=754,560,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3931/500000] + train/ActionL1Loss=0.0890 + throughput/total_tokens=754,752,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3932/500000] + train/ActionL1Loss=0.0697 + throughput/total_tokens=754,944,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3933/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=755,136,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3934/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=755,328,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3935/500000] + train/ActionL1Loss=0.0906 + throughput/total_tokens=755,520,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3936/500000] + train/ActionL1Loss=0.0620 + throughput/total_tokens=755,712,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3937/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=755,904,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3938/500000] + train/ActionL1Loss=0.0864 + throughput/total_tokens=756,096,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3939/500000] + train/ActionL1Loss=0.0720 + throughput/total_tokens=756,288,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3940/500000] + optim/total_grad_norm=6.576 + train/ActionL1Loss=0.0755 + throughput/total_tokens=756,480,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=3941/500000] + train/ActionL1Loss=0.0720 + throughput/total_tokens=756,672,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3942/500000] + train/ActionL1Loss=0.0709 + throughput/total_tokens=756,864,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3943/500000] + train/ActionL1Loss=0.0872 + throughput/total_tokens=757,056,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3944/500000] + train/ActionL1Loss=0.0673 + throughput/total_tokens=757,248,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3945/500000] + train/ActionL1Loss=0.0788 + throughput/total_tokens=757,440,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3946/500000] + train/ActionL1Loss=0.0675 + throughput/total_tokens=757,632,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3947/500000] + train/ActionL1Loss=0.0771 + throughput/total_tokens=757,824,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3948/500000] + train/ActionL1Loss=0.0847 + throughput/total_tokens=758,016,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3949/500000] + train/ActionL1Loss=0.0776 + throughput/total_tokens=758,208,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3950/500000] + train/ActionL1Loss=0.0870 + throughput/total_tokens=758,400,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=3951/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=758,592,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3952/500000] + train/ActionL1Loss=0.0826 + throughput/total_tokens=758,784,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3953/500000] + train/ActionL1Loss=0.0652 + throughput/total_tokens=758,976,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3954/500000] + train/ActionL1Loss=0.0882 + throughput/total_tokens=759,168,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=3955/500000] + train/ActionL1Loss=0.0938 + throughput/total_tokens=759,360,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3956/500000] + train/ActionL1Loss=0.0864 + throughput/total_tokens=759,552,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3957/500000] + train/ActionL1Loss=0.0832 + throughput/total_tokens=759,744,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3958/500000] + train/ActionL1Loss=0.0958 + throughput/total_tokens=759,936,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3959/500000] + train/ActionL1Loss=0.0761 + throughput/total_tokens=760,128,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3960/500000] + optim/total_grad_norm=5.031 + train/ActionL1Loss=0.0686 + throughput/total_tokens=760,320,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3961/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=760,512,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3962/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=760,704,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3963/500000] + train/ActionL1Loss=0.0714 + throughput/total_tokens=760,896,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3964/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=761,088,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3965/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=761,280,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3966/500000] + train/ActionL1Loss=0.0738 + throughput/total_tokens=761,472,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3967/500000] + train/ActionL1Loss=0.0716 + throughput/total_tokens=761,664,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3968/500000] + train/ActionL1Loss=0.0736 + throughput/total_tokens=761,856,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3969/500000] + train/ActionL1Loss=0.0863 + throughput/total_tokens=762,048,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=3970/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=762,240,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=3971/500000] + train/ActionL1Loss=0.0863 + throughput/total_tokens=762,432,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=3972/500000] + train/ActionL1Loss=0.0753 + throughput/total_tokens=762,624,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=3973/500000] + train/ActionL1Loss=0.0849 + throughput/total_tokens=762,816,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=3974/500000] + train/ActionL1Loss=0.0761 + throughput/total_tokens=763,008,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3975/500000] + train/ActionL1Loss=0.0758 + throughput/total_tokens=763,200,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3976/500000] + train/ActionL1Loss=0.0788 + throughput/total_tokens=763,392,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3977/500000] + train/ActionL1Loss=0.0893 + throughput/total_tokens=763,584,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3978/500000] + train/ActionL1Loss=0.0743 + throughput/total_tokens=763,776,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3979/500000] + train/ActionL1Loss=0.0833 + throughput/total_tokens=763,968,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3980/500000] + optim/total_grad_norm=6.595 + train/ActionL1Loss=0.0582 + throughput/total_tokens=764,160,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=3981/500000] + train/ActionL1Loss=0.0848 + throughput/total_tokens=764,352,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3982/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=764,544,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3983/500000] + train/ActionL1Loss=0.0929 + throughput/total_tokens=764,736,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3984/500000] + train/ActionL1Loss=0.0656 + throughput/total_tokens=764,928,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3985/500000] + train/ActionL1Loss=0.0754 + throughput/total_tokens=765,120,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3986/500000] + train/ActionL1Loss=0.0859 + throughput/total_tokens=765,312,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3987/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=765,504,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3988/500000] + train/ActionL1Loss=0.0616 + throughput/total_tokens=765,696,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3989/500000] + train/ActionL1Loss=0.0903 + throughput/total_tokens=765,888,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3990/500000] + train/ActionL1Loss=0.0904 + throughput/total_tokens=766,080,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=3991/500000] + train/ActionL1Loss=0.0950 + throughput/total_tokens=766,272,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=3992/500000] + train/ActionL1Loss=0.0851 + throughput/total_tokens=766,464,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3993/500000] + train/ActionL1Loss=0.0841 + throughput/total_tokens=766,656,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3994/500000] + train/ActionL1Loss=0.0786 + throughput/total_tokens=766,848,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=3995/500000] + train/ActionL1Loss=0.0873 + throughput/total_tokens=767,040,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3996/500000] + train/ActionL1Loss=0.0883 + throughput/total_tokens=767,232,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3997/500000] + train/ActionL1Loss=0.0935 + throughput/total_tokens=767,424,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3998/500000] + train/ActionL1Loss=0.0862 + throughput/total_tokens=767,616,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=3999/500000] + train/ActionL1Loss=0.0989 + throughput/total_tokens=767,808,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4000/500000] + optim/total_grad_norm=5.346 + train/ActionL1Loss=0.0729 + throughput/total_tokens=768,000,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +10/01 [12:47:00] INFO | >> Saving config... checkpoint.py:608 +10/01 [12:47:33] INFO | >> Saving model state... checkpoint.py:796 +10/01 [12:48:45] INFO | >> Saving optim state... checkpoint.py:811 +10/01 [12:50:13] INFO | >> Saving trainer state... checkpoint.py:670 + INFO | >> Saving config... checkpoint.py:608 +[step=4001/500000] + train/ActionL1Loss=0.0878 + throughput/total_tokens=768,192,000 + throughput/device/tokens_per_second=1,208 + throughput/device/batches_per_second=0.0504 +[step=4002/500000] + train/ActionL1Loss=0.0780 + throughput/total_tokens=768,384,000 + throughput/device/tokens_per_second=1,186 + throughput/device/batches_per_second=0.0494 +[step=4003/500000] + train/ActionL1Loss=0.1040 + throughput/total_tokens=768,576,000 + throughput/device/tokens_per_second=1,178 + throughput/device/batches_per_second=0.0491 +[step=4004/500000] + train/ActionL1Loss=0.0966 + throughput/total_tokens=768,768,000 + throughput/device/tokens_per_second=1,174 + throughput/device/batches_per_second=0.0489 +[step=4005/500000] + train/ActionL1Loss=0.0900 + throughput/total_tokens=768,960,000 + throughput/device/tokens_per_second=1,172 + throughput/device/batches_per_second=0.0488 +[step=4006/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=769,152,000 + throughput/device/tokens_per_second=1,170 + throughput/device/batches_per_second=0.0488 +[step=4007/500000] + train/ActionL1Loss=0.0888 + throughput/total_tokens=769,344,000 + throughput/device/tokens_per_second=1,170 + throughput/device/batches_per_second=0.0488 +[step=4008/500000] + train/ActionL1Loss=0.1063 + throughput/total_tokens=769,536,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=4009/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=769,728,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=4010/500000] + train/ActionL1Loss=0.0784 + throughput/total_tokens=769,920,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=4011/500000] + train/ActionL1Loss=0.0660 + throughput/total_tokens=770,112,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=4012/500000] + train/ActionL1Loss=0.0803 + throughput/total_tokens=770,304,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=4013/500000] + train/ActionL1Loss=0.0942 + throughput/total_tokens=770,496,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4014/500000] + train/ActionL1Loss=0.0714 + throughput/total_tokens=770,688,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4015/500000] + train/ActionL1Loss=0.0748 + throughput/total_tokens=770,880,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4016/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=771,072,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4017/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=771,264,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4018/500000] + train/ActionL1Loss=0.0738 + throughput/total_tokens=771,456,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4019/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=771,648,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4020/500000] + optim/total_grad_norm=7.769 + train/ActionL1Loss=0.0791 + throughput/total_tokens=771,840,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4021/500000] + train/ActionL1Loss=0.0858 + throughput/total_tokens=772,032,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4022/500000] + train/ActionL1Loss=0.0664 + throughput/total_tokens=772,224,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4023/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=772,416,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4024/500000] + train/ActionL1Loss=0.0849 + throughput/total_tokens=772,608,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4025/500000] + train/ActionL1Loss=0.0761 + throughput/total_tokens=772,800,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4026/500000] + train/ActionL1Loss=0.0785 + throughput/total_tokens=772,992,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4027/500000] + train/ActionL1Loss=0.0787 + throughput/total_tokens=773,184,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4028/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=773,376,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4029/500000] + train/ActionL1Loss=0.0877 + throughput/total_tokens=773,568,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4030/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=773,760,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4031/500000] + train/ActionL1Loss=0.0816 + throughput/total_tokens=773,952,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4032/500000] + train/ActionL1Loss=0.0755 + throughput/total_tokens=774,144,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4033/500000] + train/ActionL1Loss=0.0802 + throughput/total_tokens=774,336,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4034/500000] + train/ActionL1Loss=0.0743 + throughput/total_tokens=774,528,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4035/500000] + train/ActionL1Loss=0.0789 + throughput/total_tokens=774,720,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4036/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=774,912,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4037/500000] + train/ActionL1Loss=0.0727 + throughput/total_tokens=775,104,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4038/500000] + train/ActionL1Loss=0.0723 + throughput/total_tokens=775,296,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4039/500000] + train/ActionL1Loss=0.0780 + throughput/total_tokens=775,488,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4040/500000] + optim/total_grad_norm=7.766 + train/ActionL1Loss=0.0768 + throughput/total_tokens=775,680,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4041/500000] + train/ActionL1Loss=0.0983 + throughput/total_tokens=775,872,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4042/500000] + train/ActionL1Loss=0.0864 + throughput/total_tokens=776,064,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4043/500000] + train/ActionL1Loss=0.0623 + throughput/total_tokens=776,256,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4044/500000] + train/ActionL1Loss=0.0866 + throughput/total_tokens=776,448,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4045/500000] + train/ActionL1Loss=0.0723 + throughput/total_tokens=776,640,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4046/500000] + train/ActionL1Loss=0.0648 + throughput/total_tokens=776,832,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4047/500000] + train/ActionL1Loss=0.0835 + throughput/total_tokens=777,024,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4048/500000] + train/ActionL1Loss=0.0893 + throughput/total_tokens=777,216,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4049/500000] + train/ActionL1Loss=0.0709 + throughput/total_tokens=777,408,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4050/500000] + train/ActionL1Loss=0.1044 + throughput/total_tokens=777,600,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4051/500000] + train/ActionL1Loss=0.0874 + throughput/total_tokens=777,792,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4052/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=777,984,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4053/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=778,176,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4054/500000] + train/ActionL1Loss=0.0757 + throughput/total_tokens=778,368,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4055/500000] + train/ActionL1Loss=0.0757 + throughput/total_tokens=778,560,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4056/500000] + train/ActionL1Loss=0.0769 + throughput/total_tokens=778,752,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4057/500000] + train/ActionL1Loss=0.0835 + throughput/total_tokens=778,944,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4058/500000] + train/ActionL1Loss=0.0796 + throughput/total_tokens=779,136,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4059/500000] + train/ActionL1Loss=0.1051 + throughput/total_tokens=779,328,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4060/500000] + optim/total_grad_norm=7.752 + train/ActionL1Loss=0.0921 + throughput/total_tokens=779,520,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=4061/500000] + train/ActionL1Loss=0.0637 + throughput/total_tokens=779,712,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4062/500000] + train/ActionL1Loss=0.0788 + throughput/total_tokens=779,904,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4063/500000] + train/ActionL1Loss=0.0830 + throughput/total_tokens=780,096,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4064/500000] + train/ActionL1Loss=0.0762 + throughput/total_tokens=780,288,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4065/500000] + train/ActionL1Loss=0.0687 + throughput/total_tokens=780,480,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4066/500000] + train/ActionL1Loss=0.0982 + throughput/total_tokens=780,672,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4067/500000] + train/ActionL1Loss=0.0891 + throughput/total_tokens=780,864,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4068/500000] + train/ActionL1Loss=0.0873 + throughput/total_tokens=781,056,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4069/500000] + train/ActionL1Loss=0.0827 + throughput/total_tokens=781,248,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4070/500000] + train/ActionL1Loss=0.0816 + throughput/total_tokens=781,440,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4071/500000] + train/ActionL1Loss=0.1063 + throughput/total_tokens=781,632,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4072/500000] + train/ActionL1Loss=0.0778 + throughput/total_tokens=781,824,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4073/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=782,016,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4074/500000] + train/ActionL1Loss=0.0969 + throughput/total_tokens=782,208,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4075/500000] + train/ActionL1Loss=0.0808 + throughput/total_tokens=782,400,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4076/500000] + train/ActionL1Loss=0.0917 + throughput/total_tokens=782,592,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4077/500000] + train/ActionL1Loss=0.0762 + throughput/total_tokens=782,784,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4078/500000] + train/ActionL1Loss=0.1176 + throughput/total_tokens=782,976,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4079/500000] + train/ActionL1Loss=0.0936 + throughput/total_tokens=783,168,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4080/500000] + optim/total_grad_norm=5.745 + train/ActionL1Loss=0.0628 + throughput/total_tokens=783,360,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4081/500000] + train/ActionL1Loss=0.0856 + throughput/total_tokens=783,552,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4082/500000] + train/ActionL1Loss=0.0970 + throughput/total_tokens=783,744,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4083/500000] + train/ActionL1Loss=0.0803 + throughput/total_tokens=783,936,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4084/500000] + train/ActionL1Loss=0.0758 + throughput/total_tokens=784,128,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4085/500000] + train/ActionL1Loss=0.0824 + throughput/total_tokens=784,320,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4086/500000] + train/ActionL1Loss=0.0752 + throughput/total_tokens=784,512,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4087/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=784,704,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4088/500000] + train/ActionL1Loss=0.0828 + throughput/total_tokens=784,896,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4089/500000] + train/ActionL1Loss=0.0835 + throughput/total_tokens=785,088,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4090/500000] + train/ActionL1Loss=0.0796 + throughput/total_tokens=785,280,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4091/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=785,472,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4092/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=785,664,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4093/500000] + train/ActionL1Loss=0.0691 + throughput/total_tokens=785,856,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4094/500000] + train/ActionL1Loss=0.0790 + throughput/total_tokens=786,048,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4095/500000] + train/ActionL1Loss=0.0761 + throughput/total_tokens=786,240,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4096/500000] + train/ActionL1Loss=0.0779 + throughput/total_tokens=786,432,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4097/500000] + train/ActionL1Loss=0.0721 + throughput/total_tokens=786,624,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4098/500000] + train/ActionL1Loss=0.0717 + throughput/total_tokens=786,816,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4099/500000] + train/ActionL1Loss=0.0789 + throughput/total_tokens=787,008,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4100/500000] + optim/total_grad_norm=4.718 + train/ActionL1Loss=0.0788 + throughput/total_tokens=787,200,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4101/500000] + train/ActionL1Loss=0.0738 + throughput/total_tokens=787,392,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4102/500000] + train/ActionL1Loss=0.0624 + throughput/total_tokens=787,584,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4103/500000] + train/ActionL1Loss=0.0676 + throughput/total_tokens=787,776,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4104/500000] + train/ActionL1Loss=0.1070 + throughput/total_tokens=787,968,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4105/500000] + train/ActionL1Loss=0.0621 + throughput/total_tokens=788,160,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4106/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=788,352,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4107/500000] + train/ActionL1Loss=0.0793 + throughput/total_tokens=788,544,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4108/500000] + train/ActionL1Loss=0.0819 + throughput/total_tokens=788,736,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4109/500000] + train/ActionL1Loss=0.0833 + throughput/total_tokens=788,928,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4110/500000] + train/ActionL1Loss=0.0809 + throughput/total_tokens=789,120,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=4111/500000] + train/ActionL1Loss=0.0746 + throughput/total_tokens=789,312,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4112/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=789,504,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4113/500000] + train/ActionL1Loss=0.0831 + throughput/total_tokens=789,696,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4114/500000] + train/ActionL1Loss=0.0862 + throughput/total_tokens=789,888,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4115/500000] + train/ActionL1Loss=0.0790 + throughput/total_tokens=790,080,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4116/500000] + train/ActionL1Loss=0.0834 + throughput/total_tokens=790,272,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4117/500000] + train/ActionL1Loss=0.0925 + throughput/total_tokens=790,464,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4118/500000] + train/ActionL1Loss=0.0773 + throughput/total_tokens=790,656,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4119/500000] + train/ActionL1Loss=0.0662 + throughput/total_tokens=790,848,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4120/500000] + optim/total_grad_norm=5.062 + train/ActionL1Loss=0.1062 + throughput/total_tokens=791,040,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4121/500000] + train/ActionL1Loss=0.0800 + throughput/total_tokens=791,232,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4122/500000] + train/ActionL1Loss=0.0894 + throughput/total_tokens=791,424,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4123/500000] + train/ActionL1Loss=0.0896 + throughput/total_tokens=791,616,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4124/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=791,808,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4125/500000] + train/ActionL1Loss=0.0962 + throughput/total_tokens=792,000,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4126/500000] + train/ActionL1Loss=0.0782 + throughput/total_tokens=792,192,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4127/500000] + train/ActionL1Loss=0.0755 + throughput/total_tokens=792,384,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4128/500000] + train/ActionL1Loss=0.0835 + throughput/total_tokens=792,576,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4129/500000] + train/ActionL1Loss=0.0738 + throughput/total_tokens=792,768,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4130/500000] + train/ActionL1Loss=0.0861 + throughput/total_tokens=792,960,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4131/500000] + train/ActionL1Loss=0.0838 + throughput/total_tokens=793,152,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4132/500000] + train/ActionL1Loss=0.0770 + throughput/total_tokens=793,344,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4133/500000] + train/ActionL1Loss=0.0799 + throughput/total_tokens=793,536,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4134/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=793,728,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4135/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=793,920,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4136/500000] + train/ActionL1Loss=0.0721 + throughput/total_tokens=794,112,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4137/500000] + train/ActionL1Loss=0.0703 + throughput/total_tokens=794,304,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4138/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=794,496,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4139/500000] + train/ActionL1Loss=0.0629 + throughput/total_tokens=794,688,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4140/500000] + optim/total_grad_norm=5.645 + train/ActionL1Loss=0.0715 + throughput/total_tokens=794,880,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4141/500000] + train/ActionL1Loss=0.0691 + throughput/total_tokens=795,072,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4142/500000] + train/ActionL1Loss=0.0608 + throughput/total_tokens=795,264,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4143/500000] + train/ActionL1Loss=0.0713 + throughput/total_tokens=795,456,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4144/500000] + train/ActionL1Loss=0.0706 + throughput/total_tokens=795,648,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4145/500000] + train/ActionL1Loss=0.0660 + throughput/total_tokens=795,840,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4146/500000] + train/ActionL1Loss=0.0804 + throughput/total_tokens=796,032,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4147/500000] + train/ActionL1Loss=0.0677 + throughput/total_tokens=796,224,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4148/500000] + train/ActionL1Loss=0.0956 + throughput/total_tokens=796,416,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4149/500000] + train/ActionL1Loss=0.0827 + throughput/total_tokens=796,608,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4150/500000] + train/ActionL1Loss=0.0770 + throughput/total_tokens=796,800,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4151/500000] + train/ActionL1Loss=0.0884 + throughput/total_tokens=796,992,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4152/500000] + train/ActionL1Loss=0.0728 + throughput/total_tokens=797,184,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4153/500000] + train/ActionL1Loss=0.0828 + throughput/total_tokens=797,376,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4154/500000] + train/ActionL1Loss=0.0958 + throughput/total_tokens=797,568,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4155/500000] + train/ActionL1Loss=0.0746 + throughput/total_tokens=797,760,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4156/500000] + train/ActionL1Loss=0.0745 + throughput/total_tokens=797,952,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4157/500000] + train/ActionL1Loss=0.0814 + throughput/total_tokens=798,144,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4158/500000] + train/ActionL1Loss=0.0785 + throughput/total_tokens=798,336,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4159/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=798,528,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4160/500000] + optim/total_grad_norm=7.920 + train/ActionL1Loss=0.0874 + throughput/total_tokens=798,720,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=4161/500000] + train/ActionL1Loss=0.0838 + throughput/total_tokens=798,912,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4162/500000] + train/ActionL1Loss=0.0761 + throughput/total_tokens=799,104,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4163/500000] + train/ActionL1Loss=0.0806 + throughput/total_tokens=799,296,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4164/500000] + train/ActionL1Loss=0.0708 + throughput/total_tokens=799,488,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4165/500000] + train/ActionL1Loss=0.0875 + throughput/total_tokens=799,680,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4166/500000] + train/ActionL1Loss=0.0848 + throughput/total_tokens=799,872,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4167/500000] + train/ActionL1Loss=0.0735 + throughput/total_tokens=800,064,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4168/500000] + train/ActionL1Loss=0.0566 + throughput/total_tokens=800,256,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4169/500000] + train/ActionL1Loss=0.0734 + throughput/total_tokens=800,448,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4170/500000] + train/ActionL1Loss=0.0609 + throughput/total_tokens=800,640,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4171/500000] + train/ActionL1Loss=0.0805 + throughput/total_tokens=800,832,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4172/500000] + train/ActionL1Loss=0.0924 + throughput/total_tokens=801,024,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4173/500000] + train/ActionL1Loss=0.0702 + throughput/total_tokens=801,216,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4174/500000] + train/ActionL1Loss=0.0820 + throughput/total_tokens=801,408,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4175/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=801,600,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4176/500000] + train/ActionL1Loss=0.0556 + throughput/total_tokens=801,792,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4177/500000] + train/ActionL1Loss=0.0750 + throughput/total_tokens=801,984,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4178/500000] + train/ActionL1Loss=0.0729 + throughput/total_tokens=802,176,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4179/500000] + train/ActionL1Loss=0.0630 + throughput/total_tokens=802,368,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4180/500000] + optim/total_grad_norm=5.140 + train/ActionL1Loss=0.0912 + throughput/total_tokens=802,560,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4181/500000] + train/ActionL1Loss=0.0779 + throughput/total_tokens=802,752,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4182/500000] + train/ActionL1Loss=0.0703 + throughput/total_tokens=802,944,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4183/500000] + train/ActionL1Loss=0.0802 + throughput/total_tokens=803,136,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4184/500000] + train/ActionL1Loss=0.0752 + throughput/total_tokens=803,328,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4185/500000] + train/ActionL1Loss=0.0820 + throughput/total_tokens=803,520,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4186/500000] + train/ActionL1Loss=0.0874 + throughput/total_tokens=803,712,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4187/500000] + train/ActionL1Loss=0.0779 + throughput/total_tokens=803,904,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4188/500000] + train/ActionL1Loss=0.0706 + throughput/total_tokens=804,096,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4189/500000] + train/ActionL1Loss=0.1050 + throughput/total_tokens=804,288,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4190/500000] + train/ActionL1Loss=0.0637 + throughput/total_tokens=804,480,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4191/500000] + train/ActionL1Loss=0.0972 + throughput/total_tokens=804,672,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4192/500000] + train/ActionL1Loss=0.0751 + throughput/total_tokens=804,864,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4193/500000] + train/ActionL1Loss=0.0745 + throughput/total_tokens=805,056,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4194/500000] + train/ActionL1Loss=0.0988 + throughput/total_tokens=805,248,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4195/500000] + train/ActionL1Loss=0.0799 + throughput/total_tokens=805,440,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4196/500000] + train/ActionL1Loss=0.0829 + throughput/total_tokens=805,632,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4197/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=805,824,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4198/500000] + train/ActionL1Loss=0.0672 + throughput/total_tokens=806,016,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4199/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=806,208,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4200/500000] + optim/total_grad_norm=6.487 + train/ActionL1Loss=0.0899 + throughput/total_tokens=806,400,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4201/500000] + train/ActionL1Loss=0.0837 + throughput/total_tokens=806,592,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4202/500000] + train/ActionL1Loss=0.0934 + throughput/total_tokens=806,784,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4203/500000] + train/ActionL1Loss=0.0734 + throughput/total_tokens=806,976,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4204/500000] + train/ActionL1Loss=0.0846 + throughput/total_tokens=807,168,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4205/500000] + train/ActionL1Loss=0.0787 + throughput/total_tokens=807,360,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4206/500000] + train/ActionL1Loss=0.0910 + throughput/total_tokens=807,552,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4207/500000] + train/ActionL1Loss=0.0823 + throughput/total_tokens=807,744,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4208/500000] + train/ActionL1Loss=0.0851 + throughput/total_tokens=807,936,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4209/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=808,128,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4210/500000] + train/ActionL1Loss=0.0633 + throughput/total_tokens=808,320,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=4211/500000] + train/ActionL1Loss=0.0689 + throughput/total_tokens=808,512,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4212/500000] + train/ActionL1Loss=0.0768 + throughput/total_tokens=808,704,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4213/500000] + train/ActionL1Loss=0.0925 + throughput/total_tokens=808,896,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4214/500000] + train/ActionL1Loss=0.0717 + throughput/total_tokens=809,088,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4215/500000] + train/ActionL1Loss=0.0769 + throughput/total_tokens=809,280,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4216/500000] + train/ActionL1Loss=0.0850 + throughput/total_tokens=809,472,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4217/500000] + train/ActionL1Loss=0.0803 + throughput/total_tokens=809,664,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4218/500000] + train/ActionL1Loss=0.0739 + throughput/total_tokens=809,856,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4219/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=810,048,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4220/500000] + optim/total_grad_norm=5.329 + train/ActionL1Loss=0.0866 + throughput/total_tokens=810,240,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4221/500000] + train/ActionL1Loss=0.0800 + throughput/total_tokens=810,432,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4222/500000] + train/ActionL1Loss=0.0708 + throughput/total_tokens=810,624,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4223/500000] + train/ActionL1Loss=0.0830 + throughput/total_tokens=810,816,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4224/500000] + train/ActionL1Loss=0.0863 + throughput/total_tokens=811,008,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4225/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=811,200,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4226/500000] + train/ActionL1Loss=0.0897 + throughput/total_tokens=811,392,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4227/500000] + train/ActionL1Loss=0.0942 + throughput/total_tokens=811,584,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4228/500000] + train/ActionL1Loss=0.0823 + throughput/total_tokens=811,776,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4229/500000] + train/ActionL1Loss=0.1045 + throughput/total_tokens=811,968,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4230/500000] + train/ActionL1Loss=0.0710 + throughput/total_tokens=812,160,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4231/500000] + train/ActionL1Loss=0.0877 + throughput/total_tokens=812,352,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4232/500000] + train/ActionL1Loss=0.0809 + throughput/total_tokens=812,544,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4233/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=812,736,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4234/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=812,928,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4235/500000] + train/ActionL1Loss=0.0649 + throughput/total_tokens=813,120,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4236/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=813,312,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4237/500000] + train/ActionL1Loss=0.0852 + throughput/total_tokens=813,504,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4238/500000] + train/ActionL1Loss=0.0730 + throughput/total_tokens=813,696,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4239/500000] + train/ActionL1Loss=0.0980 + throughput/total_tokens=813,888,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4240/500000] + optim/total_grad_norm=5.508 + train/ActionL1Loss=0.1117 + throughput/total_tokens=814,080,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4241/500000] + train/ActionL1Loss=0.0637 + throughput/total_tokens=814,272,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4242/500000] + train/ActionL1Loss=0.0713 + throughput/total_tokens=814,464,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4243/500000] + train/ActionL1Loss=0.0859 + throughput/total_tokens=814,656,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4244/500000] + train/ActionL1Loss=0.0769 + throughput/total_tokens=814,848,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4245/500000] + train/ActionL1Loss=0.0795 + throughput/total_tokens=815,040,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4246/500000] + train/ActionL1Loss=0.0901 + throughput/total_tokens=815,232,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4247/500000] + train/ActionL1Loss=0.0910 + throughput/total_tokens=815,424,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4248/500000] + train/ActionL1Loss=0.0738 + throughput/total_tokens=815,616,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4249/500000] + train/ActionL1Loss=0.0828 + throughput/total_tokens=815,808,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4250/500000] + train/ActionL1Loss=0.0748 + throughput/total_tokens=816,000,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4251/500000] + train/ActionL1Loss=0.0781 + throughput/total_tokens=816,192,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4252/500000] + train/ActionL1Loss=0.0736 + throughput/total_tokens=816,384,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4253/500000] + train/ActionL1Loss=0.0687 + throughput/total_tokens=816,576,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4254/500000] + train/ActionL1Loss=0.0953 + throughput/total_tokens=816,768,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4255/500000] + train/ActionL1Loss=0.0916 + throughput/total_tokens=816,960,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4256/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=817,152,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4257/500000] + train/ActionL1Loss=0.0868 + throughput/total_tokens=817,344,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4258/500000] + train/ActionL1Loss=0.0762 + throughput/total_tokens=817,536,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4259/500000] + train/ActionL1Loss=0.0658 + throughput/total_tokens=817,728,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4260/500000] + optim/total_grad_norm=7.235 + train/ActionL1Loss=0.0804 + throughput/total_tokens=817,920,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=4261/500000] + train/ActionL1Loss=0.0754 + throughput/total_tokens=818,112,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4262/500000] + train/ActionL1Loss=0.0606 + throughput/total_tokens=818,304,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4263/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=818,496,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4264/500000] + train/ActionL1Loss=0.0978 + throughput/total_tokens=818,688,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4265/500000] + train/ActionL1Loss=0.0873 + throughput/total_tokens=818,880,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4266/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=819,072,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4267/500000] + train/ActionL1Loss=0.0877 + throughput/total_tokens=819,264,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4268/500000] + train/ActionL1Loss=0.0707 + throughput/total_tokens=819,456,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4269/500000] + train/ActionL1Loss=0.0878 + throughput/total_tokens=819,648,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4270/500000] + train/ActionL1Loss=0.0786 + throughput/total_tokens=819,840,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4271/500000] + train/ActionL1Loss=0.0678 + throughput/total_tokens=820,032,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4272/500000] + train/ActionL1Loss=0.0814 + throughput/total_tokens=820,224,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4273/500000] + train/ActionL1Loss=0.0867 + throughput/total_tokens=820,416,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4274/500000] + train/ActionL1Loss=0.0653 + throughput/total_tokens=820,608,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4275/500000] + train/ActionL1Loss=0.0800 + throughput/total_tokens=820,800,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4276/500000] + train/ActionL1Loss=0.0771 + throughput/total_tokens=820,992,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4277/500000] + train/ActionL1Loss=0.0691 + throughput/total_tokens=821,184,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4278/500000] + train/ActionL1Loss=0.0771 + throughput/total_tokens=821,376,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4279/500000] + train/ActionL1Loss=0.0829 + throughput/total_tokens=821,568,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4280/500000] + optim/total_grad_norm=7.295 + train/ActionL1Loss=0.1101 + throughput/total_tokens=821,760,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4281/500000] + train/ActionL1Loss=0.0776 + throughput/total_tokens=821,952,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4282/500000] + train/ActionL1Loss=0.0592 + throughput/total_tokens=822,144,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4283/500000] + train/ActionL1Loss=0.0780 + throughput/total_tokens=822,336,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4284/500000] + train/ActionL1Loss=0.0683 + throughput/total_tokens=822,528,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4285/500000] + train/ActionL1Loss=0.0773 + throughput/total_tokens=822,720,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4286/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=822,912,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4287/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=823,104,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4288/500000] + train/ActionL1Loss=0.0829 + throughput/total_tokens=823,296,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4289/500000] + train/ActionL1Loss=0.0805 + throughput/total_tokens=823,488,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4290/500000] + train/ActionL1Loss=0.0738 + throughput/total_tokens=823,680,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4291/500000] + train/ActionL1Loss=0.0791 + throughput/total_tokens=823,872,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4292/500000] + train/ActionL1Loss=0.0914 + throughput/total_tokens=824,064,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4293/500000] + train/ActionL1Loss=0.0866 + throughput/total_tokens=824,256,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4294/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=824,448,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4295/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=824,640,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4296/500000] + train/ActionL1Loss=0.0669 + throughput/total_tokens=824,832,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4297/500000] + train/ActionL1Loss=0.0698 + throughput/total_tokens=825,024,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4298/500000] + train/ActionL1Loss=0.0824 + throughput/total_tokens=825,216,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4299/500000] + train/ActionL1Loss=0.0772 + throughput/total_tokens=825,408,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4300/500000] + optim/total_grad_norm=6.946 + train/ActionL1Loss=0.0782 + throughput/total_tokens=825,600,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4301/500000] + train/ActionL1Loss=0.0641 + throughput/total_tokens=825,792,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4302/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=825,984,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4303/500000] + train/ActionL1Loss=0.0893 + throughput/total_tokens=826,176,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4304/500000] + train/ActionL1Loss=0.0860 + throughput/total_tokens=826,368,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4305/500000] + train/ActionL1Loss=0.0849 + throughput/total_tokens=826,560,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4306/500000] + train/ActionL1Loss=0.0596 + throughput/total_tokens=826,752,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4307/500000] + train/ActionL1Loss=0.0860 + throughput/total_tokens=826,944,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4308/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=827,136,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4309/500000] + train/ActionL1Loss=0.0802 + throughput/total_tokens=827,328,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4310/500000] + train/ActionL1Loss=0.0757 + throughput/total_tokens=827,520,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4311/500000] + train/ActionL1Loss=0.0753 + throughput/total_tokens=827,712,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4312/500000] + train/ActionL1Loss=0.0843 + throughput/total_tokens=827,904,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4313/500000] + train/ActionL1Loss=0.0871 + throughput/total_tokens=828,096,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4314/500000] + train/ActionL1Loss=0.0759 + throughput/total_tokens=828,288,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4315/500000] + train/ActionL1Loss=0.0766 + throughput/total_tokens=828,480,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4316/500000] + train/ActionL1Loss=0.0798 + throughput/total_tokens=828,672,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4317/500000] + train/ActionL1Loss=0.0878 + throughput/total_tokens=828,864,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4318/500000] + train/ActionL1Loss=0.0718 + throughput/total_tokens=829,056,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4319/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=829,248,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4320/500000] + optim/total_grad_norm=6.188 + train/ActionL1Loss=0.0693 + throughput/total_tokens=829,440,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4321/500000] + train/ActionL1Loss=0.0801 + throughput/total_tokens=829,632,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4322/500000] + train/ActionL1Loss=0.0796 + throughput/total_tokens=829,824,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4323/500000] + train/ActionL1Loss=0.0799 + throughput/total_tokens=830,016,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4324/500000] + train/ActionL1Loss=0.1093 + throughput/total_tokens=830,208,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4325/500000] + train/ActionL1Loss=0.0741 + throughput/total_tokens=830,400,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4326/500000] + train/ActionL1Loss=0.0905 + throughput/total_tokens=830,592,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4327/500000] + train/ActionL1Loss=0.0781 + throughput/total_tokens=830,784,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4328/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=830,976,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4329/500000] + train/ActionL1Loss=0.0718 + throughput/total_tokens=831,168,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4330/500000] + train/ActionL1Loss=0.0785 + throughput/total_tokens=831,360,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4331/500000] + train/ActionL1Loss=0.0770 + throughput/total_tokens=831,552,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4332/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=831,744,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4333/500000] + train/ActionL1Loss=0.0793 + throughput/total_tokens=831,936,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4334/500000] + train/ActionL1Loss=0.0630 + throughput/total_tokens=832,128,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4335/500000] + train/ActionL1Loss=0.0674 + throughput/total_tokens=832,320,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4336/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=832,512,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4337/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=832,704,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4338/500000] + train/ActionL1Loss=0.0616 + throughput/total_tokens=832,896,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4339/500000] + train/ActionL1Loss=0.0536 + throughput/total_tokens=833,088,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4340/500000] + optim/total_grad_norm=7.227 + train/ActionL1Loss=0.0831 + throughput/total_tokens=833,280,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4341/500000] + train/ActionL1Loss=0.0965 + throughput/total_tokens=833,472,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4342/500000] + train/ActionL1Loss=0.0867 + throughput/total_tokens=833,664,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4343/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=833,856,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4344/500000] + train/ActionL1Loss=0.0792 + throughput/total_tokens=834,048,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4345/500000] + train/ActionL1Loss=0.0850 + throughput/total_tokens=834,240,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4346/500000] + train/ActionL1Loss=0.0578 + throughput/total_tokens=834,432,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4347/500000] + train/ActionL1Loss=0.0885 + throughput/total_tokens=834,624,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4348/500000] + train/ActionL1Loss=0.0939 + throughput/total_tokens=834,816,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4349/500000] + train/ActionL1Loss=0.0788 + throughput/total_tokens=835,008,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4350/500000] + train/ActionL1Loss=0.0800 + throughput/total_tokens=835,200,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4351/500000] + train/ActionL1Loss=0.0734 + throughput/total_tokens=835,392,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4352/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=835,584,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4353/500000] + train/ActionL1Loss=0.0869 + throughput/total_tokens=835,776,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4354/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=835,968,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4355/500000] + train/ActionL1Loss=0.0976 + throughput/total_tokens=836,160,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4356/500000] + train/ActionL1Loss=0.0926 + throughput/total_tokens=836,352,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4357/500000] + train/ActionL1Loss=0.0780 + throughput/total_tokens=836,544,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4358/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=836,736,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4359/500000] + train/ActionL1Loss=0.0904 + throughput/total_tokens=836,928,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4360/500000] + optim/total_grad_norm=5.045 + train/ActionL1Loss=0.0691 + throughput/total_tokens=837,120,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4361/500000] + train/ActionL1Loss=0.0633 + throughput/total_tokens=837,312,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4362/500000] + train/ActionL1Loss=0.0703 + throughput/total_tokens=837,504,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4363/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=837,696,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4364/500000] + train/ActionL1Loss=0.0643 + throughput/total_tokens=837,888,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4365/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=838,080,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4366/500000] + train/ActionL1Loss=0.0624 + throughput/total_tokens=838,272,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4367/500000] + train/ActionL1Loss=0.0869 + throughput/total_tokens=838,464,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4368/500000] + train/ActionL1Loss=0.0823 + throughput/total_tokens=838,656,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4369/500000] + train/ActionL1Loss=0.0759 + throughput/total_tokens=838,848,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4370/500000] + train/ActionL1Loss=0.0840 + throughput/total_tokens=839,040,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4371/500000] + train/ActionL1Loss=0.0873 + throughput/total_tokens=839,232,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4372/500000] + train/ActionL1Loss=0.0630 + throughput/total_tokens=839,424,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4373/500000] + train/ActionL1Loss=0.0653 + throughput/total_tokens=839,616,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4374/500000] + train/ActionL1Loss=0.0733 + throughput/total_tokens=839,808,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4375/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=840,000,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4376/500000] + train/ActionL1Loss=0.0718 + throughput/total_tokens=840,192,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4377/500000] + train/ActionL1Loss=0.0666 + throughput/total_tokens=840,384,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4378/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=840,576,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4379/500000] + train/ActionL1Loss=0.0583 + throughput/total_tokens=840,768,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4380/500000] + optim/total_grad_norm=6.402 + train/ActionL1Loss=0.0792 + throughput/total_tokens=840,960,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4381/500000] + train/ActionL1Loss=0.0735 + throughput/total_tokens=841,152,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4382/500000] + train/ActionL1Loss=0.0818 + throughput/total_tokens=841,344,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4383/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=841,536,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4384/500000] + train/ActionL1Loss=0.0730 + throughput/total_tokens=841,728,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4385/500000] + train/ActionL1Loss=0.0885 + throughput/total_tokens=841,920,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4386/500000] + train/ActionL1Loss=0.0773 + throughput/total_tokens=842,112,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4387/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=842,304,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4388/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=842,496,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4389/500000] + train/ActionL1Loss=0.0514 + throughput/total_tokens=842,688,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4390/500000] + train/ActionL1Loss=0.0666 + throughput/total_tokens=842,880,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4391/500000] + train/ActionL1Loss=0.0724 + throughput/total_tokens=843,072,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4392/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=843,264,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4393/500000] + train/ActionL1Loss=0.0571 + throughput/total_tokens=843,456,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4394/500000] + train/ActionL1Loss=0.0834 + throughput/total_tokens=843,648,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4395/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=843,840,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4396/500000] + train/ActionL1Loss=0.0759 + throughput/total_tokens=844,032,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4397/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=844,224,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4398/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=844,416,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4399/500000] + train/ActionL1Loss=0.0743 + throughput/total_tokens=844,608,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4400/500000] + optim/total_grad_norm=5.066 + train/ActionL1Loss=0.0784 + throughput/total_tokens=844,800,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4401/500000] + train/ActionL1Loss=0.0769 + throughput/total_tokens=844,992,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4402/500000] + train/ActionL1Loss=0.0650 + throughput/total_tokens=845,184,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4403/500000] + train/ActionL1Loss=0.0744 + throughput/total_tokens=845,376,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4404/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=845,568,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4405/500000] + train/ActionL1Loss=0.0676 + throughput/total_tokens=845,760,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4406/500000] + train/ActionL1Loss=0.0637 + throughput/total_tokens=845,952,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4407/500000] + train/ActionL1Loss=0.0691 + throughput/total_tokens=846,144,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4408/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=846,336,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4409/500000] + train/ActionL1Loss=0.0643 + throughput/total_tokens=846,528,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4410/500000] + train/ActionL1Loss=0.0699 + throughput/total_tokens=846,720,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4411/500000] + train/ActionL1Loss=0.0611 + throughput/total_tokens=846,912,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4412/500000] + train/ActionL1Loss=0.0784 + throughput/total_tokens=847,104,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4413/500000] + train/ActionL1Loss=0.0657 + throughput/total_tokens=847,296,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4414/500000] + train/ActionL1Loss=0.0949 + throughput/total_tokens=847,488,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4415/500000] + train/ActionL1Loss=0.0752 + throughput/total_tokens=847,680,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4416/500000] + train/ActionL1Loss=0.0743 + throughput/total_tokens=847,872,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4417/500000] + train/ActionL1Loss=0.0758 + throughput/total_tokens=848,064,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4418/500000] + train/ActionL1Loss=0.0907 + throughput/total_tokens=848,256,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4419/500000] + train/ActionL1Loss=0.0722 + throughput/total_tokens=848,448,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4420/500000] + optim/total_grad_norm=7.887 + train/ActionL1Loss=0.0698 + throughput/total_tokens=848,640,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4421/500000] + train/ActionL1Loss=0.0816 + throughput/total_tokens=848,832,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4422/500000] + train/ActionL1Loss=0.0673 + throughput/total_tokens=849,024,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4423/500000] + train/ActionL1Loss=0.0766 + throughput/total_tokens=849,216,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4424/500000] + train/ActionL1Loss=0.0916 + throughput/total_tokens=849,408,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4425/500000] + train/ActionL1Loss=0.0864 + throughput/total_tokens=849,600,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4426/500000] + train/ActionL1Loss=0.1027 + throughput/total_tokens=849,792,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4427/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=849,984,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4428/500000] + train/ActionL1Loss=0.0674 + throughput/total_tokens=850,176,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4429/500000] + train/ActionL1Loss=0.0735 + throughput/total_tokens=850,368,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4430/500000] + train/ActionL1Loss=0.0969 + throughput/total_tokens=850,560,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4431/500000] + train/ActionL1Loss=0.0932 + throughput/total_tokens=850,752,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4432/500000] + train/ActionL1Loss=0.0797 + throughput/total_tokens=850,944,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4433/500000] + train/ActionL1Loss=0.0713 + throughput/total_tokens=851,136,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4434/500000] + train/ActionL1Loss=0.0780 + throughput/total_tokens=851,328,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4435/500000] + train/ActionL1Loss=0.0732 + throughput/total_tokens=851,520,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4436/500000] + train/ActionL1Loss=0.0790 + throughput/total_tokens=851,712,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4437/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=851,904,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4438/500000] + train/ActionL1Loss=0.0710 + throughput/total_tokens=852,096,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4439/500000] + train/ActionL1Loss=0.0640 + throughput/total_tokens=852,288,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4440/500000] + optim/total_grad_norm=6.849 + train/ActionL1Loss=0.0776 + throughput/total_tokens=852,480,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4441/500000] + train/ActionL1Loss=0.0724 + throughput/total_tokens=852,672,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4442/500000] + train/ActionL1Loss=0.0787 + throughput/total_tokens=852,864,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4443/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=853,056,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4444/500000] + train/ActionL1Loss=0.0761 + throughput/total_tokens=853,248,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4445/500000] + train/ActionL1Loss=0.0920 + throughput/total_tokens=853,440,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4446/500000] + train/ActionL1Loss=0.0837 + throughput/total_tokens=853,632,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4447/500000] + train/ActionL1Loss=0.0865 + throughput/total_tokens=853,824,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4448/500000] + train/ActionL1Loss=0.0738 + throughput/total_tokens=854,016,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4449/500000] + train/ActionL1Loss=0.0969 + throughput/total_tokens=854,208,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4450/500000] + train/ActionL1Loss=0.0782 + throughput/total_tokens=854,400,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4451/500000] + train/ActionL1Loss=0.0779 + throughput/total_tokens=854,592,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4452/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=854,784,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4453/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=854,976,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4454/500000] + train/ActionL1Loss=0.0765 + throughput/total_tokens=855,168,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4455/500000] + train/ActionL1Loss=0.0784 + throughput/total_tokens=855,360,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4456/500000] + train/ActionL1Loss=0.0653 + throughput/total_tokens=855,552,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4457/500000] + train/ActionL1Loss=0.0662 + throughput/total_tokens=855,744,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4458/500000] + train/ActionL1Loss=0.0689 + throughput/total_tokens=855,936,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4459/500000] + train/ActionL1Loss=0.0707 + throughput/total_tokens=856,128,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4460/500000] + optim/total_grad_norm=8.478 + train/ActionL1Loss=0.0892 + throughput/total_tokens=856,320,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=4461/500000] + train/ActionL1Loss=0.0760 + throughput/total_tokens=856,512,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4462/500000] + train/ActionL1Loss=0.0758 + throughput/total_tokens=856,704,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4463/500000] + train/ActionL1Loss=0.0713 + throughput/total_tokens=856,896,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4464/500000] + train/ActionL1Loss=0.0856 + throughput/total_tokens=857,088,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4465/500000] + train/ActionL1Loss=0.0758 + throughput/total_tokens=857,280,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4466/500000] + train/ActionL1Loss=0.0749 + throughput/total_tokens=857,472,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4467/500000] + train/ActionL1Loss=0.0773 + throughput/total_tokens=857,664,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4468/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=857,856,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4469/500000] + train/ActionL1Loss=0.0722 + throughput/total_tokens=858,048,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4470/500000] + train/ActionL1Loss=0.0582 + throughput/total_tokens=858,240,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4471/500000] + train/ActionL1Loss=0.0731 + throughput/total_tokens=858,432,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4472/500000] + train/ActionL1Loss=0.0893 + throughput/total_tokens=858,624,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4473/500000] + train/ActionL1Loss=0.0927 + throughput/total_tokens=858,816,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4474/500000] + train/ActionL1Loss=0.0744 + throughput/total_tokens=859,008,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4475/500000] + train/ActionL1Loss=0.0655 + throughput/total_tokens=859,200,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4476/500000] + train/ActionL1Loss=0.0568 + throughput/total_tokens=859,392,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4477/500000] + train/ActionL1Loss=0.0830 + throughput/total_tokens=859,584,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4478/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=859,776,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4479/500000] + train/ActionL1Loss=0.0847 + throughput/total_tokens=859,968,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4480/500000] + optim/total_grad_norm=7.960 + train/ActionL1Loss=0.0752 + throughput/total_tokens=860,160,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4481/500000] + train/ActionL1Loss=0.0657 + throughput/total_tokens=860,352,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4482/500000] + train/ActionL1Loss=0.0900 + throughput/total_tokens=860,544,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4483/500000] + train/ActionL1Loss=0.0870 + throughput/total_tokens=860,736,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4484/500000] + train/ActionL1Loss=0.0537 + throughput/total_tokens=860,928,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4485/500000] + train/ActionL1Loss=0.0792 + throughput/total_tokens=861,120,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4486/500000] + train/ActionL1Loss=0.0780 + throughput/total_tokens=861,312,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4487/500000] + train/ActionL1Loss=0.0847 + throughput/total_tokens=861,504,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4488/500000] + train/ActionL1Loss=0.0808 + throughput/total_tokens=861,696,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4489/500000] + train/ActionL1Loss=0.0867 + throughput/total_tokens=861,888,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4490/500000] + train/ActionL1Loss=0.0814 + throughput/total_tokens=862,080,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4491/500000] + train/ActionL1Loss=0.0762 + throughput/total_tokens=862,272,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4492/500000] + train/ActionL1Loss=0.1046 + throughput/total_tokens=862,464,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4493/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=862,656,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4494/500000] + train/ActionL1Loss=0.0834 + throughput/total_tokens=862,848,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4495/500000] + train/ActionL1Loss=0.0836 + throughput/total_tokens=863,040,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4496/500000] + train/ActionL1Loss=0.0578 + throughput/total_tokens=863,232,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4497/500000] + train/ActionL1Loss=0.0735 + throughput/total_tokens=863,424,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4498/500000] + train/ActionL1Loss=0.0716 + throughput/total_tokens=863,616,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4499/500000] + train/ActionL1Loss=0.0699 + throughput/total_tokens=863,808,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4500/500000] + optim/total_grad_norm=6.681 + train/ActionL1Loss=0.0638 + throughput/total_tokens=864,000,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +10/01 [15:43:27] INFO | >> Saving config... checkpoint.py:608 +10/01 [15:44:12] INFO | >> Saving model state... checkpoint.py:796 +10/01 [15:45:23] INFO | >> Saving optim state... checkpoint.py:811 +10/01 [15:46:49] INFO | >> Saving trainer state... checkpoint.py:670 + INFO | >> Saving config... checkpoint.py:608 +[step=4501/500000] + train/ActionL1Loss=0.0863 + throughput/total_tokens=864,192,000 + throughput/device/tokens_per_second=1,211 + throughput/device/batches_per_second=0.0505 +[step=4502/500000] + train/ActionL1Loss=0.0778 + throughput/total_tokens=864,384,000 + throughput/device/tokens_per_second=1,172 + throughput/device/batches_per_second=0.0489 +[step=4503/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=864,576,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4504/500000] + train/ActionL1Loss=0.0809 + throughput/total_tokens=864,768,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4505/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=864,960,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4506/500000] + train/ActionL1Loss=0.0801 + throughput/total_tokens=865,152,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4507/500000] + train/ActionL1Loss=0.0510 + throughput/total_tokens=865,344,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4508/500000] + train/ActionL1Loss=0.0792 + throughput/total_tokens=865,536,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4509/500000] + train/ActionL1Loss=0.0741 + throughput/total_tokens=865,728,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4510/500000] + train/ActionL1Loss=0.0731 + throughput/total_tokens=865,920,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4511/500000] + train/ActionL1Loss=0.0762 + throughput/total_tokens=866,112,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4512/500000] + train/ActionL1Loss=0.0646 + throughput/total_tokens=866,304,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4513/500000] + train/ActionL1Loss=0.0744 + throughput/total_tokens=866,496,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4514/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=866,688,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4515/500000] + train/ActionL1Loss=0.0780 + throughput/total_tokens=866,880,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4516/500000] + train/ActionL1Loss=0.0721 + throughput/total_tokens=867,072,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4517/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=867,264,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4518/500000] + train/ActionL1Loss=0.0787 + throughput/total_tokens=867,456,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4519/500000] + train/ActionL1Loss=0.0707 + throughput/total_tokens=867,648,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4520/500000] + optim/total_grad_norm=5.821 + train/ActionL1Loss=0.0544 + throughput/total_tokens=867,840,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4521/500000] + train/ActionL1Loss=0.0666 + throughput/total_tokens=868,032,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4522/500000] + train/ActionL1Loss=0.0716 + throughput/total_tokens=868,224,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4523/500000] + train/ActionL1Loss=0.0750 + throughput/total_tokens=868,416,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4524/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=868,608,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4525/500000] + train/ActionL1Loss=0.0789 + throughput/total_tokens=868,800,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4526/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=868,992,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4527/500000] + train/ActionL1Loss=0.0909 + throughput/total_tokens=869,184,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4528/500000] + train/ActionL1Loss=0.0568 + throughput/total_tokens=869,376,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4529/500000] + train/ActionL1Loss=0.0672 + throughput/total_tokens=869,568,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4530/500000] + train/ActionL1Loss=0.0757 + throughput/total_tokens=869,760,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4531/500000] + train/ActionL1Loss=0.0886 + throughput/total_tokens=869,952,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4532/500000] + train/ActionL1Loss=0.0580 + throughput/total_tokens=870,144,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4533/500000] + train/ActionL1Loss=0.0710 + throughput/total_tokens=870,336,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4534/500000] + train/ActionL1Loss=0.0746 + throughput/total_tokens=870,528,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4535/500000] + train/ActionL1Loss=0.0758 + throughput/total_tokens=870,720,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4536/500000] + train/ActionL1Loss=0.0657 + throughput/total_tokens=870,912,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4537/500000] + train/ActionL1Loss=0.0640 + throughput/total_tokens=871,104,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4538/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=871,296,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4539/500000] + train/ActionL1Loss=0.0747 + throughput/total_tokens=871,488,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4540/500000] + optim/total_grad_norm=5.997 + train/ActionL1Loss=0.0674 + throughput/total_tokens=871,680,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4541/500000] + train/ActionL1Loss=0.0606 + throughput/total_tokens=871,872,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4542/500000] + train/ActionL1Loss=0.0653 + throughput/total_tokens=872,064,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4543/500000] + train/ActionL1Loss=0.0996 + throughput/total_tokens=872,256,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4544/500000] + train/ActionL1Loss=0.0801 + throughput/total_tokens=872,448,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4545/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=872,640,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4546/500000] + train/ActionL1Loss=0.0704 + throughput/total_tokens=872,832,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4547/500000] + train/ActionL1Loss=0.0746 + throughput/total_tokens=873,024,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4548/500000] + train/ActionL1Loss=0.0701 + throughput/total_tokens=873,216,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4549/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=873,408,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4550/500000] + train/ActionL1Loss=0.0736 + throughput/total_tokens=873,600,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4551/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=873,792,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4552/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=873,984,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4553/500000] + train/ActionL1Loss=0.0900 + throughput/total_tokens=874,176,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4554/500000] + train/ActionL1Loss=0.0993 + throughput/total_tokens=874,368,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4555/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=874,560,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4556/500000] + train/ActionL1Loss=0.0809 + throughput/total_tokens=874,752,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4557/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=874,944,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4558/500000] + train/ActionL1Loss=0.0791 + throughput/total_tokens=875,136,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4559/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=875,328,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4560/500000] + optim/total_grad_norm=7.121 + train/ActionL1Loss=0.0797 + throughput/total_tokens=875,520,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4561/500000] + train/ActionL1Loss=0.0717 + throughput/total_tokens=875,712,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4562/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=875,904,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4563/500000] + train/ActionL1Loss=0.0636 + throughput/total_tokens=876,096,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4564/500000] + train/ActionL1Loss=0.0683 + throughput/total_tokens=876,288,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4565/500000] + train/ActionL1Loss=0.0919 + throughput/total_tokens=876,480,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4566/500000] + train/ActionL1Loss=0.0759 + throughput/total_tokens=876,672,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4567/500000] + train/ActionL1Loss=0.0616 + throughput/total_tokens=876,864,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4568/500000] + train/ActionL1Loss=0.0743 + throughput/total_tokens=877,056,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4569/500000] + train/ActionL1Loss=0.0770 + throughput/total_tokens=877,248,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4570/500000] + train/ActionL1Loss=0.0789 + throughput/total_tokens=877,440,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4571/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=877,632,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4572/500000] + train/ActionL1Loss=0.0578 + throughput/total_tokens=877,824,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4573/500000] + train/ActionL1Loss=0.0552 + throughput/total_tokens=878,016,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4574/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=878,208,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4575/500000] + train/ActionL1Loss=0.0791 + throughput/total_tokens=878,400,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4576/500000] + train/ActionL1Loss=0.0879 + throughput/total_tokens=878,592,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4577/500000] + train/ActionL1Loss=0.0767 + throughput/total_tokens=878,784,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4578/500000] + train/ActionL1Loss=0.0541 + throughput/total_tokens=878,976,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4579/500000] + train/ActionL1Loss=0.0790 + throughput/total_tokens=879,168,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4580/500000] + optim/total_grad_norm=4.994 + train/ActionL1Loss=0.0643 + throughput/total_tokens=879,360,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4581/500000] + train/ActionL1Loss=0.0620 + throughput/total_tokens=879,552,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4582/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=879,744,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4583/500000] + train/ActionL1Loss=0.0743 + throughput/total_tokens=879,936,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4584/500000] + train/ActionL1Loss=0.0933 + throughput/total_tokens=880,128,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4585/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=880,320,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4586/500000] + train/ActionL1Loss=0.0873 + throughput/total_tokens=880,512,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4587/500000] + train/ActionL1Loss=0.0793 + throughput/total_tokens=880,704,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4588/500000] + train/ActionL1Loss=0.0745 + throughput/total_tokens=880,896,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4589/500000] + train/ActionL1Loss=0.0637 + throughput/total_tokens=881,088,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4590/500000] + train/ActionL1Loss=0.0789 + throughput/total_tokens=881,280,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4591/500000] + train/ActionL1Loss=0.0718 + throughput/total_tokens=881,472,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4592/500000] + train/ActionL1Loss=0.0541 + throughput/total_tokens=881,664,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4593/500000] + train/ActionL1Loss=0.0988 + throughput/total_tokens=881,856,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4594/500000] + train/ActionL1Loss=0.0675 + throughput/total_tokens=882,048,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4595/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=882,240,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4596/500000] + train/ActionL1Loss=0.0678 + throughput/total_tokens=882,432,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4597/500000] + train/ActionL1Loss=0.0755 + throughput/total_tokens=882,624,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4598/500000] + train/ActionL1Loss=0.0867 + throughput/total_tokens=882,816,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4599/500000] + train/ActionL1Loss=0.0633 + throughput/total_tokens=883,008,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4600/500000] + optim/total_grad_norm=7.739 + train/ActionL1Loss=0.0967 + throughput/total_tokens=883,200,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4601/500000] + train/ActionL1Loss=0.0888 + throughput/total_tokens=883,392,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4602/500000] + train/ActionL1Loss=0.0848 + throughput/total_tokens=883,584,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4603/500000] + train/ActionL1Loss=0.0680 + throughput/total_tokens=883,776,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4604/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=883,968,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4605/500000] + train/ActionL1Loss=0.1050 + throughput/total_tokens=884,160,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4606/500000] + train/ActionL1Loss=0.0738 + throughput/total_tokens=884,352,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4607/500000] + train/ActionL1Loss=0.1061 + throughput/total_tokens=884,544,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4608/500000] + train/ActionL1Loss=0.0772 + throughput/total_tokens=884,736,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4609/500000] + train/ActionL1Loss=0.0759 + throughput/total_tokens=884,928,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4610/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=885,120,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=4611/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=885,312,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4612/500000] + train/ActionL1Loss=0.0776 + throughput/total_tokens=885,504,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4613/500000] + train/ActionL1Loss=0.0873 + throughput/total_tokens=885,696,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4614/500000] + train/ActionL1Loss=0.0885 + throughput/total_tokens=885,888,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4615/500000] + train/ActionL1Loss=0.1097 + throughput/total_tokens=886,080,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4616/500000] + train/ActionL1Loss=0.0707 + throughput/total_tokens=886,272,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4617/500000] + train/ActionL1Loss=0.0733 + throughput/total_tokens=886,464,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4618/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=886,656,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4619/500000] + train/ActionL1Loss=0.0748 + throughput/total_tokens=886,848,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4620/500000] + optim/total_grad_norm=6.824 + train/ActionL1Loss=0.0932 + throughput/total_tokens=887,040,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4621/500000] + train/ActionL1Loss=0.0766 + throughput/total_tokens=887,232,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4622/500000] + train/ActionL1Loss=0.0708 + throughput/total_tokens=887,424,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4623/500000] + train/ActionL1Loss=0.0816 + throughput/total_tokens=887,616,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4624/500000] + train/ActionL1Loss=0.0942 + throughput/total_tokens=887,808,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4625/500000] + train/ActionL1Loss=0.0669 + throughput/total_tokens=888,000,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4626/500000] + train/ActionL1Loss=0.0769 + throughput/total_tokens=888,192,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4627/500000] + train/ActionL1Loss=0.0729 + throughput/total_tokens=888,384,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4628/500000] + train/ActionL1Loss=0.0736 + throughput/total_tokens=888,576,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4629/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=888,768,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4630/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=888,960,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4631/500000] + train/ActionL1Loss=0.0936 + throughput/total_tokens=889,152,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4632/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=889,344,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4633/500000] + train/ActionL1Loss=0.0775 + throughput/total_tokens=889,536,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4634/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=889,728,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4635/500000] + train/ActionL1Loss=0.0819 + throughput/total_tokens=889,920,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4636/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=890,112,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4637/500000] + train/ActionL1Loss=0.0830 + throughput/total_tokens=890,304,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4638/500000] + train/ActionL1Loss=0.0780 + throughput/total_tokens=890,496,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4639/500000] + train/ActionL1Loss=0.0702 + throughput/total_tokens=890,688,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4640/500000] + optim/total_grad_norm=7.559 + train/ActionL1Loss=0.0706 + throughput/total_tokens=890,880,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4641/500000] + train/ActionL1Loss=0.0909 + throughput/total_tokens=891,072,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4642/500000] + train/ActionL1Loss=0.0760 + throughput/total_tokens=891,264,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4643/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=891,456,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4644/500000] + train/ActionL1Loss=0.0811 + throughput/total_tokens=891,648,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4645/500000] + train/ActionL1Loss=0.0676 + throughput/total_tokens=891,840,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4646/500000] + train/ActionL1Loss=0.0806 + throughput/total_tokens=892,032,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4647/500000] + train/ActionL1Loss=0.0739 + throughput/total_tokens=892,224,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4648/500000] + train/ActionL1Loss=0.0564 + throughput/total_tokens=892,416,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4649/500000] + train/ActionL1Loss=0.0675 + throughput/total_tokens=892,608,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4650/500000] + train/ActionL1Loss=0.0698 + throughput/total_tokens=892,800,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4651/500000] + train/ActionL1Loss=0.0899 + throughput/total_tokens=892,992,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4652/500000] + train/ActionL1Loss=0.0768 + throughput/total_tokens=893,184,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4653/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=893,376,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4654/500000] + train/ActionL1Loss=0.0755 + throughput/total_tokens=893,568,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4655/500000] + train/ActionL1Loss=0.0676 + throughput/total_tokens=893,760,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4656/500000] + train/ActionL1Loss=0.0862 + throughput/total_tokens=893,952,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4657/500000] + train/ActionL1Loss=0.0893 + throughput/total_tokens=894,144,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4658/500000] + train/ActionL1Loss=0.0675 + throughput/total_tokens=894,336,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4659/500000] + train/ActionL1Loss=0.0939 + throughput/total_tokens=894,528,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4660/500000] + optim/total_grad_norm=6.307 + train/ActionL1Loss=0.0816 + throughput/total_tokens=894,720,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=4661/500000] + train/ActionL1Loss=0.0864 + throughput/total_tokens=894,912,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4662/500000] + train/ActionL1Loss=0.0723 + throughput/total_tokens=895,104,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4663/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=895,296,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4664/500000] + train/ActionL1Loss=0.0851 + throughput/total_tokens=895,488,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4665/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=895,680,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4666/500000] + train/ActionL1Loss=0.0714 + throughput/total_tokens=895,872,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4667/500000] + train/ActionL1Loss=0.0759 + throughput/total_tokens=896,064,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4668/500000] + train/ActionL1Loss=0.0690 + throughput/total_tokens=896,256,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4669/500000] + train/ActionL1Loss=0.0723 + throughput/total_tokens=896,448,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4670/500000] + train/ActionL1Loss=0.0532 + throughput/total_tokens=896,640,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4671/500000] + train/ActionL1Loss=0.0832 + throughput/total_tokens=896,832,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4672/500000] + train/ActionL1Loss=0.0676 + throughput/total_tokens=897,024,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4673/500000] + train/ActionL1Loss=0.0863 + throughput/total_tokens=897,216,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4674/500000] + train/ActionL1Loss=0.0800 + throughput/total_tokens=897,408,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4675/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=897,600,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4676/500000] + train/ActionL1Loss=0.0728 + throughput/total_tokens=897,792,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4677/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=897,984,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4678/500000] + train/ActionL1Loss=0.0558 + throughput/total_tokens=898,176,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4679/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=898,368,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4680/500000] + optim/total_grad_norm=5.929 + train/ActionL1Loss=0.0691 + throughput/total_tokens=898,560,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4681/500000] + train/ActionL1Loss=0.0788 + throughput/total_tokens=898,752,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4682/500000] + train/ActionL1Loss=0.0781 + throughput/total_tokens=898,944,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4683/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=899,136,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4684/500000] + train/ActionL1Loss=0.0878 + throughput/total_tokens=899,328,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4685/500000] + train/ActionL1Loss=0.0689 + throughput/total_tokens=899,520,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4686/500000] + train/ActionL1Loss=0.0840 + throughput/total_tokens=899,712,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4687/500000] + train/ActionL1Loss=0.0872 + throughput/total_tokens=899,904,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4688/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=900,096,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4689/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=900,288,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4690/500000] + train/ActionL1Loss=0.0741 + throughput/total_tokens=900,480,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4691/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=900,672,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4692/500000] + train/ActionL1Loss=0.0962 + throughput/total_tokens=900,864,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4693/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=901,056,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4694/500000] + train/ActionL1Loss=0.0713 + throughput/total_tokens=901,248,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4695/500000] + train/ActionL1Loss=0.0605 + throughput/total_tokens=901,440,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4696/500000] + train/ActionL1Loss=0.0646 + throughput/total_tokens=901,632,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4697/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=901,824,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4698/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=902,016,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4699/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=902,208,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4700/500000] + optim/total_grad_norm=6.346 + train/ActionL1Loss=0.0790 + throughput/total_tokens=902,400,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4701/500000] + train/ActionL1Loss=0.0713 + throughput/total_tokens=902,592,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4702/500000] + train/ActionL1Loss=0.0740 + throughput/total_tokens=902,784,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4703/500000] + train/ActionL1Loss=0.0882 + throughput/total_tokens=902,976,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4704/500000] + train/ActionL1Loss=0.0611 + throughput/total_tokens=903,168,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4705/500000] + train/ActionL1Loss=0.0660 + throughput/total_tokens=903,360,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4706/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=903,552,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4707/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=903,744,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4708/500000] + train/ActionL1Loss=0.0680 + throughput/total_tokens=903,936,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4709/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=904,128,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4710/500000] + train/ActionL1Loss=0.0554 + throughput/total_tokens=904,320,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4711/500000] + train/ActionL1Loss=0.0743 + throughput/total_tokens=904,512,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4712/500000] + train/ActionL1Loss=0.0765 + throughput/total_tokens=904,704,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4713/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=904,896,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4714/500000] + train/ActionL1Loss=0.0701 + throughput/total_tokens=905,088,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4715/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=905,280,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4716/500000] + train/ActionL1Loss=0.0748 + throughput/total_tokens=905,472,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4717/500000] + train/ActionL1Loss=0.0726 + throughput/total_tokens=905,664,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4718/500000] + train/ActionL1Loss=0.0728 + throughput/total_tokens=905,856,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4719/500000] + train/ActionL1Loss=0.0787 + throughput/total_tokens=906,048,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4720/500000] + optim/total_grad_norm=5.812 + train/ActionL1Loss=0.0761 + throughput/total_tokens=906,240,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=4721/500000] + train/ActionL1Loss=0.0701 + throughput/total_tokens=906,432,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=4722/500000] + train/ActionL1Loss=0.0568 + throughput/total_tokens=906,624,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=4723/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=906,816,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=4724/500000] + train/ActionL1Loss=0.0578 + throughput/total_tokens=907,008,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=4725/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=907,200,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=4726/500000] + train/ActionL1Loss=0.0702 + throughput/total_tokens=907,392,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4727/500000] + train/ActionL1Loss=0.0717 + throughput/total_tokens=907,584,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4728/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=907,776,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4729/500000] + train/ActionL1Loss=0.0835 + throughput/total_tokens=907,968,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4730/500000] + train/ActionL1Loss=0.0961 + throughput/total_tokens=908,160,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4731/500000] + train/ActionL1Loss=0.0752 + throughput/total_tokens=908,352,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4732/500000] + train/ActionL1Loss=0.0648 + throughput/total_tokens=908,544,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4733/500000] + train/ActionL1Loss=0.0778 + throughput/total_tokens=908,736,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4734/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=908,928,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4735/500000] + train/ActionL1Loss=0.0601 + throughput/total_tokens=909,120,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4736/500000] + train/ActionL1Loss=0.0968 + throughput/total_tokens=909,312,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4737/500000] + train/ActionL1Loss=0.0640 + throughput/total_tokens=909,504,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4738/500000] + train/ActionL1Loss=0.0741 + throughput/total_tokens=909,696,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4739/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=909,888,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4740/500000] + optim/total_grad_norm=7.200 + train/ActionL1Loss=0.0746 + throughput/total_tokens=910,080,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4741/500000] + train/ActionL1Loss=0.0777 + throughput/total_tokens=910,272,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4742/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=910,464,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4743/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=910,656,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4744/500000] + train/ActionL1Loss=0.0697 + throughput/total_tokens=910,848,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4745/500000] + train/ActionL1Loss=0.0673 + throughput/total_tokens=911,040,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4746/500000] + train/ActionL1Loss=0.0730 + throughput/total_tokens=911,232,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4747/500000] + train/ActionL1Loss=0.0739 + throughput/total_tokens=911,424,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4748/500000] + train/ActionL1Loss=0.0687 + throughput/total_tokens=911,616,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4749/500000] + train/ActionL1Loss=0.0681 + throughput/total_tokens=911,808,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4750/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=912,000,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4751/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=912,192,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=4752/500000] + train/ActionL1Loss=0.0768 + throughput/total_tokens=912,384,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=4753/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=912,576,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=4754/500000] + train/ActionL1Loss=0.0709 + throughput/total_tokens=912,768,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0481 +[step=4755/500000] + train/ActionL1Loss=0.0795 + throughput/total_tokens=912,960,000 + throughput/device/tokens_per_second=1,153 + throughput/device/batches_per_second=0.0480 +[step=4756/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=913,152,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 +[step=4757/500000] + train/ActionL1Loss=0.0535 + throughput/total_tokens=913,344,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 +[step=4758/500000] + train/ActionL1Loss=0.0720 + throughput/total_tokens=913,536,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 +[step=4759/500000] + train/ActionL1Loss=0.0775 + throughput/total_tokens=913,728,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=4760/500000] + optim/total_grad_norm=6.466 + train/ActionL1Loss=0.0758 + throughput/total_tokens=913,920,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 + System/Peak GPU Memory (MB)=46,917 +[step=4761/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=914,112,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=4762/500000] + train/ActionL1Loss=0.0775 + throughput/total_tokens=914,304,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=4763/500000] + train/ActionL1Loss=0.0714 + throughput/total_tokens=914,496,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=4764/500000] + train/ActionL1Loss=0.0785 + throughput/total_tokens=914,688,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=4765/500000] + train/ActionL1Loss=0.0755 + throughput/total_tokens=914,880,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=4766/500000] + train/ActionL1Loss=0.0865 + throughput/total_tokens=915,072,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=4767/500000] + train/ActionL1Loss=0.0716 + throughput/total_tokens=915,264,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=4768/500000] + train/ActionL1Loss=0.0766 + throughput/total_tokens=915,456,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=4769/500000] + train/ActionL1Loss=0.0795 + throughput/total_tokens=915,648,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=4770/500000] + train/ActionL1Loss=0.0826 + throughput/total_tokens=915,840,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4771/500000] + train/ActionL1Loss=0.0698 + throughput/total_tokens=916,032,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4772/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=916,224,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4773/500000] + train/ActionL1Loss=0.0796 + throughput/total_tokens=916,416,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4774/500000] + train/ActionL1Loss=0.0707 + throughput/total_tokens=916,608,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4775/500000] + train/ActionL1Loss=0.0691 + throughput/total_tokens=916,800,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4776/500000] + train/ActionL1Loss=0.0546 + throughput/total_tokens=916,992,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4777/500000] + train/ActionL1Loss=0.0842 + throughput/total_tokens=917,184,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4778/500000] + train/ActionL1Loss=0.0768 + throughput/total_tokens=917,376,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4779/500000] + train/ActionL1Loss=0.0716 + throughput/total_tokens=917,568,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4780/500000] + optim/total_grad_norm=4.762 + train/ActionL1Loss=0.0710 + throughput/total_tokens=917,760,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4781/500000] + train/ActionL1Loss=0.0739 + throughput/total_tokens=917,952,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4782/500000] + train/ActionL1Loss=0.0944 + throughput/total_tokens=918,144,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4783/500000] + train/ActionL1Loss=0.0898 + throughput/total_tokens=918,336,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4784/500000] + train/ActionL1Loss=0.0733 + throughput/total_tokens=918,528,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4785/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=918,720,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4786/500000] + train/ActionL1Loss=0.0746 + throughput/total_tokens=918,912,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4787/500000] + train/ActionL1Loss=0.1073 + throughput/total_tokens=919,104,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4788/500000] + train/ActionL1Loss=0.0834 + throughput/total_tokens=919,296,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4789/500000] + train/ActionL1Loss=0.0898 + throughput/total_tokens=919,488,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4790/500000] + train/ActionL1Loss=0.0774 + throughput/total_tokens=919,680,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4791/500000] + train/ActionL1Loss=0.0779 + throughput/total_tokens=919,872,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4792/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=920,064,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4793/500000] + train/ActionL1Loss=0.1059 + throughput/total_tokens=920,256,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4794/500000] + train/ActionL1Loss=0.0757 + throughput/total_tokens=920,448,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4795/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=920,640,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4796/500000] + train/ActionL1Loss=0.0818 + throughput/total_tokens=920,832,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4797/500000] + train/ActionL1Loss=0.0861 + throughput/total_tokens=921,024,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4798/500000] + train/ActionL1Loss=0.0795 + throughput/total_tokens=921,216,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4799/500000] + train/ActionL1Loss=0.0620 + throughput/total_tokens=921,408,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4800/500000] + optim/total_grad_norm=6.525 + train/ActionL1Loss=0.0651 + throughput/total_tokens=921,600,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4801/500000] + train/ActionL1Loss=0.0794 + throughput/total_tokens=921,792,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4802/500000] + train/ActionL1Loss=0.0894 + throughput/total_tokens=921,984,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4803/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=922,176,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4804/500000] + train/ActionL1Loss=0.0766 + throughput/total_tokens=922,368,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4805/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=922,560,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4806/500000] + train/ActionL1Loss=0.0789 + throughput/total_tokens=922,752,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4807/500000] + train/ActionL1Loss=0.0744 + throughput/total_tokens=922,944,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4808/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=923,136,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4809/500000] + train/ActionL1Loss=0.0597 + throughput/total_tokens=923,328,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4810/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=923,520,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=4811/500000] + train/ActionL1Loss=0.0640 + throughput/total_tokens=923,712,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4812/500000] + train/ActionL1Loss=0.0906 + throughput/total_tokens=923,904,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4813/500000] + train/ActionL1Loss=0.0791 + throughput/total_tokens=924,096,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4814/500000] + train/ActionL1Loss=0.0749 + throughput/total_tokens=924,288,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4815/500000] + train/ActionL1Loss=0.0876 + throughput/total_tokens=924,480,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4816/500000] + train/ActionL1Loss=0.0603 + throughput/total_tokens=924,672,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4817/500000] + train/ActionL1Loss=0.0765 + throughput/total_tokens=924,864,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4818/500000] + train/ActionL1Loss=0.0800 + throughput/total_tokens=925,056,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4819/500000] + train/ActionL1Loss=0.0586 + throughput/total_tokens=925,248,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4820/500000] + optim/total_grad_norm=6.409 + train/ActionL1Loss=0.0859 + throughput/total_tokens=925,440,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4821/500000] + train/ActionL1Loss=0.0601 + throughput/total_tokens=925,632,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4822/500000] + train/ActionL1Loss=0.0970 + throughput/total_tokens=925,824,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4823/500000] + train/ActionL1Loss=0.0787 + throughput/total_tokens=926,016,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4824/500000] + train/ActionL1Loss=0.0818 + throughput/total_tokens=926,208,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4825/500000] + train/ActionL1Loss=0.0766 + throughput/total_tokens=926,400,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4826/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=926,592,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4827/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=926,784,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4828/500000] + train/ActionL1Loss=0.0838 + throughput/total_tokens=926,976,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4829/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=927,168,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4830/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=927,360,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4831/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=927,552,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4832/500000] + train/ActionL1Loss=0.0684 + throughput/total_tokens=927,744,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4833/500000] + train/ActionL1Loss=0.0689 + throughput/total_tokens=927,936,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4834/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=928,128,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4835/500000] + train/ActionL1Loss=0.0660 + throughput/total_tokens=928,320,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4836/500000] + train/ActionL1Loss=0.0633 + throughput/total_tokens=928,512,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4837/500000] + train/ActionL1Loss=0.0683 + throughput/total_tokens=928,704,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4838/500000] + train/ActionL1Loss=0.0666 + throughput/total_tokens=928,896,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4839/500000] + train/ActionL1Loss=0.0599 + throughput/total_tokens=929,088,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4840/500000] + optim/total_grad_norm=7.313 + train/ActionL1Loss=0.0828 + throughput/total_tokens=929,280,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4841/500000] + train/ActionL1Loss=0.0745 + throughput/total_tokens=929,472,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4842/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=929,664,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4843/500000] + train/ActionL1Loss=0.0722 + throughput/total_tokens=929,856,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4844/500000] + train/ActionL1Loss=0.0678 + throughput/total_tokens=930,048,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4845/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=930,240,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4846/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=930,432,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4847/500000] + train/ActionL1Loss=0.0832 + throughput/total_tokens=930,624,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4848/500000] + train/ActionL1Loss=0.0707 + throughput/total_tokens=930,816,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4849/500000] + train/ActionL1Loss=0.0808 + throughput/total_tokens=931,008,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4850/500000] + train/ActionL1Loss=0.0748 + throughput/total_tokens=931,200,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4851/500000] + train/ActionL1Loss=0.0495 + throughput/total_tokens=931,392,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4852/500000] + train/ActionL1Loss=0.0809 + throughput/total_tokens=931,584,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4853/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=931,776,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4854/500000] + train/ActionL1Loss=0.0710 + throughput/total_tokens=931,968,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4855/500000] + train/ActionL1Loss=0.0675 + throughput/total_tokens=932,160,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4856/500000] + train/ActionL1Loss=0.0548 + throughput/total_tokens=932,352,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=4857/500000] + train/ActionL1Loss=0.0690 + throughput/total_tokens=932,544,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=4858/500000] + train/ActionL1Loss=0.0842 + throughput/total_tokens=932,736,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=4859/500000] + train/ActionL1Loss=0.0781 + throughput/total_tokens=932,928,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=4860/500000] + optim/total_grad_norm=7.210 + train/ActionL1Loss=0.0741 + throughput/total_tokens=933,120,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=4861/500000] + train/ActionL1Loss=0.0565 + throughput/total_tokens=933,312,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=4862/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=933,504,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=4863/500000] + train/ActionL1Loss=0.0614 + throughput/total_tokens=933,696,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=4864/500000] + train/ActionL1Loss=0.0767 + throughput/total_tokens=933,888,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4865/500000] + train/ActionL1Loss=0.0559 + throughput/total_tokens=934,080,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4866/500000] + train/ActionL1Loss=0.0797 + throughput/total_tokens=934,272,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4867/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=934,464,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4868/500000] + train/ActionL1Loss=0.0677 + throughput/total_tokens=934,656,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4869/500000] + train/ActionL1Loss=0.0886 + throughput/total_tokens=934,848,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4870/500000] + train/ActionL1Loss=0.0809 + throughput/total_tokens=935,040,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4871/500000] + train/ActionL1Loss=0.0629 + throughput/total_tokens=935,232,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4872/500000] + train/ActionL1Loss=0.0646 + throughput/total_tokens=935,424,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4873/500000] + train/ActionL1Loss=0.0913 + throughput/total_tokens=935,616,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4874/500000] + train/ActionL1Loss=0.0854 + throughput/total_tokens=935,808,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4875/500000] + train/ActionL1Loss=0.0853 + throughput/total_tokens=936,000,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4876/500000] + train/ActionL1Loss=0.0874 + throughput/total_tokens=936,192,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4877/500000] + train/ActionL1Loss=0.0778 + throughput/total_tokens=936,384,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4878/500000] + train/ActionL1Loss=0.0854 + throughput/total_tokens=936,576,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4879/500000] + train/ActionL1Loss=0.0886 + throughput/total_tokens=936,768,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4880/500000] + optim/total_grad_norm=5.196 + train/ActionL1Loss=0.0642 + throughput/total_tokens=936,960,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4881/500000] + train/ActionL1Loss=0.0620 + throughput/total_tokens=937,152,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4882/500000] + train/ActionL1Loss=0.0814 + throughput/total_tokens=937,344,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4883/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=937,536,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4884/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=937,728,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4885/500000] + train/ActionL1Loss=0.0761 + throughput/total_tokens=937,920,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4886/500000] + train/ActionL1Loss=0.0891 + throughput/total_tokens=938,112,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4887/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=938,304,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4888/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=938,496,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4889/500000] + train/ActionL1Loss=0.0823 + throughput/total_tokens=938,688,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4890/500000] + train/ActionL1Loss=0.0655 + throughput/total_tokens=938,880,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4891/500000] + train/ActionL1Loss=0.0904 + throughput/total_tokens=939,072,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4892/500000] + train/ActionL1Loss=0.0626 + throughput/total_tokens=939,264,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4893/500000] + train/ActionL1Loss=0.1056 + throughput/total_tokens=939,456,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4894/500000] + train/ActionL1Loss=0.0690 + throughput/total_tokens=939,648,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4895/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=939,840,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4896/500000] + train/ActionL1Loss=0.0727 + throughput/total_tokens=940,032,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4897/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=940,224,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4898/500000] + train/ActionL1Loss=0.0697 + throughput/total_tokens=940,416,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4899/500000] + train/ActionL1Loss=0.0683 + throughput/total_tokens=940,608,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4900/500000] + optim/total_grad_norm=4.499 + train/ActionL1Loss=0.0691 + throughput/total_tokens=940,800,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4901/500000] + train/ActionL1Loss=0.0729 + throughput/total_tokens=940,992,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4902/500000] + train/ActionL1Loss=0.0776 + throughput/total_tokens=941,184,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4903/500000] + train/ActionL1Loss=0.0794 + throughput/total_tokens=941,376,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4904/500000] + train/ActionL1Loss=0.0830 + throughput/total_tokens=941,568,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4905/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=941,760,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=4906/500000] + train/ActionL1Loss=0.0624 + throughput/total_tokens=941,952,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4907/500000] + train/ActionL1Loss=0.0542 + throughput/total_tokens=942,144,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4908/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=942,336,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4909/500000] + train/ActionL1Loss=0.0727 + throughput/total_tokens=942,528,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4910/500000] + train/ActionL1Loss=0.0942 + throughput/total_tokens=942,720,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=4911/500000] + train/ActionL1Loss=0.0655 + throughput/total_tokens=942,912,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4912/500000] + train/ActionL1Loss=0.0731 + throughput/total_tokens=943,104,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4913/500000] + train/ActionL1Loss=0.0655 + throughput/total_tokens=943,296,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4914/500000] + train/ActionL1Loss=0.0744 + throughput/total_tokens=943,488,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=4915/500000] + train/ActionL1Loss=0.0827 + throughput/total_tokens=943,680,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4916/500000] + train/ActionL1Loss=0.0579 + throughput/total_tokens=943,872,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4917/500000] + train/ActionL1Loss=0.0787 + throughput/total_tokens=944,064,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4918/500000] + train/ActionL1Loss=0.0872 + throughput/total_tokens=944,256,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4919/500000] + train/ActionL1Loss=0.0593 + throughput/total_tokens=944,448,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=4920/500000] + optim/total_grad_norm=6.650 + train/ActionL1Loss=0.0734 + throughput/total_tokens=944,640,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4921/500000] + train/ActionL1Loss=0.0697 + throughput/total_tokens=944,832,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4922/500000] + train/ActionL1Loss=0.0578 + throughput/total_tokens=945,024,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4923/500000] + train/ActionL1Loss=0.0670 + throughput/total_tokens=945,216,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4924/500000] + train/ActionL1Loss=0.0594 + throughput/total_tokens=945,408,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4925/500000] + train/ActionL1Loss=0.0572 + throughput/total_tokens=945,600,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4926/500000] + train/ActionL1Loss=0.0687 + throughput/total_tokens=945,792,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4927/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=945,984,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4928/500000] + train/ActionL1Loss=0.0570 + throughput/total_tokens=946,176,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4929/500000] + train/ActionL1Loss=0.0790 + throughput/total_tokens=946,368,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4930/500000] + train/ActionL1Loss=0.0857 + throughput/total_tokens=946,560,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4931/500000] + train/ActionL1Loss=0.0879 + throughput/total_tokens=946,752,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4932/500000] + train/ActionL1Loss=0.0874 + throughput/total_tokens=946,944,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4933/500000] + train/ActionL1Loss=0.0657 + throughput/total_tokens=947,136,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4934/500000] + train/ActionL1Loss=0.0727 + throughput/total_tokens=947,328,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4935/500000] + train/ActionL1Loss=0.0508 + throughput/total_tokens=947,520,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4936/500000] + train/ActionL1Loss=0.0887 + throughput/total_tokens=947,712,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4937/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=947,904,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4938/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=948,096,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=4939/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=948,288,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4940/500000] + optim/total_grad_norm=7.143 + train/ActionL1Loss=0.0536 + throughput/total_tokens=948,480,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4941/500000] + train/ActionL1Loss=0.0551 + throughput/total_tokens=948,672,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4942/500000] + train/ActionL1Loss=0.0597 + throughput/total_tokens=948,864,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4943/500000] + train/ActionL1Loss=0.0798 + throughput/total_tokens=949,056,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4944/500000] + train/ActionL1Loss=0.0680 + throughput/total_tokens=949,248,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4945/500000] + train/ActionL1Loss=0.0690 + throughput/total_tokens=949,440,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4946/500000] + train/ActionL1Loss=0.0573 + throughput/total_tokens=949,632,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4947/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=949,824,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4948/500000] + train/ActionL1Loss=0.0709 + throughput/total_tokens=950,016,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4949/500000] + train/ActionL1Loss=0.0507 + throughput/total_tokens=950,208,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=4950/500000] + train/ActionL1Loss=0.0710 + throughput/total_tokens=950,400,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4951/500000] + train/ActionL1Loss=0.0598 + throughput/total_tokens=950,592,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4952/500000] + train/ActionL1Loss=0.0650 + throughput/total_tokens=950,784,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4953/500000] + train/ActionL1Loss=0.0674 + throughput/total_tokens=950,976,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4954/500000] + train/ActionL1Loss=0.0883 + throughput/total_tokens=951,168,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4955/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=951,360,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4956/500000] + train/ActionL1Loss=0.0794 + throughput/total_tokens=951,552,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4957/500000] + train/ActionL1Loss=0.0728 + throughput/total_tokens=951,744,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4958/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=951,936,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4959/500000] + train/ActionL1Loss=0.0795 + throughput/total_tokens=952,128,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4960/500000] + optim/total_grad_norm=6.449 + train/ActionL1Loss=0.0762 + throughput/total_tokens=952,320,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4961/500000] + train/ActionL1Loss=0.0544 + throughput/total_tokens=952,512,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4962/500000] + train/ActionL1Loss=0.0734 + throughput/total_tokens=952,704,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4963/500000] + train/ActionL1Loss=0.0980 + throughput/total_tokens=952,896,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4964/500000] + train/ActionL1Loss=0.0807 + throughput/total_tokens=953,088,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4965/500000] + train/ActionL1Loss=0.0683 + throughput/total_tokens=953,280,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4966/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=953,472,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4967/500000] + train/ActionL1Loss=0.0776 + throughput/total_tokens=953,664,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4968/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=953,856,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4969/500000] + train/ActionL1Loss=0.0683 + throughput/total_tokens=954,048,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=4970/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=954,240,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4971/500000] + train/ActionL1Loss=0.0775 + throughput/total_tokens=954,432,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4972/500000] + train/ActionL1Loss=0.0859 + throughput/total_tokens=954,624,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4973/500000] + train/ActionL1Loss=0.0701 + throughput/total_tokens=954,816,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4974/500000] + train/ActionL1Loss=0.0837 + throughput/total_tokens=955,008,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4975/500000] + train/ActionL1Loss=0.0813 + throughput/total_tokens=955,200,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4976/500000] + train/ActionL1Loss=0.0726 + throughput/total_tokens=955,392,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4977/500000] + train/ActionL1Loss=0.0545 + throughput/total_tokens=955,584,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4978/500000] + train/ActionL1Loss=0.0747 + throughput/total_tokens=955,776,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4979/500000] + train/ActionL1Loss=0.0829 + throughput/total_tokens=955,968,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4980/500000] + optim/total_grad_norm=7.788 + train/ActionL1Loss=0.0648 + throughput/total_tokens=956,160,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=4981/500000] + train/ActionL1Loss=0.0666 + throughput/total_tokens=956,352,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4982/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=956,544,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4983/500000] + train/ActionL1Loss=0.0629 + throughput/total_tokens=956,736,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4984/500000] + train/ActionL1Loss=0.0862 + throughput/total_tokens=956,928,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4985/500000] + train/ActionL1Loss=0.0779 + throughput/total_tokens=957,120,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4986/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=957,312,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=4987/500000] + train/ActionL1Loss=0.0809 + throughput/total_tokens=957,504,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4988/500000] + train/ActionL1Loss=0.0837 + throughput/total_tokens=957,696,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4989/500000] + train/ActionL1Loss=0.0836 + throughput/total_tokens=957,888,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=4990/500000] + train/ActionL1Loss=0.0709 + throughput/total_tokens=958,080,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=4991/500000] + train/ActionL1Loss=0.0840 + throughput/total_tokens=958,272,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4992/500000] + train/ActionL1Loss=0.0888 + throughput/total_tokens=958,464,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4993/500000] + train/ActionL1Loss=0.0542 + throughput/total_tokens=958,656,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4994/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=958,848,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4995/500000] + train/ActionL1Loss=0.0790 + throughput/total_tokens=959,040,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4996/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=959,232,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4997/500000] + train/ActionL1Loss=0.0760 + throughput/total_tokens=959,424,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4998/500000] + train/ActionL1Loss=0.0767 + throughput/total_tokens=959,616,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=4999/500000] + train/ActionL1Loss=0.0769 + throughput/total_tokens=959,808,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5000/500000] + optim/total_grad_norm=9.134 + train/ActionL1Loss=0.0600 + throughput/total_tokens=960,000,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +10/01 [18:40:18] INFO | >> Saving config... checkpoint.py:608 +10/01 [18:40:48] INFO | >> Saving model state... checkpoint.py:796 +10/01 [18:42:00] INFO | >> Saving optim state... checkpoint.py:811 +10/01 [18:43:32] INFO | >> Saving trainer state... checkpoint.py:670 + INFO | >> Saving config... checkpoint.py:608 +[step=5001/500000] + train/ActionL1Loss=0.1007 + throughput/total_tokens=960,192,000 + throughput/device/tokens_per_second=1,212 + throughput/device/batches_per_second=0.0505 +[step=5002/500000] + train/ActionL1Loss=0.0846 + throughput/total_tokens=960,384,000 + throughput/device/tokens_per_second=1,173 + throughput/device/batches_per_second=0.0489 +[step=5003/500000] + train/ActionL1Loss=0.0754 + throughput/total_tokens=960,576,000 + throughput/device/tokens_per_second=1,144 + throughput/device/batches_per_second=0.0477 +[step=5004/500000] + train/ActionL1Loss=0.0623 + throughput/total_tokens=960,768,000 + throughput/device/tokens_per_second=1,142 + throughput/device/batches_per_second=0.0476 +[step=5005/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=960,960,000 + throughput/device/tokens_per_second=1,146 + throughput/device/batches_per_second=0.0478 +[step=5006/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=961,152,000 + throughput/device/tokens_per_second=1,150 + throughput/device/batches_per_second=0.0479 +[step=5007/500000] + train/ActionL1Loss=0.0769 + throughput/total_tokens=961,344,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 +[step=5008/500000] + train/ActionL1Loss=0.0760 + throughput/total_tokens=961,536,000 + throughput/device/tokens_per_second=1,155 + throughput/device/batches_per_second=0.0481 +[step=5009/500000] + train/ActionL1Loss=0.0837 + throughput/total_tokens=961,728,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=5010/500000] + train/ActionL1Loss=0.0677 + throughput/total_tokens=961,920,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 + System/Peak GPU Memory (MB)=46,917 +[step=5011/500000] + train/ActionL1Loss=0.0824 + throughput/total_tokens=962,112,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=5012/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=962,304,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=5013/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=962,496,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=5014/500000] + train/ActionL1Loss=0.0706 + throughput/total_tokens=962,688,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=5015/500000] + train/ActionL1Loss=0.0848 + throughput/total_tokens=962,880,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=5016/500000] + train/ActionL1Loss=0.0669 + throughput/total_tokens=963,072,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=5017/500000] + train/ActionL1Loss=0.0840 + throughput/total_tokens=963,264,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=5018/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=963,456,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=5019/500000] + train/ActionL1Loss=0.0576 + throughput/total_tokens=963,648,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=5020/500000] + optim/total_grad_norm=6.731 + train/ActionL1Loss=0.0657 + throughput/total_tokens=963,840,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=5021/500000] + train/ActionL1Loss=0.0765 + throughput/total_tokens=964,032,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=5022/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=964,224,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5023/500000] + train/ActionL1Loss=0.0755 + throughput/total_tokens=964,416,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5024/500000] + train/ActionL1Loss=0.0909 + throughput/total_tokens=964,608,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5025/500000] + train/ActionL1Loss=0.0770 + throughput/total_tokens=964,800,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5026/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=964,992,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5027/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=965,184,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5028/500000] + train/ActionL1Loss=0.0634 + throughput/total_tokens=965,376,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5029/500000] + train/ActionL1Loss=0.0863 + throughput/total_tokens=965,568,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5030/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=965,760,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5031/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=965,952,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5032/500000] + train/ActionL1Loss=0.0733 + throughput/total_tokens=966,144,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5033/500000] + train/ActionL1Loss=0.0650 + throughput/total_tokens=966,336,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5034/500000] + train/ActionL1Loss=0.0543 + throughput/total_tokens=966,528,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5035/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=966,720,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5036/500000] + train/ActionL1Loss=0.0648 + throughput/total_tokens=966,912,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5037/500000] + train/ActionL1Loss=0.0608 + throughput/total_tokens=967,104,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5038/500000] + train/ActionL1Loss=0.0592 + throughput/total_tokens=967,296,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5039/500000] + train/ActionL1Loss=0.0655 + throughput/total_tokens=967,488,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5040/500000] + optim/total_grad_norm=6.433 + train/ActionL1Loss=0.0700 + throughput/total_tokens=967,680,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=5041/500000] + train/ActionL1Loss=0.0560 + throughput/total_tokens=967,872,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5042/500000] + train/ActionL1Loss=0.0623 + throughput/total_tokens=968,064,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5043/500000] + train/ActionL1Loss=0.0704 + throughput/total_tokens=968,256,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5044/500000] + train/ActionL1Loss=0.0684 + throughput/total_tokens=968,448,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5045/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=968,640,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5046/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=968,832,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5047/500000] + train/ActionL1Loss=0.0680 + throughput/total_tokens=969,024,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5048/500000] + train/ActionL1Loss=0.0729 + throughput/total_tokens=969,216,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5049/500000] + train/ActionL1Loss=0.0839 + throughput/total_tokens=969,408,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5050/500000] + train/ActionL1Loss=0.0609 + throughput/total_tokens=969,600,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5051/500000] + train/ActionL1Loss=0.0741 + throughput/total_tokens=969,792,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5052/500000] + train/ActionL1Loss=0.0864 + throughput/total_tokens=969,984,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5053/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=970,176,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5054/500000] + train/ActionL1Loss=0.0732 + throughput/total_tokens=970,368,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5055/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=970,560,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5056/500000] + train/ActionL1Loss=0.0758 + throughput/total_tokens=970,752,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5057/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=970,944,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5058/500000] + train/ActionL1Loss=0.0641 + throughput/total_tokens=971,136,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5059/500000] + train/ActionL1Loss=0.0833 + throughput/total_tokens=971,328,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5060/500000] + optim/total_grad_norm=5.551 + train/ActionL1Loss=0.0815 + throughput/total_tokens=971,520,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5061/500000] + train/ActionL1Loss=0.0701 + throughput/total_tokens=971,712,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5062/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=971,904,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5063/500000] + train/ActionL1Loss=0.0779 + throughput/total_tokens=972,096,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5064/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=972,288,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5065/500000] + train/ActionL1Loss=0.0609 + throughput/total_tokens=972,480,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5066/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=972,672,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5067/500000] + train/ActionL1Loss=0.0767 + throughput/total_tokens=972,864,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5068/500000] + train/ActionL1Loss=0.0857 + throughput/total_tokens=973,056,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5069/500000] + train/ActionL1Loss=0.0678 + throughput/total_tokens=973,248,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5070/500000] + train/ActionL1Loss=0.0865 + throughput/total_tokens=973,440,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5071/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=973,632,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5072/500000] + train/ActionL1Loss=0.0726 + throughput/total_tokens=973,824,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5073/500000] + train/ActionL1Loss=0.0830 + throughput/total_tokens=974,016,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5074/500000] + train/ActionL1Loss=0.0927 + throughput/total_tokens=974,208,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5075/500000] + train/ActionL1Loss=0.0674 + throughput/total_tokens=974,400,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5076/500000] + train/ActionL1Loss=0.0581 + throughput/total_tokens=974,592,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5077/500000] + train/ActionL1Loss=0.0704 + throughput/total_tokens=974,784,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5078/500000] + train/ActionL1Loss=0.0676 + throughput/total_tokens=974,976,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5079/500000] + train/ActionL1Loss=0.0867 + throughput/total_tokens=975,168,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5080/500000] + optim/total_grad_norm=6.563 + train/ActionL1Loss=0.0860 + throughput/total_tokens=975,360,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5081/500000] + train/ActionL1Loss=0.0855 + throughput/total_tokens=975,552,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5082/500000] + train/ActionL1Loss=0.0706 + throughput/total_tokens=975,744,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5083/500000] + train/ActionL1Loss=0.0722 + throughput/total_tokens=975,936,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5084/500000] + train/ActionL1Loss=0.0709 + throughput/total_tokens=976,128,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5085/500000] + train/ActionL1Loss=0.0665 + throughput/total_tokens=976,320,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5086/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=976,512,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5087/500000] + train/ActionL1Loss=0.0603 + throughput/total_tokens=976,704,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5088/500000] + train/ActionL1Loss=0.0598 + throughput/total_tokens=976,896,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5089/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=977,088,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5090/500000] + train/ActionL1Loss=0.0646 + throughput/total_tokens=977,280,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5091/500000] + train/ActionL1Loss=0.0801 + throughput/total_tokens=977,472,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5092/500000] + train/ActionL1Loss=0.0585 + throughput/total_tokens=977,664,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5093/500000] + train/ActionL1Loss=0.0704 + throughput/total_tokens=977,856,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5094/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=978,048,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5095/500000] + train/ActionL1Loss=0.0803 + throughput/total_tokens=978,240,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5096/500000] + train/ActionL1Loss=0.0829 + throughput/total_tokens=978,432,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5097/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=978,624,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5098/500000] + train/ActionL1Loss=0.0554 + throughput/total_tokens=978,816,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5099/500000] + train/ActionL1Loss=0.0771 + throughput/total_tokens=979,008,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5100/500000] + optim/total_grad_norm=7.381 + train/ActionL1Loss=0.0846 + throughput/total_tokens=979,200,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5101/500000] + train/ActionL1Loss=0.0805 + throughput/total_tokens=979,392,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5102/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=979,584,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5103/500000] + train/ActionL1Loss=0.0657 + throughput/total_tokens=979,776,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5104/500000] + train/ActionL1Loss=0.0550 + throughput/total_tokens=979,968,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5105/500000] + train/ActionL1Loss=0.0840 + throughput/total_tokens=980,160,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5106/500000] + train/ActionL1Loss=0.0861 + throughput/total_tokens=980,352,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5107/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=980,544,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5108/500000] + train/ActionL1Loss=0.0512 + throughput/total_tokens=980,736,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5109/500000] + train/ActionL1Loss=0.0545 + throughput/total_tokens=980,928,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5110/500000] + train/ActionL1Loss=0.0611 + throughput/total_tokens=981,120,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5111/500000] + train/ActionL1Loss=0.0610 + throughput/total_tokens=981,312,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5112/500000] + train/ActionL1Loss=0.0757 + throughput/total_tokens=981,504,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5113/500000] + train/ActionL1Loss=0.0579 + throughput/total_tokens=981,696,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5114/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=981,888,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5115/500000] + train/ActionL1Loss=0.0684 + throughput/total_tokens=982,080,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5116/500000] + train/ActionL1Loss=0.0511 + throughput/total_tokens=982,272,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5117/500000] + train/ActionL1Loss=0.0820 + throughput/total_tokens=982,464,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5118/500000] + train/ActionL1Loss=0.1020 + throughput/total_tokens=982,656,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5119/500000] + train/ActionL1Loss=0.0977 + throughput/total_tokens=982,848,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5120/500000] + optim/total_grad_norm=6.422 + train/ActionL1Loss=0.0601 + throughput/total_tokens=983,040,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5121/500000] + train/ActionL1Loss=0.0634 + throughput/total_tokens=983,232,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5122/500000] + train/ActionL1Loss=0.0643 + throughput/total_tokens=983,424,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5123/500000] + train/ActionL1Loss=0.0793 + throughput/total_tokens=983,616,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5124/500000] + train/ActionL1Loss=0.0829 + throughput/total_tokens=983,808,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5125/500000] + train/ActionL1Loss=0.0741 + throughput/total_tokens=984,000,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5126/500000] + train/ActionL1Loss=0.0698 + throughput/total_tokens=984,192,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5127/500000] + train/ActionL1Loss=0.0649 + throughput/total_tokens=984,384,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5128/500000] + train/ActionL1Loss=0.0790 + throughput/total_tokens=984,576,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5129/500000] + train/ActionL1Loss=0.0800 + throughput/total_tokens=984,768,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5130/500000] + train/ActionL1Loss=0.0724 + throughput/total_tokens=984,960,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5131/500000] + train/ActionL1Loss=0.0543 + throughput/total_tokens=985,152,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5132/500000] + train/ActionL1Loss=0.0697 + throughput/total_tokens=985,344,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5133/500000] + train/ActionL1Loss=0.0641 + throughput/total_tokens=985,536,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5134/500000] + train/ActionL1Loss=0.0660 + throughput/total_tokens=985,728,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5135/500000] + train/ActionL1Loss=0.0528 + throughput/total_tokens=985,920,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5136/500000] + train/ActionL1Loss=0.0720 + throughput/total_tokens=986,112,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5137/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=986,304,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5138/500000] + train/ActionL1Loss=0.0843 + throughput/total_tokens=986,496,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5139/500000] + train/ActionL1Loss=0.0746 + throughput/total_tokens=986,688,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5140/500000] + optim/total_grad_norm=6.562 + train/ActionL1Loss=0.0856 + throughput/total_tokens=986,880,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5141/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=987,072,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5142/500000] + train/ActionL1Loss=0.0531 + throughput/total_tokens=987,264,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5143/500000] + train/ActionL1Loss=0.0792 + throughput/total_tokens=987,456,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5144/500000] + train/ActionL1Loss=0.0787 + throughput/total_tokens=987,648,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5145/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=987,840,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5146/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=988,032,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5147/500000] + train/ActionL1Loss=0.0621 + throughput/total_tokens=988,224,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5148/500000] + train/ActionL1Loss=0.0822 + throughput/total_tokens=988,416,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5149/500000] + train/ActionL1Loss=0.0765 + throughput/total_tokens=988,608,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5150/500000] + train/ActionL1Loss=0.0648 + throughput/total_tokens=988,800,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5151/500000] + train/ActionL1Loss=0.0739 + throughput/total_tokens=988,992,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5152/500000] + train/ActionL1Loss=0.0718 + throughput/total_tokens=989,184,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5153/500000] + train/ActionL1Loss=0.0729 + throughput/total_tokens=989,376,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5154/500000] + train/ActionL1Loss=0.0792 + throughput/total_tokens=989,568,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5155/500000] + train/ActionL1Loss=0.0726 + throughput/total_tokens=989,760,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5156/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=989,952,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5157/500000] + train/ActionL1Loss=0.0684 + throughput/total_tokens=990,144,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5158/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=990,336,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5159/500000] + train/ActionL1Loss=0.0702 + throughput/total_tokens=990,528,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5160/500000] + optim/total_grad_norm=5.951 + train/ActionL1Loss=0.0816 + throughput/total_tokens=990,720,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5161/500000] + train/ActionL1Loss=0.0517 + throughput/total_tokens=990,912,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5162/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=991,104,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5163/500000] + train/ActionL1Loss=0.0749 + throughput/total_tokens=991,296,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5164/500000] + train/ActionL1Loss=0.0859 + throughput/total_tokens=991,488,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5165/500000] + train/ActionL1Loss=0.0740 + throughput/total_tokens=991,680,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5166/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=991,872,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5167/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=992,064,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5168/500000] + train/ActionL1Loss=0.0714 + throughput/total_tokens=992,256,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5169/500000] + train/ActionL1Loss=0.0847 + throughput/total_tokens=992,448,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5170/500000] + train/ActionL1Loss=0.0936 + throughput/total_tokens=992,640,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5171/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=992,832,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5172/500000] + train/ActionL1Loss=0.0683 + throughput/total_tokens=993,024,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5173/500000] + train/ActionL1Loss=0.0684 + throughput/total_tokens=993,216,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5174/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=993,408,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5175/500000] + train/ActionL1Loss=0.0664 + throughput/total_tokens=993,600,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5176/500000] + train/ActionL1Loss=0.0673 + throughput/total_tokens=993,792,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5177/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=993,984,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5178/500000] + train/ActionL1Loss=0.0830 + throughput/total_tokens=994,176,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5179/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=994,368,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5180/500000] + optim/total_grad_norm=5.814 + train/ActionL1Loss=0.0612 + throughput/total_tokens=994,560,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5181/500000] + train/ActionL1Loss=0.0640 + throughput/total_tokens=994,752,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5182/500000] + train/ActionL1Loss=0.0814 + throughput/total_tokens=994,944,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5183/500000] + train/ActionL1Loss=0.0703 + throughput/total_tokens=995,136,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5184/500000] + train/ActionL1Loss=0.0867 + throughput/total_tokens=995,328,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5185/500000] + train/ActionL1Loss=0.0652 + throughput/total_tokens=995,520,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5186/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=995,712,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5187/500000] + train/ActionL1Loss=0.0720 + throughput/total_tokens=995,904,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5188/500000] + train/ActionL1Loss=0.0837 + throughput/total_tokens=996,096,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5189/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=996,288,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5190/500000] + train/ActionL1Loss=0.0854 + throughput/total_tokens=996,480,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5191/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=996,672,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5192/500000] + train/ActionL1Loss=0.0543 + throughput/total_tokens=996,864,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5193/500000] + train/ActionL1Loss=0.0734 + throughput/total_tokens=997,056,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5194/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=997,248,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5195/500000] + train/ActionL1Loss=0.0826 + throughput/total_tokens=997,440,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5196/500000] + train/ActionL1Loss=0.0736 + throughput/total_tokens=997,632,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5197/500000] + train/ActionL1Loss=0.0707 + throughput/total_tokens=997,824,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5198/500000] + train/ActionL1Loss=0.0555 + throughput/total_tokens=998,016,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5199/500000] + train/ActionL1Loss=0.0720 + throughput/total_tokens=998,208,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5200/500000] + optim/total_grad_norm=6.282 + train/ActionL1Loss=0.0632 + throughput/total_tokens=998,400,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5201/500000] + train/ActionL1Loss=0.0718 + throughput/total_tokens=998,592,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5202/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=998,784,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5203/500000] + train/ActionL1Loss=0.0683 + throughput/total_tokens=998,976,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5204/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=999,168,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5205/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=999,360,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5206/500000] + train/ActionL1Loss=0.0522 + throughput/total_tokens=999,552,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5207/500000] + train/ActionL1Loss=0.0752 + throughput/total_tokens=999,744,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5208/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=999,936,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5209/500000] + train/ActionL1Loss=0.0755 + throughput/total_tokens=1,000,128,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5210/500000] + train/ActionL1Loss=0.0649 + throughput/total_tokens=1,000,320,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=5211/500000] + train/ActionL1Loss=0.0904 + throughput/total_tokens=1,000,512,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5212/500000] + train/ActionL1Loss=0.0808 + throughput/total_tokens=1,000,704,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5213/500000] + train/ActionL1Loss=0.0928 + throughput/total_tokens=1,000,896,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5214/500000] + train/ActionL1Loss=0.0846 + throughput/total_tokens=1,001,088,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5215/500000] + train/ActionL1Loss=0.0629 + throughput/total_tokens=1,001,280,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5216/500000] + train/ActionL1Loss=0.0800 + throughput/total_tokens=1,001,472,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5217/500000] + train/ActionL1Loss=0.0499 + throughput/total_tokens=1,001,664,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5218/500000] + train/ActionL1Loss=0.0794 + throughput/total_tokens=1,001,856,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5219/500000] + train/ActionL1Loss=0.0706 + throughput/total_tokens=1,002,048,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5220/500000] + optim/total_grad_norm=7.165 + train/ActionL1Loss=0.0696 + throughput/total_tokens=1,002,240,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5221/500000] + train/ActionL1Loss=0.0740 + throughput/total_tokens=1,002,432,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5222/500000] + train/ActionL1Loss=0.0936 + throughput/total_tokens=1,002,624,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5223/500000] + train/ActionL1Loss=0.0760 + throughput/total_tokens=1,002,816,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5224/500000] + train/ActionL1Loss=0.0680 + throughput/total_tokens=1,003,008,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5225/500000] + train/ActionL1Loss=0.0835 + throughput/total_tokens=1,003,200,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5226/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=1,003,392,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5227/500000] + train/ActionL1Loss=0.0828 + throughput/total_tokens=1,003,584,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5228/500000] + train/ActionL1Loss=0.0609 + throughput/total_tokens=1,003,776,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5229/500000] + train/ActionL1Loss=0.0635 + throughput/total_tokens=1,003,968,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5230/500000] + train/ActionL1Loss=0.0766 + throughput/total_tokens=1,004,160,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5231/500000] + train/ActionL1Loss=0.0547 + throughput/total_tokens=1,004,352,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5232/500000] + train/ActionL1Loss=0.0623 + throughput/total_tokens=1,004,544,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5233/500000] + train/ActionL1Loss=0.0587 + throughput/total_tokens=1,004,736,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5234/500000] + train/ActionL1Loss=0.0601 + throughput/total_tokens=1,004,928,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5235/500000] + train/ActionL1Loss=0.0626 + throughput/total_tokens=1,005,120,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5236/500000] + train/ActionL1Loss=0.0784 + throughput/total_tokens=1,005,312,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5237/500000] + train/ActionL1Loss=0.0744 + throughput/total_tokens=1,005,504,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5238/500000] + train/ActionL1Loss=0.0687 + throughput/total_tokens=1,005,696,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5239/500000] + train/ActionL1Loss=0.0560 + throughput/total_tokens=1,005,888,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5240/500000] + optim/total_grad_norm=6.904 + train/ActionL1Loss=0.0653 + throughput/total_tokens=1,006,080,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5241/500000] + train/ActionL1Loss=0.0804 + throughput/total_tokens=1,006,272,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5242/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,006,464,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5243/500000] + train/ActionL1Loss=0.0543 + throughput/total_tokens=1,006,656,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5244/500000] + train/ActionL1Loss=0.0845 + throughput/total_tokens=1,006,848,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5245/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=1,007,040,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5246/500000] + train/ActionL1Loss=0.0734 + throughput/total_tokens=1,007,232,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5247/500000] + train/ActionL1Loss=0.0761 + throughput/total_tokens=1,007,424,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5248/500000] + train/ActionL1Loss=0.0832 + throughput/total_tokens=1,007,616,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5249/500000] + train/ActionL1Loss=0.0748 + throughput/total_tokens=1,007,808,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5250/500000] + train/ActionL1Loss=0.0564 + throughput/total_tokens=1,008,000,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5251/500000] + train/ActionL1Loss=0.0865 + throughput/total_tokens=1,008,192,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5252/500000] + train/ActionL1Loss=0.0560 + throughput/total_tokens=1,008,384,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5253/500000] + train/ActionL1Loss=0.0629 + throughput/total_tokens=1,008,576,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5254/500000] + train/ActionL1Loss=0.0747 + throughput/total_tokens=1,008,768,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5255/500000] + train/ActionL1Loss=0.0881 + throughput/total_tokens=1,008,960,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5256/500000] + train/ActionL1Loss=0.0585 + throughput/total_tokens=1,009,152,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5257/500000] + train/ActionL1Loss=0.0797 + throughput/total_tokens=1,009,344,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5258/500000] + train/ActionL1Loss=0.0797 + throughput/total_tokens=1,009,536,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5259/500000] + train/ActionL1Loss=0.0911 + throughput/total_tokens=1,009,728,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5260/500000] + optim/total_grad_norm=7.898 + train/ActionL1Loss=0.0669 + throughput/total_tokens=1,009,920,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5261/500000] + train/ActionL1Loss=0.0762 + throughput/total_tokens=1,010,112,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5262/500000] + train/ActionL1Loss=0.0870 + throughput/total_tokens=1,010,304,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5263/500000] + train/ActionL1Loss=0.0850 + throughput/total_tokens=1,010,496,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5264/500000] + train/ActionL1Loss=0.0735 + throughput/total_tokens=1,010,688,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5265/500000] + train/ActionL1Loss=0.0909 + throughput/total_tokens=1,010,880,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5266/500000] + train/ActionL1Loss=0.0772 + throughput/total_tokens=1,011,072,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5267/500000] + train/ActionL1Loss=0.0795 + throughput/total_tokens=1,011,264,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5268/500000] + train/ActionL1Loss=0.0766 + throughput/total_tokens=1,011,456,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5269/500000] + train/ActionL1Loss=0.0641 + throughput/total_tokens=1,011,648,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5270/500000] + train/ActionL1Loss=0.0907 + throughput/total_tokens=1,011,840,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5271/500000] + train/ActionL1Loss=0.0875 + throughput/total_tokens=1,012,032,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5272/500000] + train/ActionL1Loss=0.0820 + throughput/total_tokens=1,012,224,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5273/500000] + train/ActionL1Loss=0.0957 + throughput/total_tokens=1,012,416,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5274/500000] + train/ActionL1Loss=0.0560 + throughput/total_tokens=1,012,608,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5275/500000] + train/ActionL1Loss=0.0599 + throughput/total_tokens=1,012,800,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5276/500000] + train/ActionL1Loss=0.0821 + throughput/total_tokens=1,012,992,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5277/500000] + train/ActionL1Loss=0.0787 + throughput/total_tokens=1,013,184,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5278/500000] + train/ActionL1Loss=0.0704 + throughput/total_tokens=1,013,376,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5279/500000] + train/ActionL1Loss=0.0757 + throughput/total_tokens=1,013,568,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5280/500000] + optim/total_grad_norm=7.729 + train/ActionL1Loss=0.0623 + throughput/total_tokens=1,013,760,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5281/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,013,952,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5282/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,014,144,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5283/500000] + train/ActionL1Loss=0.0801 + throughput/total_tokens=1,014,336,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5284/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,014,528,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5285/500000] + train/ActionL1Loss=0.0670 + throughput/total_tokens=1,014,720,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5286/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=1,014,912,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5287/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=1,015,104,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5288/500000] + train/ActionL1Loss=0.0601 + throughput/total_tokens=1,015,296,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5289/500000] + train/ActionL1Loss=0.0571 + throughput/total_tokens=1,015,488,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5290/500000] + train/ActionL1Loss=0.0913 + throughput/total_tokens=1,015,680,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5291/500000] + train/ActionL1Loss=0.0605 + throughput/total_tokens=1,015,872,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5292/500000] + train/ActionL1Loss=0.0599 + throughput/total_tokens=1,016,064,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5293/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,016,256,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5294/500000] + train/ActionL1Loss=0.0573 + throughput/total_tokens=1,016,448,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5295/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=1,016,640,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5296/500000] + train/ActionL1Loss=0.0710 + throughput/total_tokens=1,016,832,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5297/500000] + train/ActionL1Loss=0.0689 + throughput/total_tokens=1,017,024,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5298/500000] + train/ActionL1Loss=0.0545 + throughput/total_tokens=1,017,216,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5299/500000] + train/ActionL1Loss=0.0662 + throughput/total_tokens=1,017,408,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5300/500000] + optim/total_grad_norm=5.918 + train/ActionL1Loss=0.0730 + throughput/total_tokens=1,017,600,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5301/500000] + train/ActionL1Loss=0.0716 + throughput/total_tokens=1,017,792,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5302/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=1,017,984,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5303/500000] + train/ActionL1Loss=0.0882 + throughput/total_tokens=1,018,176,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5304/500000] + train/ActionL1Loss=0.0800 + throughput/total_tokens=1,018,368,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5305/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=1,018,560,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5306/500000] + train/ActionL1Loss=0.0701 + throughput/total_tokens=1,018,752,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5307/500000] + train/ActionL1Loss=0.0656 + throughput/total_tokens=1,018,944,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5308/500000] + train/ActionL1Loss=0.0820 + throughput/total_tokens=1,019,136,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5309/500000] + train/ActionL1Loss=0.0570 + throughput/total_tokens=1,019,328,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5310/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=1,019,520,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=5311/500000] + train/ActionL1Loss=0.0560 + throughput/total_tokens=1,019,712,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5312/500000] + train/ActionL1Loss=0.0546 + throughput/total_tokens=1,019,904,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5313/500000] + train/ActionL1Loss=0.0713 + throughput/total_tokens=1,020,096,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5314/500000] + train/ActionL1Loss=0.0691 + throughput/total_tokens=1,020,288,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5315/500000] + train/ActionL1Loss=0.0562 + throughput/total_tokens=1,020,480,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5316/500000] + train/ActionL1Loss=0.0814 + throughput/total_tokens=1,020,672,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5317/500000] + train/ActionL1Loss=0.0733 + throughput/total_tokens=1,020,864,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5318/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=1,021,056,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5319/500000] + train/ActionL1Loss=0.0721 + throughput/total_tokens=1,021,248,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5320/500000] + optim/total_grad_norm=5.386 + train/ActionL1Loss=0.0616 + throughput/total_tokens=1,021,440,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5321/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=1,021,632,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5322/500000] + train/ActionL1Loss=0.0701 + throughput/total_tokens=1,021,824,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5323/500000] + train/ActionL1Loss=0.0723 + throughput/total_tokens=1,022,016,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5324/500000] + train/ActionL1Loss=0.0573 + throughput/total_tokens=1,022,208,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5325/500000] + train/ActionL1Loss=0.0559 + throughput/total_tokens=1,022,400,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5326/500000] + train/ActionL1Loss=0.0734 + throughput/total_tokens=1,022,592,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5327/500000] + train/ActionL1Loss=0.0648 + throughput/total_tokens=1,022,784,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5328/500000] + train/ActionL1Loss=0.0746 + throughput/total_tokens=1,022,976,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5329/500000] + train/ActionL1Loss=0.0598 + throughput/total_tokens=1,023,168,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5330/500000] + train/ActionL1Loss=0.0997 + throughput/total_tokens=1,023,360,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5331/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=1,023,552,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5332/500000] + train/ActionL1Loss=0.0607 + throughput/total_tokens=1,023,744,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5333/500000] + train/ActionL1Loss=0.0730 + throughput/total_tokens=1,023,936,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5334/500000] + train/ActionL1Loss=0.0760 + throughput/total_tokens=1,024,128,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5335/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=1,024,320,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5336/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=1,024,512,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5337/500000] + train/ActionL1Loss=0.0800 + throughput/total_tokens=1,024,704,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5338/500000] + train/ActionL1Loss=0.0943 + throughput/total_tokens=1,024,896,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5339/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,025,088,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5340/500000] + optim/total_grad_norm=6.566 + train/ActionL1Loss=0.0730 + throughput/total_tokens=1,025,280,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5341/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,025,472,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5342/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=1,025,664,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5343/500000] + train/ActionL1Loss=0.0617 + throughput/total_tokens=1,025,856,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5344/500000] + train/ActionL1Loss=0.0561 + throughput/total_tokens=1,026,048,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5345/500000] + train/ActionL1Loss=0.0655 + throughput/total_tokens=1,026,240,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5346/500000] + train/ActionL1Loss=0.0831 + throughput/total_tokens=1,026,432,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5347/500000] + train/ActionL1Loss=0.0577 + throughput/total_tokens=1,026,624,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5348/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=1,026,816,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5349/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,027,008,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5350/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=1,027,200,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=5351/500000] + train/ActionL1Loss=0.0684 + throughput/total_tokens=1,027,392,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5352/500000] + train/ActionL1Loss=0.0910 + throughput/total_tokens=1,027,584,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5353/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=1,027,776,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5354/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=1,027,968,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5355/500000] + train/ActionL1Loss=0.0664 + throughput/total_tokens=1,028,160,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5356/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,028,352,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5357/500000] + train/ActionL1Loss=0.0621 + throughput/total_tokens=1,028,544,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5358/500000] + train/ActionL1Loss=0.0807 + throughput/total_tokens=1,028,736,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5359/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=1,028,928,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5360/500000] + optim/total_grad_norm=5.633 + train/ActionL1Loss=0.0781 + throughput/total_tokens=1,029,120,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5361/500000] + train/ActionL1Loss=0.0806 + throughput/total_tokens=1,029,312,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5362/500000] + train/ActionL1Loss=0.0678 + throughput/total_tokens=1,029,504,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5363/500000] + train/ActionL1Loss=0.0753 + throughput/total_tokens=1,029,696,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5364/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,029,888,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5365/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=1,030,080,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5366/500000] + train/ActionL1Loss=0.0842 + throughput/total_tokens=1,030,272,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5367/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=1,030,464,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5368/500000] + train/ActionL1Loss=0.0648 + throughput/total_tokens=1,030,656,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5369/500000] + train/ActionL1Loss=0.0496 + throughput/total_tokens=1,030,848,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5370/500000] + train/ActionL1Loss=0.0780 + throughput/total_tokens=1,031,040,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5371/500000] + train/ActionL1Loss=0.0790 + throughput/total_tokens=1,031,232,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5372/500000] + train/ActionL1Loss=0.0875 + throughput/total_tokens=1,031,424,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5373/500000] + train/ActionL1Loss=0.0684 + throughput/total_tokens=1,031,616,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5374/500000] + train/ActionL1Loss=0.0653 + throughput/total_tokens=1,031,808,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5375/500000] + train/ActionL1Loss=0.0553 + throughput/total_tokens=1,032,000,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5376/500000] + train/ActionL1Loss=0.0702 + throughput/total_tokens=1,032,192,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5377/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,032,384,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5378/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=1,032,576,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5379/500000] + train/ActionL1Loss=0.0698 + throughput/total_tokens=1,032,768,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5380/500000] + optim/total_grad_norm=5.925 + train/ActionL1Loss=0.0630 + throughput/total_tokens=1,032,960,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5381/500000] + train/ActionL1Loss=0.0553 + throughput/total_tokens=1,033,152,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5382/500000] + train/ActionL1Loss=0.0690 + throughput/total_tokens=1,033,344,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5383/500000] + train/ActionL1Loss=0.0792 + throughput/total_tokens=1,033,536,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5384/500000] + train/ActionL1Loss=0.0902 + throughput/total_tokens=1,033,728,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5385/500000] + train/ActionL1Loss=0.0729 + throughput/total_tokens=1,033,920,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5386/500000] + train/ActionL1Loss=0.0636 + throughput/total_tokens=1,034,112,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5387/500000] + train/ActionL1Loss=0.0531 + throughput/total_tokens=1,034,304,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5388/500000] + train/ActionL1Loss=0.0831 + throughput/total_tokens=1,034,496,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5389/500000] + train/ActionL1Loss=0.0778 + throughput/total_tokens=1,034,688,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5390/500000] + train/ActionL1Loss=0.0703 + throughput/total_tokens=1,034,880,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5391/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=1,035,072,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5392/500000] + train/ActionL1Loss=0.0623 + throughput/total_tokens=1,035,264,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5393/500000] + train/ActionL1Loss=0.0623 + throughput/total_tokens=1,035,456,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5394/500000] + train/ActionL1Loss=0.0710 + throughput/total_tokens=1,035,648,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5395/500000] + train/ActionL1Loss=0.0669 + throughput/total_tokens=1,035,840,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5396/500000] + train/ActionL1Loss=0.0710 + throughput/total_tokens=1,036,032,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5397/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=1,036,224,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5398/500000] + train/ActionL1Loss=0.0540 + throughput/total_tokens=1,036,416,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5399/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,036,608,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5400/500000] + optim/total_grad_norm=7.516 + train/ActionL1Loss=0.0641 + throughput/total_tokens=1,036,800,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5401/500000] + train/ActionL1Loss=0.0620 + throughput/total_tokens=1,036,992,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5402/500000] + train/ActionL1Loss=0.0778 + throughput/total_tokens=1,037,184,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5403/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,037,376,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5404/500000] + train/ActionL1Loss=0.0860 + throughput/total_tokens=1,037,568,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=5405/500000] + train/ActionL1Loss=0.0732 + throughput/total_tokens=1,037,760,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=5406/500000] + train/ActionL1Loss=0.0721 + throughput/total_tokens=1,037,952,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5407/500000] + train/ActionL1Loss=0.0892 + throughput/total_tokens=1,038,144,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5408/500000] + train/ActionL1Loss=0.0766 + throughput/total_tokens=1,038,336,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5409/500000] + train/ActionL1Loss=0.0708 + throughput/total_tokens=1,038,528,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5410/500000] + train/ActionL1Loss=0.0691 + throughput/total_tokens=1,038,720,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=5411/500000] + train/ActionL1Loss=0.0793 + throughput/total_tokens=1,038,912,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5412/500000] + train/ActionL1Loss=0.0883 + throughput/total_tokens=1,039,104,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5413/500000] + train/ActionL1Loss=0.0768 + throughput/total_tokens=1,039,296,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5414/500000] + train/ActionL1Loss=0.0829 + throughput/total_tokens=1,039,488,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5415/500000] + train/ActionL1Loss=0.0649 + throughput/total_tokens=1,039,680,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5416/500000] + train/ActionL1Loss=0.0495 + throughput/total_tokens=1,039,872,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5417/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=1,040,064,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5418/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=1,040,256,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5419/500000] + train/ActionL1Loss=0.0548 + throughput/total_tokens=1,040,448,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5420/500000] + optim/total_grad_norm=5.191 + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,040,640,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5421/500000] + train/ActionL1Loss=0.0701 + throughput/total_tokens=1,040,832,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5422/500000] + train/ActionL1Loss=0.0718 + throughput/total_tokens=1,041,024,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5423/500000] + train/ActionL1Loss=0.0965 + throughput/total_tokens=1,041,216,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5424/500000] + train/ActionL1Loss=0.0649 + throughput/total_tokens=1,041,408,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5425/500000] + train/ActionL1Loss=0.0623 + throughput/total_tokens=1,041,600,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5426/500000] + train/ActionL1Loss=0.0918 + throughput/total_tokens=1,041,792,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5427/500000] + train/ActionL1Loss=0.0650 + throughput/total_tokens=1,041,984,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5428/500000] + train/ActionL1Loss=0.0785 + throughput/total_tokens=1,042,176,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5429/500000] + train/ActionL1Loss=0.0635 + throughput/total_tokens=1,042,368,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5430/500000] + train/ActionL1Loss=0.0786 + throughput/total_tokens=1,042,560,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5431/500000] + train/ActionL1Loss=0.0526 + throughput/total_tokens=1,042,752,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5432/500000] + train/ActionL1Loss=0.0732 + throughput/total_tokens=1,042,944,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5433/500000] + train/ActionL1Loss=0.0664 + throughput/total_tokens=1,043,136,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5434/500000] + train/ActionL1Loss=0.0830 + throughput/total_tokens=1,043,328,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5435/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,043,520,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5436/500000] + train/ActionL1Loss=0.0716 + throughput/total_tokens=1,043,712,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5437/500000] + train/ActionL1Loss=0.0591 + throughput/total_tokens=1,043,904,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5438/500000] + train/ActionL1Loss=0.0732 + throughput/total_tokens=1,044,096,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5439/500000] + train/ActionL1Loss=0.0707 + throughput/total_tokens=1,044,288,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5440/500000] + optim/total_grad_norm=5.696 + train/ActionL1Loss=0.0877 + throughput/total_tokens=1,044,480,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5441/500000] + train/ActionL1Loss=0.0635 + throughput/total_tokens=1,044,672,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5442/500000] + train/ActionL1Loss=0.0747 + throughput/total_tokens=1,044,864,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5443/500000] + train/ActionL1Loss=0.0833 + throughput/total_tokens=1,045,056,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5444/500000] + train/ActionL1Loss=0.0788 + throughput/total_tokens=1,045,248,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5445/500000] + train/ActionL1Loss=0.0710 + throughput/total_tokens=1,045,440,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5446/500000] + train/ActionL1Loss=0.0649 + throughput/total_tokens=1,045,632,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5447/500000] + train/ActionL1Loss=0.0716 + throughput/total_tokens=1,045,824,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5448/500000] + train/ActionL1Loss=0.0548 + throughput/total_tokens=1,046,016,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5449/500000] + train/ActionL1Loss=0.0759 + throughput/total_tokens=1,046,208,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5450/500000] + train/ActionL1Loss=0.0717 + throughput/total_tokens=1,046,400,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5451/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,046,592,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5452/500000] + train/ActionL1Loss=0.0698 + throughput/total_tokens=1,046,784,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5453/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=1,046,976,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5454/500000] + train/ActionL1Loss=0.0814 + throughput/total_tokens=1,047,168,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5455/500000] + train/ActionL1Loss=0.0714 + throughput/total_tokens=1,047,360,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5456/500000] + train/ActionL1Loss=0.0710 + throughput/total_tokens=1,047,552,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5457/500000] + train/ActionL1Loss=0.0777 + throughput/total_tokens=1,047,744,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5458/500000] + train/ActionL1Loss=0.0718 + throughput/total_tokens=1,047,936,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5459/500000] + train/ActionL1Loss=0.0624 + throughput/total_tokens=1,048,128,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5460/500000] + optim/total_grad_norm=5.067 + train/ActionL1Loss=0.0662 + throughput/total_tokens=1,048,320,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=5461/500000] + train/ActionL1Loss=0.0536 + throughput/total_tokens=1,048,512,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5462/500000] + train/ActionL1Loss=0.0623 + throughput/total_tokens=1,048,704,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5463/500000] + train/ActionL1Loss=0.0535 + throughput/total_tokens=1,048,896,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5464/500000] + train/ActionL1Loss=0.0623 + throughput/total_tokens=1,049,088,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5465/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=1,049,280,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5466/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=1,049,472,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5467/500000] + train/ActionL1Loss=0.0774 + throughput/total_tokens=1,049,664,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5468/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=1,049,856,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5469/500000] + train/ActionL1Loss=0.0672 + throughput/total_tokens=1,050,048,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5470/500000] + train/ActionL1Loss=0.0724 + throughput/total_tokens=1,050,240,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5471/500000] + train/ActionL1Loss=0.0805 + throughput/total_tokens=1,050,432,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5472/500000] + train/ActionL1Loss=0.0750 + throughput/total_tokens=1,050,624,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5473/500000] + train/ActionL1Loss=0.0727 + throughput/total_tokens=1,050,816,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5474/500000] + train/ActionL1Loss=0.0680 + throughput/total_tokens=1,051,008,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5475/500000] + train/ActionL1Loss=0.0706 + throughput/total_tokens=1,051,200,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5476/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,051,392,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5477/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=1,051,584,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5478/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=1,051,776,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5479/500000] + train/ActionL1Loss=0.0525 + throughput/total_tokens=1,051,968,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5480/500000] + optim/total_grad_norm=5.095 + train/ActionL1Loss=0.0826 + throughput/total_tokens=1,052,160,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5481/500000] + train/ActionL1Loss=0.0519 + throughput/total_tokens=1,052,352,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5482/500000] + train/ActionL1Loss=0.0733 + throughput/total_tokens=1,052,544,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5483/500000] + train/ActionL1Loss=0.0754 + throughput/total_tokens=1,052,736,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5484/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,052,928,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5485/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=1,053,120,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5486/500000] + train/ActionL1Loss=0.0675 + throughput/total_tokens=1,053,312,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5487/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=1,053,504,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5488/500000] + train/ActionL1Loss=0.0740 + throughput/total_tokens=1,053,696,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5489/500000] + train/ActionL1Loss=0.0891 + throughput/total_tokens=1,053,888,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5490/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=1,054,080,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5491/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,054,272,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5492/500000] + train/ActionL1Loss=0.0709 + throughput/total_tokens=1,054,464,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5493/500000] + train/ActionL1Loss=0.0565 + throughput/total_tokens=1,054,656,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5494/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=1,054,848,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5495/500000] + train/ActionL1Loss=0.0566 + throughput/total_tokens=1,055,040,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5496/500000] + train/ActionL1Loss=0.0779 + throughput/total_tokens=1,055,232,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5497/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,055,424,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5498/500000] + train/ActionL1Loss=0.0516 + throughput/total_tokens=1,055,616,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5499/500000] + train/ActionL1Loss=0.0640 + throughput/total_tokens=1,055,808,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5500/500000] + optim/total_grad_norm=5.667 + train/ActionL1Loss=0.0592 + throughput/total_tokens=1,056,000,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +10/01 [21:36:35] INFO | >> Saving config... checkpoint.py:608 +10/01 [21:37:26] INFO | >> Saving model state... checkpoint.py:796 +10/01 [21:38:38] INFO | >> Saving optim state... checkpoint.py:811 +10/01 [21:40:09] INFO | >> Saving trainer state... checkpoint.py:670 + INFO | >> Saving config... checkpoint.py:608 +[step=5501/500000] + train/ActionL1Loss=0.0830 + throughput/total_tokens=1,056,192,000 + throughput/device/tokens_per_second=1,208 + throughput/device/batches_per_second=0.0504 +[step=5502/500000] + train/ActionL1Loss=0.0814 + throughput/total_tokens=1,056,384,000 + throughput/device/tokens_per_second=1,181 + throughput/device/batches_per_second=0.0492 +[step=5503/500000] + train/ActionL1Loss=0.0681 + throughput/total_tokens=1,056,576,000 + throughput/device/tokens_per_second=1,176 + throughput/device/batches_per_second=0.0490 +[step=5504/500000] + train/ActionL1Loss=0.1006 + throughput/total_tokens=1,056,768,000 + throughput/device/tokens_per_second=1,173 + throughput/device/batches_per_second=0.0489 +[step=5505/500000] + train/ActionL1Loss=0.0908 + throughput/total_tokens=1,056,960,000 + throughput/device/tokens_per_second=1,172 + throughput/device/batches_per_second=0.0489 +[step=5506/500000] + train/ActionL1Loss=0.0655 + throughput/total_tokens=1,057,152,000 + throughput/device/tokens_per_second=1,170 + throughput/device/batches_per_second=0.0488 +[step=5507/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,057,344,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=5508/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=1,057,536,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=5509/500000] + train/ActionL1Loss=0.0669 + throughput/total_tokens=1,057,728,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5510/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=1,057,920,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=5511/500000] + train/ActionL1Loss=0.0853 + throughput/total_tokens=1,058,112,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5512/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=1,058,304,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5513/500000] + train/ActionL1Loss=0.0609 + throughput/total_tokens=1,058,496,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5514/500000] + train/ActionL1Loss=0.0729 + throughput/total_tokens=1,058,688,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5515/500000] + train/ActionL1Loss=0.0593 + throughput/total_tokens=1,058,880,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5516/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,059,072,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5517/500000] + train/ActionL1Loss=0.0826 + throughput/total_tokens=1,059,264,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5518/500000] + train/ActionL1Loss=0.0842 + throughput/total_tokens=1,059,456,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5519/500000] + train/ActionL1Loss=0.0895 + throughput/total_tokens=1,059,648,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5520/500000] + optim/total_grad_norm=5.789 + train/ActionL1Loss=0.0617 + throughput/total_tokens=1,059,840,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5521/500000] + train/ActionL1Loss=0.0689 + throughput/total_tokens=1,060,032,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5522/500000] + train/ActionL1Loss=0.0607 + throughput/total_tokens=1,060,224,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5523/500000] + train/ActionL1Loss=0.0794 + throughput/total_tokens=1,060,416,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5524/500000] + train/ActionL1Loss=0.0792 + throughput/total_tokens=1,060,608,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5525/500000] + train/ActionL1Loss=0.0794 + throughput/total_tokens=1,060,800,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5526/500000] + train/ActionL1Loss=0.0967 + throughput/total_tokens=1,060,992,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5527/500000] + train/ActionL1Loss=0.0741 + throughput/total_tokens=1,061,184,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5528/500000] + train/ActionL1Loss=0.0603 + throughput/total_tokens=1,061,376,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5529/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=1,061,568,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5530/500000] + train/ActionL1Loss=0.0691 + throughput/total_tokens=1,061,760,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5531/500000] + train/ActionL1Loss=0.0831 + throughput/total_tokens=1,061,952,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5532/500000] + train/ActionL1Loss=0.0792 + throughput/total_tokens=1,062,144,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5533/500000] + train/ActionL1Loss=0.1037 + throughput/total_tokens=1,062,336,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5534/500000] + train/ActionL1Loss=0.0676 + throughput/total_tokens=1,062,528,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5535/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=1,062,720,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5536/500000] + train/ActionL1Loss=0.0806 + throughput/total_tokens=1,062,912,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5537/500000] + train/ActionL1Loss=0.0753 + throughput/total_tokens=1,063,104,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5538/500000] + train/ActionL1Loss=0.0747 + throughput/total_tokens=1,063,296,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5539/500000] + train/ActionL1Loss=0.0819 + throughput/total_tokens=1,063,488,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5540/500000] + optim/total_grad_norm=6.441 + train/ActionL1Loss=0.0745 + throughput/total_tokens=1,063,680,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5541/500000] + train/ActionL1Loss=0.0584 + throughput/total_tokens=1,063,872,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5542/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=1,064,064,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5543/500000] + train/ActionL1Loss=0.0821 + throughput/total_tokens=1,064,256,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5544/500000] + train/ActionL1Loss=0.0733 + throughput/total_tokens=1,064,448,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5545/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=1,064,640,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5546/500000] + train/ActionL1Loss=0.0740 + throughput/total_tokens=1,064,832,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5547/500000] + train/ActionL1Loss=0.0570 + throughput/total_tokens=1,065,024,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5548/500000] + train/ActionL1Loss=0.0713 + throughput/total_tokens=1,065,216,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5549/500000] + train/ActionL1Loss=0.0547 + throughput/total_tokens=1,065,408,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5550/500000] + train/ActionL1Loss=0.0758 + throughput/total_tokens=1,065,600,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=5551/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,065,792,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5552/500000] + train/ActionL1Loss=0.0730 + throughput/total_tokens=1,065,984,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5553/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,066,176,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5554/500000] + train/ActionL1Loss=0.0924 + throughput/total_tokens=1,066,368,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5555/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=1,066,560,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5556/500000] + train/ActionL1Loss=0.0571 + throughput/total_tokens=1,066,752,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5557/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=1,066,944,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5558/500000] + train/ActionL1Loss=0.0806 + throughput/total_tokens=1,067,136,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5559/500000] + train/ActionL1Loss=0.0629 + throughput/total_tokens=1,067,328,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5560/500000] + optim/total_grad_norm=5.226 + train/ActionL1Loss=0.0863 + throughput/total_tokens=1,067,520,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5561/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,067,712,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5562/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,067,904,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5563/500000] + train/ActionL1Loss=0.0652 + throughput/total_tokens=1,068,096,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5564/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=1,068,288,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5565/500000] + train/ActionL1Loss=0.0773 + throughput/total_tokens=1,068,480,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5566/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=1,068,672,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5567/500000] + train/ActionL1Loss=0.0590 + throughput/total_tokens=1,068,864,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5568/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,069,056,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5569/500000] + train/ActionL1Loss=0.0828 + throughput/total_tokens=1,069,248,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5570/500000] + train/ActionL1Loss=0.0473 + throughput/total_tokens=1,069,440,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5571/500000] + train/ActionL1Loss=0.0536 + throughput/total_tokens=1,069,632,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5572/500000] + train/ActionL1Loss=0.0574 + throughput/total_tokens=1,069,824,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5573/500000] + train/ActionL1Loss=0.0907 + throughput/total_tokens=1,070,016,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5574/500000] + train/ActionL1Loss=0.0581 + throughput/total_tokens=1,070,208,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5575/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=1,070,400,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5576/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=1,070,592,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5577/500000] + train/ActionL1Loss=0.0827 + throughput/total_tokens=1,070,784,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5578/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=1,070,976,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5579/500000] + train/ActionL1Loss=0.0546 + throughput/total_tokens=1,071,168,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5580/500000] + optim/total_grad_norm=4.591 + train/ActionL1Loss=0.0458 + throughput/total_tokens=1,071,360,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5581/500000] + train/ActionL1Loss=0.0548 + throughput/total_tokens=1,071,552,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5582/500000] + train/ActionL1Loss=0.0777 + throughput/total_tokens=1,071,744,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5583/500000] + train/ActionL1Loss=0.0557 + throughput/total_tokens=1,071,936,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5584/500000] + train/ActionL1Loss=0.0795 + throughput/total_tokens=1,072,128,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5585/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,072,320,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5586/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=1,072,512,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5587/500000] + train/ActionL1Loss=0.0574 + throughput/total_tokens=1,072,704,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5588/500000] + train/ActionL1Loss=0.0542 + throughput/total_tokens=1,072,896,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5589/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,073,088,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5590/500000] + train/ActionL1Loss=0.0753 + throughput/total_tokens=1,073,280,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5591/500000] + train/ActionL1Loss=0.0926 + throughput/total_tokens=1,073,472,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5592/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=1,073,664,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5593/500000] + train/ActionL1Loss=0.0793 + throughput/total_tokens=1,073,856,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5594/500000] + train/ActionL1Loss=0.0707 + throughput/total_tokens=1,074,048,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5595/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=1,074,240,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5596/500000] + train/ActionL1Loss=0.0590 + throughput/total_tokens=1,074,432,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5597/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=1,074,624,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5598/500000] + train/ActionL1Loss=0.0812 + throughput/total_tokens=1,074,816,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5599/500000] + train/ActionL1Loss=0.0689 + throughput/total_tokens=1,075,008,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5600/500000] + optim/total_grad_norm=4.339 + train/ActionL1Loss=0.0899 + throughput/total_tokens=1,075,200,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5601/500000] + train/ActionL1Loss=0.0614 + throughput/total_tokens=1,075,392,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5602/500000] + train/ActionL1Loss=0.0961 + throughput/total_tokens=1,075,584,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5603/500000] + train/ActionL1Loss=0.0871 + throughput/total_tokens=1,075,776,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5604/500000] + train/ActionL1Loss=0.0782 + throughput/total_tokens=1,075,968,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5605/500000] + train/ActionL1Loss=0.0671 + throughput/total_tokens=1,076,160,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5606/500000] + train/ActionL1Loss=0.0610 + throughput/total_tokens=1,076,352,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5607/500000] + train/ActionL1Loss=0.0597 + throughput/total_tokens=1,076,544,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5608/500000] + train/ActionL1Loss=0.0753 + throughput/total_tokens=1,076,736,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5609/500000] + train/ActionL1Loss=0.0804 + throughput/total_tokens=1,076,928,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5610/500000] + train/ActionL1Loss=0.0675 + throughput/total_tokens=1,077,120,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5611/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,077,312,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5612/500000] + train/ActionL1Loss=0.0637 + throughput/total_tokens=1,077,504,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5613/500000] + train/ActionL1Loss=0.0577 + throughput/total_tokens=1,077,696,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5614/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=1,077,888,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5615/500000] + train/ActionL1Loss=0.0821 + throughput/total_tokens=1,078,080,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5616/500000] + train/ActionL1Loss=0.0641 + throughput/total_tokens=1,078,272,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5617/500000] + train/ActionL1Loss=0.0878 + throughput/total_tokens=1,078,464,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5618/500000] + train/ActionL1Loss=0.0755 + throughput/total_tokens=1,078,656,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5619/500000] + train/ActionL1Loss=0.0656 + throughput/total_tokens=1,078,848,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5620/500000] + optim/total_grad_norm=7.582 + train/ActionL1Loss=0.0685 + throughput/total_tokens=1,079,040,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=5621/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=1,079,232,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5622/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=1,079,424,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5623/500000] + train/ActionL1Loss=0.0624 + throughput/total_tokens=1,079,616,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5624/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,079,808,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5625/500000] + train/ActionL1Loss=0.0599 + throughput/total_tokens=1,080,000,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5626/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,080,192,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5627/500000] + train/ActionL1Loss=0.0748 + throughput/total_tokens=1,080,384,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5628/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=1,080,576,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5629/500000] + train/ActionL1Loss=0.0780 + throughput/total_tokens=1,080,768,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5630/500000] + train/ActionL1Loss=0.0585 + throughput/total_tokens=1,080,960,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=5631/500000] + train/ActionL1Loss=0.0599 + throughput/total_tokens=1,081,152,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=5632/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=1,081,344,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5633/500000] + train/ActionL1Loss=0.0608 + throughput/total_tokens=1,081,536,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5634/500000] + train/ActionL1Loss=0.0780 + throughput/total_tokens=1,081,728,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5635/500000] + train/ActionL1Loss=0.0717 + throughput/total_tokens=1,081,920,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5636/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=1,082,112,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5637/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=1,082,304,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5638/500000] + train/ActionL1Loss=0.0627 + throughput/total_tokens=1,082,496,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5639/500000] + train/ActionL1Loss=0.0468 + throughput/total_tokens=1,082,688,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5640/500000] + optim/total_grad_norm=4.931 + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,082,880,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5641/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,083,072,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5642/500000] + train/ActionL1Loss=0.0587 + throughput/total_tokens=1,083,264,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5643/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=1,083,456,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5644/500000] + train/ActionL1Loss=0.0800 + throughput/total_tokens=1,083,648,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5645/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=1,083,840,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5646/500000] + train/ActionL1Loss=0.0754 + throughput/total_tokens=1,084,032,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5647/500000] + train/ActionL1Loss=0.0529 + throughput/total_tokens=1,084,224,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5648/500000] + train/ActionL1Loss=0.0687 + throughput/total_tokens=1,084,416,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5649/500000] + train/ActionL1Loss=0.0585 + throughput/total_tokens=1,084,608,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5650/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=1,084,800,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5651/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=1,084,992,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5652/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=1,085,184,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5653/500000] + train/ActionL1Loss=0.0503 + throughput/total_tokens=1,085,376,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5654/500000] + train/ActionL1Loss=0.0629 + throughput/total_tokens=1,085,568,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5655/500000] + train/ActionL1Loss=0.0660 + throughput/total_tokens=1,085,760,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5656/500000] + train/ActionL1Loss=0.0724 + throughput/total_tokens=1,085,952,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5657/500000] + train/ActionL1Loss=0.0556 + throughput/total_tokens=1,086,144,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5658/500000] + train/ActionL1Loss=0.0820 + throughput/total_tokens=1,086,336,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5659/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=1,086,528,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5660/500000] + optim/total_grad_norm=7.552 + train/ActionL1Loss=0.0699 + throughput/total_tokens=1,086,720,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5661/500000] + train/ActionL1Loss=0.0526 + throughput/total_tokens=1,086,912,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5662/500000] + train/ActionL1Loss=0.0731 + throughput/total_tokens=1,087,104,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5663/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,087,296,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5664/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=1,087,488,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5665/500000] + train/ActionL1Loss=0.0623 + throughput/total_tokens=1,087,680,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5666/500000] + train/ActionL1Loss=0.0562 + throughput/total_tokens=1,087,872,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5667/500000] + train/ActionL1Loss=0.0493 + throughput/total_tokens=1,088,064,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5668/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=1,088,256,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5669/500000] + train/ActionL1Loss=0.0670 + throughput/total_tokens=1,088,448,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5670/500000] + train/ActionL1Loss=0.0697 + throughput/total_tokens=1,088,640,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5671/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=1,088,832,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5672/500000] + train/ActionL1Loss=0.0525 + throughput/total_tokens=1,089,024,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5673/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=1,089,216,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5674/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=1,089,408,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5675/500000] + train/ActionL1Loss=0.0697 + throughput/total_tokens=1,089,600,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5676/500000] + train/ActionL1Loss=0.0643 + throughput/total_tokens=1,089,792,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5677/500000] + train/ActionL1Loss=0.0699 + throughput/total_tokens=1,089,984,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5678/500000] + train/ActionL1Loss=0.0633 + throughput/total_tokens=1,090,176,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5679/500000] + train/ActionL1Loss=0.0680 + throughput/total_tokens=1,090,368,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5680/500000] + optim/total_grad_norm=7.175 + train/ActionL1Loss=0.0875 + throughput/total_tokens=1,090,560,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5681/500000] + train/ActionL1Loss=0.0541 + throughput/total_tokens=1,090,752,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5682/500000] + train/ActionL1Loss=0.0749 + throughput/total_tokens=1,090,944,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5683/500000] + train/ActionL1Loss=0.0620 + throughput/total_tokens=1,091,136,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5684/500000] + train/ActionL1Loss=0.0566 + throughput/total_tokens=1,091,328,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5685/500000] + train/ActionL1Loss=0.0621 + throughput/total_tokens=1,091,520,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5686/500000] + train/ActionL1Loss=0.0698 + throughput/total_tokens=1,091,712,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5687/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,091,904,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5688/500000] + train/ActionL1Loss=0.0583 + throughput/total_tokens=1,092,096,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5689/500000] + train/ActionL1Loss=0.0565 + throughput/total_tokens=1,092,288,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5690/500000] + train/ActionL1Loss=0.0518 + throughput/total_tokens=1,092,480,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5691/500000] + train/ActionL1Loss=0.0582 + throughput/total_tokens=1,092,672,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5692/500000] + train/ActionL1Loss=0.0650 + throughput/total_tokens=1,092,864,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5693/500000] + train/ActionL1Loss=0.0919 + throughput/total_tokens=1,093,056,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5694/500000] + train/ActionL1Loss=0.0830 + throughput/total_tokens=1,093,248,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5695/500000] + train/ActionL1Loss=0.0656 + throughput/total_tokens=1,093,440,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5696/500000] + train/ActionL1Loss=0.0626 + throughput/total_tokens=1,093,632,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5697/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=1,093,824,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5698/500000] + train/ActionL1Loss=0.0617 + throughput/total_tokens=1,094,016,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5699/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=1,094,208,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5700/500000] + optim/total_grad_norm=5.973 + train/ActionL1Loss=0.0675 + throughput/total_tokens=1,094,400,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5701/500000] + train/ActionL1Loss=0.0793 + throughput/total_tokens=1,094,592,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5702/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=1,094,784,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5703/500000] + train/ActionL1Loss=0.0736 + throughput/total_tokens=1,094,976,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5704/500000] + train/ActionL1Loss=0.0732 + throughput/total_tokens=1,095,168,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5705/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,095,360,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5706/500000] + train/ActionL1Loss=0.0889 + throughput/total_tokens=1,095,552,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5707/500000] + train/ActionL1Loss=0.0677 + throughput/total_tokens=1,095,744,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5708/500000] + train/ActionL1Loss=0.0649 + throughput/total_tokens=1,095,936,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5709/500000] + train/ActionL1Loss=0.0606 + throughput/total_tokens=1,096,128,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5710/500000] + train/ActionL1Loss=0.0751 + throughput/total_tokens=1,096,320,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5711/500000] + train/ActionL1Loss=0.0606 + throughput/total_tokens=1,096,512,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5712/500000] + train/ActionL1Loss=0.0606 + throughput/total_tokens=1,096,704,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5713/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=1,096,896,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5714/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,097,088,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5715/500000] + train/ActionL1Loss=0.0785 + throughput/total_tokens=1,097,280,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5716/500000] + train/ActionL1Loss=0.0676 + throughput/total_tokens=1,097,472,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5717/500000] + train/ActionL1Loss=0.0516 + throughput/total_tokens=1,097,664,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5718/500000] + train/ActionL1Loss=0.0681 + throughput/total_tokens=1,097,856,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5719/500000] + train/ActionL1Loss=0.0743 + throughput/total_tokens=1,098,048,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5720/500000] + optim/total_grad_norm=5.560 + train/ActionL1Loss=0.0590 + throughput/total_tokens=1,098,240,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5721/500000] + train/ActionL1Loss=0.0792 + throughput/total_tokens=1,098,432,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5722/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=1,098,624,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5723/500000] + train/ActionL1Loss=0.0620 + throughput/total_tokens=1,098,816,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5724/500000] + train/ActionL1Loss=0.0634 + throughput/total_tokens=1,099,008,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5725/500000] + train/ActionL1Loss=0.0969 + throughput/total_tokens=1,099,200,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5726/500000] + train/ActionL1Loss=0.0655 + throughput/total_tokens=1,099,392,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5727/500000] + train/ActionL1Loss=0.0740 + throughput/total_tokens=1,099,584,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5728/500000] + train/ActionL1Loss=0.0702 + throughput/total_tokens=1,099,776,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5729/500000] + train/ActionL1Loss=0.0855 + throughput/total_tokens=1,099,968,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5730/500000] + train/ActionL1Loss=0.0717 + throughput/total_tokens=1,100,160,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5731/500000] + train/ActionL1Loss=0.0765 + throughput/total_tokens=1,100,352,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5732/500000] + train/ActionL1Loss=0.0713 + throughput/total_tokens=1,100,544,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5733/500000] + train/ActionL1Loss=0.0680 + throughput/total_tokens=1,100,736,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5734/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=1,100,928,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5735/500000] + train/ActionL1Loss=0.0670 + throughput/total_tokens=1,101,120,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5736/500000] + train/ActionL1Loss=0.0856 + throughput/total_tokens=1,101,312,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5737/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=1,101,504,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5738/500000] + train/ActionL1Loss=0.0579 + throughput/total_tokens=1,101,696,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5739/500000] + train/ActionL1Loss=0.0804 + throughput/total_tokens=1,101,888,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5740/500000] + optim/total_grad_norm=7.220 + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,102,080,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5741/500000] + train/ActionL1Loss=0.0656 + throughput/total_tokens=1,102,272,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5742/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=1,102,464,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5743/500000] + train/ActionL1Loss=0.0594 + throughput/total_tokens=1,102,656,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5744/500000] + train/ActionL1Loss=0.0731 + throughput/total_tokens=1,102,848,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5745/500000] + train/ActionL1Loss=0.0773 + throughput/total_tokens=1,103,040,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5746/500000] + train/ActionL1Loss=0.0843 + throughput/total_tokens=1,103,232,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5747/500000] + train/ActionL1Loss=0.0592 + throughput/total_tokens=1,103,424,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5748/500000] + train/ActionL1Loss=0.0660 + throughput/total_tokens=1,103,616,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5749/500000] + train/ActionL1Loss=0.0627 + throughput/total_tokens=1,103,808,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5750/500000] + train/ActionL1Loss=0.0834 + throughput/total_tokens=1,104,000,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5751/500000] + train/ActionL1Loss=0.0611 + throughput/total_tokens=1,104,192,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5752/500000] + train/ActionL1Loss=0.0646 + throughput/total_tokens=1,104,384,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5753/500000] + train/ActionL1Loss=0.0610 + throughput/total_tokens=1,104,576,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5754/500000] + train/ActionL1Loss=0.0796 + throughput/total_tokens=1,104,768,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5755/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=1,104,960,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5756/500000] + train/ActionL1Loss=0.0723 + throughput/total_tokens=1,105,152,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5757/500000] + train/ActionL1Loss=0.0558 + throughput/total_tokens=1,105,344,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5758/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,105,536,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5759/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=1,105,728,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5760/500000] + optim/total_grad_norm=4.860 + train/ActionL1Loss=0.0632 + throughput/total_tokens=1,105,920,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5761/500000] + train/ActionL1Loss=0.0743 + throughput/total_tokens=1,106,112,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5762/500000] + train/ActionL1Loss=0.0596 + throughput/total_tokens=1,106,304,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5763/500000] + train/ActionL1Loss=0.0636 + throughput/total_tokens=1,106,496,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5764/500000] + train/ActionL1Loss=0.0557 + throughput/total_tokens=1,106,688,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5765/500000] + train/ActionL1Loss=0.0813 + throughput/total_tokens=1,106,880,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5766/500000] + train/ActionL1Loss=0.0621 + throughput/total_tokens=1,107,072,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5767/500000] + train/ActionL1Loss=0.1093 + throughput/total_tokens=1,107,264,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5768/500000] + train/ActionL1Loss=0.0738 + throughput/total_tokens=1,107,456,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5769/500000] + train/ActionL1Loss=0.0855 + throughput/total_tokens=1,107,648,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5770/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,107,840,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5771/500000] + train/ActionL1Loss=0.0601 + throughput/total_tokens=1,108,032,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5772/500000] + train/ActionL1Loss=0.0586 + throughput/total_tokens=1,108,224,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5773/500000] + train/ActionL1Loss=0.0738 + throughput/total_tokens=1,108,416,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5774/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=1,108,608,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5775/500000] + train/ActionL1Loss=0.0577 + throughput/total_tokens=1,108,800,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5776/500000] + train/ActionL1Loss=0.0627 + throughput/total_tokens=1,108,992,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5777/500000] + train/ActionL1Loss=0.0744 + throughput/total_tokens=1,109,184,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5778/500000] + train/ActionL1Loss=0.0806 + throughput/total_tokens=1,109,376,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5779/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=1,109,568,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5780/500000] + optim/total_grad_norm=5.805 + train/ActionL1Loss=0.0841 + throughput/total_tokens=1,109,760,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5781/500000] + train/ActionL1Loss=0.0775 + throughput/total_tokens=1,109,952,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5782/500000] + train/ActionL1Loss=0.0657 + throughput/total_tokens=1,110,144,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5783/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,110,336,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5784/500000] + train/ActionL1Loss=0.0621 + throughput/total_tokens=1,110,528,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5785/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,110,720,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5786/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=1,110,912,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5787/500000] + train/ActionL1Loss=0.0603 + throughput/total_tokens=1,111,104,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5788/500000] + train/ActionL1Loss=0.0740 + throughput/total_tokens=1,111,296,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5789/500000] + train/ActionL1Loss=0.0832 + throughput/total_tokens=1,111,488,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5790/500000] + train/ActionL1Loss=0.0864 + throughput/total_tokens=1,111,680,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5791/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,111,872,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5792/500000] + train/ActionL1Loss=0.0627 + throughput/total_tokens=1,112,064,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5793/500000] + train/ActionL1Loss=0.0920 + throughput/total_tokens=1,112,256,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5794/500000] + train/ActionL1Loss=0.0686 + throughput/total_tokens=1,112,448,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5795/500000] + train/ActionL1Loss=0.0633 + throughput/total_tokens=1,112,640,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5796/500000] + train/ActionL1Loss=0.0608 + throughput/total_tokens=1,112,832,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5797/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=1,113,024,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5798/500000] + train/ActionL1Loss=0.0839 + throughput/total_tokens=1,113,216,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5799/500000] + train/ActionL1Loss=0.0762 + throughput/total_tokens=1,113,408,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5800/500000] + optim/total_grad_norm=6.939 + train/ActionL1Loss=0.0722 + throughput/total_tokens=1,113,600,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5801/500000] + train/ActionL1Loss=0.0672 + throughput/total_tokens=1,113,792,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5802/500000] + train/ActionL1Loss=0.0509 + throughput/total_tokens=1,113,984,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5803/500000] + train/ActionL1Loss=0.0639 + throughput/total_tokens=1,114,176,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5804/500000] + train/ActionL1Loss=0.0969 + throughput/total_tokens=1,114,368,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5805/500000] + train/ActionL1Loss=0.0803 + throughput/total_tokens=1,114,560,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5806/500000] + train/ActionL1Loss=0.0680 + throughput/total_tokens=1,114,752,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5807/500000] + train/ActionL1Loss=0.0581 + throughput/total_tokens=1,114,944,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5808/500000] + train/ActionL1Loss=0.0587 + throughput/total_tokens=1,115,136,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5809/500000] + train/ActionL1Loss=0.0805 + throughput/total_tokens=1,115,328,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5810/500000] + train/ActionL1Loss=0.0821 + throughput/total_tokens=1,115,520,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=5811/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=1,115,712,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5812/500000] + train/ActionL1Loss=0.0732 + throughput/total_tokens=1,115,904,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5813/500000] + train/ActionL1Loss=0.0643 + throughput/total_tokens=1,116,096,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5814/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=1,116,288,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5815/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,116,480,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5816/500000] + train/ActionL1Loss=0.0861 + throughput/total_tokens=1,116,672,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5817/500000] + train/ActionL1Loss=0.0519 + throughput/total_tokens=1,116,864,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5818/500000] + train/ActionL1Loss=0.0743 + throughput/total_tokens=1,117,056,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5819/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=1,117,248,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5820/500000] + optim/total_grad_norm=6.581 + train/ActionL1Loss=0.0802 + throughput/total_tokens=1,117,440,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5821/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,117,632,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5822/500000] + train/ActionL1Loss=0.0543 + throughput/total_tokens=1,117,824,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5823/500000] + train/ActionL1Loss=0.0607 + throughput/total_tokens=1,118,016,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5824/500000] + train/ActionL1Loss=0.0743 + throughput/total_tokens=1,118,208,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5825/500000] + train/ActionL1Loss=0.0558 + throughput/total_tokens=1,118,400,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5826/500000] + train/ActionL1Loss=0.0488 + throughput/total_tokens=1,118,592,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5827/500000] + train/ActionL1Loss=0.0669 + throughput/total_tokens=1,118,784,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5828/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=1,118,976,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5829/500000] + train/ActionL1Loss=0.0848 + throughput/total_tokens=1,119,168,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5830/500000] + train/ActionL1Loss=0.0627 + throughput/total_tokens=1,119,360,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5831/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,119,552,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5832/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,119,744,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5833/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=1,119,936,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5834/500000] + train/ActionL1Loss=0.0606 + throughput/total_tokens=1,120,128,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5835/500000] + train/ActionL1Loss=0.0605 + throughput/total_tokens=1,120,320,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5836/500000] + train/ActionL1Loss=0.0721 + throughput/total_tokens=1,120,512,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5837/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,120,704,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5838/500000] + train/ActionL1Loss=0.0646 + throughput/total_tokens=1,120,896,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5839/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=1,121,088,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5840/500000] + optim/total_grad_norm=4.631 + train/ActionL1Loss=0.0592 + throughput/total_tokens=1,121,280,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5841/500000] + train/ActionL1Loss=0.0518 + throughput/total_tokens=1,121,472,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5842/500000] + train/ActionL1Loss=0.0773 + throughput/total_tokens=1,121,664,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5843/500000] + train/ActionL1Loss=0.0751 + throughput/total_tokens=1,121,856,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5844/500000] + train/ActionL1Loss=0.1020 + throughput/total_tokens=1,122,048,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5845/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,122,240,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5846/500000] + train/ActionL1Loss=0.0708 + throughput/total_tokens=1,122,432,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5847/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=1,122,624,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5848/500000] + train/ActionL1Loss=0.1008 + throughput/total_tokens=1,122,816,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5849/500000] + train/ActionL1Loss=0.0531 + throughput/total_tokens=1,123,008,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5850/500000] + train/ActionL1Loss=0.0626 + throughput/total_tokens=1,123,200,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5851/500000] + train/ActionL1Loss=0.0656 + throughput/total_tokens=1,123,392,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5852/500000] + train/ActionL1Loss=0.0710 + throughput/total_tokens=1,123,584,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5853/500000] + train/ActionL1Loss=0.0779 + throughput/total_tokens=1,123,776,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5854/500000] + train/ActionL1Loss=0.0691 + throughput/total_tokens=1,123,968,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5855/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=1,124,160,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5856/500000] + train/ActionL1Loss=0.0626 + throughput/total_tokens=1,124,352,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5857/500000] + train/ActionL1Loss=0.0730 + throughput/total_tokens=1,124,544,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5858/500000] + train/ActionL1Loss=0.0811 + throughput/total_tokens=1,124,736,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5859/500000] + train/ActionL1Loss=0.0762 + throughput/total_tokens=1,124,928,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5860/500000] + optim/total_grad_norm=6.363 + train/ActionL1Loss=0.0699 + throughput/total_tokens=1,125,120,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5861/500000] + train/ActionL1Loss=0.0857 + throughput/total_tokens=1,125,312,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5862/500000] + train/ActionL1Loss=0.0687 + throughput/total_tokens=1,125,504,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5863/500000] + train/ActionL1Loss=0.0704 + throughput/total_tokens=1,125,696,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5864/500000] + train/ActionL1Loss=0.0775 + throughput/total_tokens=1,125,888,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5865/500000] + train/ActionL1Loss=0.0829 + throughput/total_tokens=1,126,080,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5866/500000] + train/ActionL1Loss=0.0653 + throughput/total_tokens=1,126,272,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5867/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=1,126,464,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5868/500000] + train/ActionL1Loss=0.0804 + throughput/total_tokens=1,126,656,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5869/500000] + train/ActionL1Loss=0.0555 + throughput/total_tokens=1,126,848,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5870/500000] + train/ActionL1Loss=0.0838 + throughput/total_tokens=1,127,040,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5871/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=1,127,232,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5872/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,127,424,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5873/500000] + train/ActionL1Loss=0.0746 + throughput/total_tokens=1,127,616,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5874/500000] + train/ActionL1Loss=0.0727 + throughput/total_tokens=1,127,808,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5875/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=1,128,000,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5876/500000] + train/ActionL1Loss=0.0574 + throughput/total_tokens=1,128,192,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5877/500000] + train/ActionL1Loss=0.0561 + throughput/total_tokens=1,128,384,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5878/500000] + train/ActionL1Loss=0.0560 + throughput/total_tokens=1,128,576,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5879/500000] + train/ActionL1Loss=0.0568 + throughput/total_tokens=1,128,768,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5880/500000] + optim/total_grad_norm=4.804 + train/ActionL1Loss=0.0712 + throughput/total_tokens=1,128,960,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5881/500000] + train/ActionL1Loss=0.0689 + throughput/total_tokens=1,129,152,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5882/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=1,129,344,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5883/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=1,129,536,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5884/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=1,129,728,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5885/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=1,129,920,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5886/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=1,130,112,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5887/500000] + train/ActionL1Loss=0.0724 + throughput/total_tokens=1,130,304,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5888/500000] + train/ActionL1Loss=0.0620 + throughput/total_tokens=1,130,496,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5889/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=1,130,688,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5890/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=1,130,880,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5891/500000] + train/ActionL1Loss=0.0745 + throughput/total_tokens=1,131,072,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5892/500000] + train/ActionL1Loss=0.0586 + throughput/total_tokens=1,131,264,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5893/500000] + train/ActionL1Loss=0.0755 + throughput/total_tokens=1,131,456,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=5894/500000] + train/ActionL1Loss=0.0646 + throughput/total_tokens=1,131,648,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5895/500000] + train/ActionL1Loss=0.0883 + throughput/total_tokens=1,131,840,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5896/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=1,132,032,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5897/500000] + train/ActionL1Loss=0.0576 + throughput/total_tokens=1,132,224,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5898/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=1,132,416,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5899/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=1,132,608,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5900/500000] + optim/total_grad_norm=6.576 + train/ActionL1Loss=0.0529 + throughput/total_tokens=1,132,800,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5901/500000] + train/ActionL1Loss=0.0437 + throughput/total_tokens=1,132,992,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5902/500000] + train/ActionL1Loss=0.0731 + throughput/total_tokens=1,133,184,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5903/500000] + train/ActionL1Loss=0.0639 + throughput/total_tokens=1,133,376,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5904/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,133,568,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5905/500000] + train/ActionL1Loss=0.0633 + throughput/total_tokens=1,133,760,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5906/500000] + train/ActionL1Loss=0.0607 + throughput/total_tokens=1,133,952,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5907/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=1,134,144,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=5908/500000] + train/ActionL1Loss=0.0510 + throughput/total_tokens=1,134,336,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5909/500000] + train/ActionL1Loss=0.0732 + throughput/total_tokens=1,134,528,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=5910/500000] + train/ActionL1Loss=0.0755 + throughput/total_tokens=1,134,720,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5911/500000] + train/ActionL1Loss=0.0657 + throughput/total_tokens=1,134,912,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5912/500000] + train/ActionL1Loss=0.0769 + throughput/total_tokens=1,135,104,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5913/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=1,135,296,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5914/500000] + train/ActionL1Loss=0.0595 + throughput/total_tokens=1,135,488,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5915/500000] + train/ActionL1Loss=0.0595 + throughput/total_tokens=1,135,680,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5916/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=1,135,872,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=5917/500000] + train/ActionL1Loss=0.0774 + throughput/total_tokens=1,136,064,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5918/500000] + train/ActionL1Loss=0.0584 + throughput/total_tokens=1,136,256,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5919/500000] + train/ActionL1Loss=0.0755 + throughput/total_tokens=1,136,448,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5920/500000] + optim/total_grad_norm=5.800 + train/ActionL1Loss=0.0564 + throughput/total_tokens=1,136,640,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5921/500000] + train/ActionL1Loss=0.0640 + throughput/total_tokens=1,136,832,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5922/500000] + train/ActionL1Loss=0.0671 + throughput/total_tokens=1,137,024,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5923/500000] + train/ActionL1Loss=0.0729 + throughput/total_tokens=1,137,216,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5924/500000] + train/ActionL1Loss=0.0803 + throughput/total_tokens=1,137,408,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5925/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=1,137,600,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5926/500000] + train/ActionL1Loss=0.0813 + throughput/total_tokens=1,137,792,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5927/500000] + train/ActionL1Loss=0.0754 + throughput/total_tokens=1,137,984,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5928/500000] + train/ActionL1Loss=0.0617 + throughput/total_tokens=1,138,176,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5929/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=1,138,368,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5930/500000] + train/ActionL1Loss=0.0543 + throughput/total_tokens=1,138,560,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5931/500000] + train/ActionL1Loss=0.0740 + throughput/total_tokens=1,138,752,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5932/500000] + train/ActionL1Loss=0.0777 + throughput/total_tokens=1,138,944,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5933/500000] + train/ActionL1Loss=0.0526 + throughput/total_tokens=1,139,136,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5934/500000] + train/ActionL1Loss=0.0505 + throughput/total_tokens=1,139,328,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5935/500000] + train/ActionL1Loss=0.0578 + throughput/total_tokens=1,139,520,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5936/500000] + train/ActionL1Loss=0.0721 + throughput/total_tokens=1,139,712,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5937/500000] + train/ActionL1Loss=0.0584 + throughput/total_tokens=1,139,904,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5938/500000] + train/ActionL1Loss=0.0528 + throughput/total_tokens=1,140,096,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5939/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=1,140,288,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5940/500000] + optim/total_grad_norm=6.121 + train/ActionL1Loss=0.0715 + throughput/total_tokens=1,140,480,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5941/500000] + train/ActionL1Loss=0.0870 + throughput/total_tokens=1,140,672,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=5942/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=1,140,864,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5943/500000] + train/ActionL1Loss=0.0701 + throughput/total_tokens=1,141,056,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5944/500000] + train/ActionL1Loss=0.0666 + throughput/total_tokens=1,141,248,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5945/500000] + train/ActionL1Loss=0.0574 + throughput/total_tokens=1,141,440,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5946/500000] + train/ActionL1Loss=0.0557 + throughput/total_tokens=1,141,632,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5947/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=1,141,824,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5948/500000] + train/ActionL1Loss=0.0707 + throughput/total_tokens=1,142,016,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5949/500000] + train/ActionL1Loss=0.0813 + throughput/total_tokens=1,142,208,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5950/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=1,142,400,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5951/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,142,592,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5952/500000] + train/ActionL1Loss=0.0863 + throughput/total_tokens=1,142,784,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5953/500000] + train/ActionL1Loss=0.0796 + throughput/total_tokens=1,142,976,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5954/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,143,168,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5955/500000] + train/ActionL1Loss=0.0911 + throughput/total_tokens=1,143,360,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5956/500000] + train/ActionL1Loss=0.0744 + throughput/total_tokens=1,143,552,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5957/500000] + train/ActionL1Loss=0.0748 + throughput/total_tokens=1,143,744,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5958/500000] + train/ActionL1Loss=0.0799 + throughput/total_tokens=1,143,936,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5959/500000] + train/ActionL1Loss=0.0743 + throughput/total_tokens=1,144,128,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5960/500000] + optim/total_grad_norm=7.231 + train/ActionL1Loss=0.0690 + throughput/total_tokens=1,144,320,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=5961/500000] + train/ActionL1Loss=0.0686 + throughput/total_tokens=1,144,512,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5962/500000] + train/ActionL1Loss=0.0708 + throughput/total_tokens=1,144,704,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5963/500000] + train/ActionL1Loss=0.0728 + throughput/total_tokens=1,144,896,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5964/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=1,145,088,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5965/500000] + train/ActionL1Loss=0.0670 + throughput/total_tokens=1,145,280,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5966/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,145,472,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5967/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=1,145,664,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5968/500000] + train/ActionL1Loss=0.0801 + throughput/total_tokens=1,145,856,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=5969/500000] + train/ActionL1Loss=0.0492 + throughput/total_tokens=1,146,048,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=5970/500000] + train/ActionL1Loss=0.0773 + throughput/total_tokens=1,146,240,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5971/500000] + train/ActionL1Loss=0.0586 + throughput/total_tokens=1,146,432,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5972/500000] + train/ActionL1Loss=0.0485 + throughput/total_tokens=1,146,624,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5973/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=1,146,816,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5974/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=1,147,008,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5975/500000] + train/ActionL1Loss=0.0488 + throughput/total_tokens=1,147,200,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5976/500000] + train/ActionL1Loss=0.0648 + throughput/total_tokens=1,147,392,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5977/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,147,584,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5978/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=1,147,776,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5979/500000] + train/ActionL1Loss=0.0808 + throughput/total_tokens=1,147,968,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5980/500000] + optim/total_grad_norm=5.722 + train/ActionL1Loss=0.0652 + throughput/total_tokens=1,148,160,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=5981/500000] + train/ActionL1Loss=0.0656 + throughput/total_tokens=1,148,352,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5982/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=1,148,544,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5983/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=1,148,736,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=5984/500000] + train/ActionL1Loss=0.0708 + throughput/total_tokens=1,148,928,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5985/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=1,149,120,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5986/500000] + train/ActionL1Loss=0.0733 + throughput/total_tokens=1,149,312,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5987/500000] + train/ActionL1Loss=0.0721 + throughput/total_tokens=1,149,504,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5988/500000] + train/ActionL1Loss=0.0832 + throughput/total_tokens=1,149,696,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5989/500000] + train/ActionL1Loss=0.0883 + throughput/total_tokens=1,149,888,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=5990/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,150,080,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=5991/500000] + train/ActionL1Loss=0.0728 + throughput/total_tokens=1,150,272,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5992/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=1,150,464,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5993/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=1,150,656,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5994/500000] + train/ActionL1Loss=0.0531 + throughput/total_tokens=1,150,848,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5995/500000] + train/ActionL1Loss=0.0741 + throughput/total_tokens=1,151,040,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5996/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=1,151,232,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5997/500000] + train/ActionL1Loss=0.0643 + throughput/total_tokens=1,151,424,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5998/500000] + train/ActionL1Loss=0.0586 + throughput/total_tokens=1,151,616,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=5999/500000] + train/ActionL1Loss=0.0669 + throughput/total_tokens=1,151,808,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6000/500000] + optim/total_grad_norm=6.525 + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,152,000,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +10/02 [00:33:13] INFO | >> Saving config... checkpoint.py:608 +10/02 [00:34:03] INFO | >> Saving model state... checkpoint.py:796 +10/02 [00:35:10] INFO | >> Saving optim state... checkpoint.py:811 +10/02 [00:36:42] INFO | >> Saving trainer state... checkpoint.py:670 + INFO | >> Saving config... checkpoint.py:608 +[step=6001/500000] + train/ActionL1Loss=0.0676 + throughput/total_tokens=1,152,192,000 + throughput/device/tokens_per_second=1,208 + throughput/device/batches_per_second=0.0503 +[step=6002/500000] + train/ActionL1Loss=0.0774 + throughput/total_tokens=1,152,384,000 + throughput/device/tokens_per_second=1,174 + throughput/device/batches_per_second=0.0489 +[step=6003/500000] + train/ActionL1Loss=0.0483 + throughput/total_tokens=1,152,576,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=6004/500000] + train/ActionL1Loss=0.0522 + throughput/total_tokens=1,152,768,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=6005/500000] + train/ActionL1Loss=0.0816 + throughput/total_tokens=1,152,960,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=6006/500000] + train/ActionL1Loss=0.0828 + throughput/total_tokens=1,153,152,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=6007/500000] + train/ActionL1Loss=0.0684 + throughput/total_tokens=1,153,344,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=6008/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,153,536,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=6009/500000] + train/ActionL1Loss=0.0749 + throughput/total_tokens=1,153,728,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6010/500000] + train/ActionL1Loss=0.0776 + throughput/total_tokens=1,153,920,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6011/500000] + train/ActionL1Loss=0.0713 + throughput/total_tokens=1,154,112,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6012/500000] + train/ActionL1Loss=0.0822 + throughput/total_tokens=1,154,304,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6013/500000] + train/ActionL1Loss=0.0543 + throughput/total_tokens=1,154,496,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6014/500000] + train/ActionL1Loss=0.0734 + throughput/total_tokens=1,154,688,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6015/500000] + train/ActionL1Loss=0.0807 + throughput/total_tokens=1,154,880,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6016/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=1,155,072,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6017/500000] + train/ActionL1Loss=0.0633 + throughput/total_tokens=1,155,264,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6018/500000] + train/ActionL1Loss=0.0690 + throughput/total_tokens=1,155,456,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6019/500000] + train/ActionL1Loss=0.0586 + throughput/total_tokens=1,155,648,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6020/500000] + optim/total_grad_norm=8.293 + train/ActionL1Loss=0.0662 + throughput/total_tokens=1,155,840,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6021/500000] + train/ActionL1Loss=0.0641 + throughput/total_tokens=1,156,032,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6022/500000] + train/ActionL1Loss=0.0860 + throughput/total_tokens=1,156,224,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6023/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=1,156,416,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6024/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,156,608,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6025/500000] + train/ActionL1Loss=0.0672 + throughput/total_tokens=1,156,800,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6026/500000] + train/ActionL1Loss=0.0805 + throughput/total_tokens=1,156,992,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6027/500000] + train/ActionL1Loss=0.0740 + throughput/total_tokens=1,157,184,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6028/500000] + train/ActionL1Loss=0.0664 + throughput/total_tokens=1,157,376,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6029/500000] + train/ActionL1Loss=0.0623 + throughput/total_tokens=1,157,568,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6030/500000] + train/ActionL1Loss=0.0822 + throughput/total_tokens=1,157,760,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6031/500000] + train/ActionL1Loss=0.0573 + throughput/total_tokens=1,157,952,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6032/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=1,158,144,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6033/500000] + train/ActionL1Loss=0.0660 + throughput/total_tokens=1,158,336,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6034/500000] + train/ActionL1Loss=0.0587 + throughput/total_tokens=1,158,528,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6035/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=1,158,720,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6036/500000] + train/ActionL1Loss=0.0660 + throughput/total_tokens=1,158,912,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6037/500000] + train/ActionL1Loss=0.0803 + throughput/total_tokens=1,159,104,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6038/500000] + train/ActionL1Loss=0.0558 + throughput/total_tokens=1,159,296,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6039/500000] + train/ActionL1Loss=0.0504 + throughput/total_tokens=1,159,488,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6040/500000] + optim/total_grad_norm=4.972 + train/ActionL1Loss=0.0586 + throughput/total_tokens=1,159,680,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6041/500000] + train/ActionL1Loss=0.0522 + throughput/total_tokens=1,159,872,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6042/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=1,160,064,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6043/500000] + train/ActionL1Loss=0.0653 + throughput/total_tokens=1,160,256,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6044/500000] + train/ActionL1Loss=0.0702 + throughput/total_tokens=1,160,448,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6045/500000] + train/ActionL1Loss=0.0482 + throughput/total_tokens=1,160,640,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6046/500000] + train/ActionL1Loss=0.0601 + throughput/total_tokens=1,160,832,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6047/500000] + train/ActionL1Loss=0.0766 + throughput/total_tokens=1,161,024,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6048/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,161,216,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6049/500000] + train/ActionL1Loss=0.0731 + throughput/total_tokens=1,161,408,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6050/500000] + train/ActionL1Loss=0.0634 + throughput/total_tokens=1,161,600,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6051/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,161,792,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6052/500000] + train/ActionL1Loss=0.0566 + throughput/total_tokens=1,161,984,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6053/500000] + train/ActionL1Loss=0.0735 + throughput/total_tokens=1,162,176,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6054/500000] + train/ActionL1Loss=0.0492 + throughput/total_tokens=1,162,368,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6055/500000] + train/ActionL1Loss=0.0555 + throughput/total_tokens=1,162,560,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6056/500000] + train/ActionL1Loss=0.0811 + throughput/total_tokens=1,162,752,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6057/500000] + train/ActionL1Loss=0.0522 + throughput/total_tokens=1,162,944,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6058/500000] + train/ActionL1Loss=0.0567 + throughput/total_tokens=1,163,136,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6059/500000] + train/ActionL1Loss=0.0855 + throughput/total_tokens=1,163,328,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6060/500000] + optim/total_grad_norm=5.275 + train/ActionL1Loss=0.0599 + throughput/total_tokens=1,163,520,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6061/500000] + train/ActionL1Loss=0.0769 + throughput/total_tokens=1,163,712,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6062/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=1,163,904,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6063/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=1,164,096,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6064/500000] + train/ActionL1Loss=0.0674 + throughput/total_tokens=1,164,288,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6065/500000] + train/ActionL1Loss=0.0608 + throughput/total_tokens=1,164,480,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6066/500000] + train/ActionL1Loss=0.0724 + throughput/total_tokens=1,164,672,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6067/500000] + train/ActionL1Loss=0.0534 + throughput/total_tokens=1,164,864,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6068/500000] + train/ActionL1Loss=0.0640 + throughput/total_tokens=1,165,056,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6069/500000] + train/ActionL1Loss=0.0754 + throughput/total_tokens=1,165,248,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6070/500000] + train/ActionL1Loss=0.0641 + throughput/total_tokens=1,165,440,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6071/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,165,632,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6072/500000] + train/ActionL1Loss=0.0812 + throughput/total_tokens=1,165,824,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6073/500000] + train/ActionL1Loss=0.0607 + throughput/total_tokens=1,166,016,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6074/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,166,208,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6075/500000] + train/ActionL1Loss=0.0583 + throughput/total_tokens=1,166,400,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6076/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=1,166,592,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6077/500000] + train/ActionL1Loss=0.0747 + throughput/total_tokens=1,166,784,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6078/500000] + train/ActionL1Loss=0.0795 + throughput/total_tokens=1,166,976,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6079/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,167,168,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6080/500000] + optim/total_grad_norm=6.036 + train/ActionL1Loss=0.0591 + throughput/total_tokens=1,167,360,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6081/500000] + train/ActionL1Loss=0.0724 + throughput/total_tokens=1,167,552,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6082/500000] + train/ActionL1Loss=0.0507 + throughput/total_tokens=1,167,744,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6083/500000] + train/ActionL1Loss=0.0547 + throughput/total_tokens=1,167,936,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6084/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=1,168,128,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6085/500000] + train/ActionL1Loss=0.0599 + throughput/total_tokens=1,168,320,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6086/500000] + train/ActionL1Loss=0.0495 + throughput/total_tokens=1,168,512,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6087/500000] + train/ActionL1Loss=0.0646 + throughput/total_tokens=1,168,704,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6088/500000] + train/ActionL1Loss=0.0564 + throughput/total_tokens=1,168,896,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6089/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=1,169,088,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6090/500000] + train/ActionL1Loss=0.0580 + throughput/total_tokens=1,169,280,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6091/500000] + train/ActionL1Loss=0.0689 + throughput/total_tokens=1,169,472,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6092/500000] + train/ActionL1Loss=0.0480 + throughput/total_tokens=1,169,664,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6093/500000] + train/ActionL1Loss=0.0601 + throughput/total_tokens=1,169,856,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6094/500000] + train/ActionL1Loss=0.0578 + throughput/total_tokens=1,170,048,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6095/500000] + train/ActionL1Loss=0.0626 + throughput/total_tokens=1,170,240,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6096/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=1,170,432,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6097/500000] + train/ActionL1Loss=0.0677 + throughput/total_tokens=1,170,624,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6098/500000] + train/ActionL1Loss=0.0614 + throughput/total_tokens=1,170,816,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6099/500000] + train/ActionL1Loss=0.0831 + throughput/total_tokens=1,171,008,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6100/500000] + optim/total_grad_norm=6.548 + train/ActionL1Loss=0.0727 + throughput/total_tokens=1,171,200,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6101/500000] + train/ActionL1Loss=0.0508 + throughput/total_tokens=1,171,392,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6102/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=1,171,584,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6103/500000] + train/ActionL1Loss=0.0776 + throughput/total_tokens=1,171,776,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6104/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,171,968,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6105/500000] + train/ActionL1Loss=0.0583 + throughput/total_tokens=1,172,160,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6106/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,172,352,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6107/500000] + train/ActionL1Loss=0.0792 + throughput/total_tokens=1,172,544,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6108/500000] + train/ActionL1Loss=0.0722 + throughput/total_tokens=1,172,736,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6109/500000] + train/ActionL1Loss=0.0573 + throughput/total_tokens=1,172,928,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6110/500000] + train/ActionL1Loss=0.0630 + throughput/total_tokens=1,173,120,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6111/500000] + train/ActionL1Loss=0.0740 + throughput/total_tokens=1,173,312,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6112/500000] + train/ActionL1Loss=0.0726 + throughput/total_tokens=1,173,504,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6113/500000] + train/ActionL1Loss=0.0722 + throughput/total_tokens=1,173,696,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6114/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,173,888,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6115/500000] + train/ActionL1Loss=0.0921 + throughput/total_tokens=1,174,080,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6116/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=1,174,272,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6117/500000] + train/ActionL1Loss=0.0582 + throughput/total_tokens=1,174,464,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6118/500000] + train/ActionL1Loss=0.0555 + throughput/total_tokens=1,174,656,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6119/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=1,174,848,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6120/500000] + optim/total_grad_norm=4.858 + train/ActionL1Loss=0.0501 + throughput/total_tokens=1,175,040,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6121/500000] + train/ActionL1Loss=0.0655 + throughput/total_tokens=1,175,232,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6122/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=1,175,424,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6123/500000] + train/ActionL1Loss=0.0846 + throughput/total_tokens=1,175,616,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6124/500000] + train/ActionL1Loss=0.0662 + throughput/total_tokens=1,175,808,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6125/500000] + train/ActionL1Loss=0.0690 + throughput/total_tokens=1,176,000,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6126/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,176,192,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6127/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=1,176,384,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6128/500000] + train/ActionL1Loss=0.0942 + throughput/total_tokens=1,176,576,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6129/500000] + train/ActionL1Loss=0.0574 + throughput/total_tokens=1,176,768,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6130/500000] + train/ActionL1Loss=0.0555 + throughput/total_tokens=1,176,960,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6131/500000] + train/ActionL1Loss=0.0729 + throughput/total_tokens=1,177,152,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6132/500000] + train/ActionL1Loss=0.0527 + throughput/total_tokens=1,177,344,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6133/500000] + train/ActionL1Loss=0.0753 + throughput/total_tokens=1,177,536,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6134/500000] + train/ActionL1Loss=0.0706 + throughput/total_tokens=1,177,728,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6135/500000] + train/ActionL1Loss=0.0569 + throughput/total_tokens=1,177,920,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6136/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=1,178,112,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6137/500000] + train/ActionL1Loss=0.0626 + throughput/total_tokens=1,178,304,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6138/500000] + train/ActionL1Loss=0.0726 + throughput/total_tokens=1,178,496,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6139/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=1,178,688,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6140/500000] + optim/total_grad_norm=7.645 + train/ActionL1Loss=0.0686 + throughput/total_tokens=1,178,880,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6141/500000] + train/ActionL1Loss=0.0564 + throughput/total_tokens=1,179,072,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6142/500000] + train/ActionL1Loss=0.0593 + throughput/total_tokens=1,179,264,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6143/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,179,456,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6144/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,179,648,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6145/500000] + train/ActionL1Loss=0.0702 + throughput/total_tokens=1,179,840,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6146/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,180,032,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6147/500000] + train/ActionL1Loss=0.0733 + throughput/total_tokens=1,180,224,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6148/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=1,180,416,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6149/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,180,608,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6150/500000] + train/ActionL1Loss=0.0610 + throughput/total_tokens=1,180,800,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6151/500000] + train/ActionL1Loss=0.0528 + throughput/total_tokens=1,180,992,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6152/500000] + train/ActionL1Loss=0.0646 + throughput/total_tokens=1,181,184,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6153/500000] + train/ActionL1Loss=0.0536 + throughput/total_tokens=1,181,376,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6154/500000] + train/ActionL1Loss=0.0655 + throughput/total_tokens=1,181,568,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6155/500000] + train/ActionL1Loss=0.0656 + throughput/total_tokens=1,181,760,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6156/500000] + train/ActionL1Loss=0.0547 + throughput/total_tokens=1,181,952,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6157/500000] + train/ActionL1Loss=0.0677 + throughput/total_tokens=1,182,144,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6158/500000] + train/ActionL1Loss=0.0766 + throughput/total_tokens=1,182,336,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6159/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=1,182,528,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6160/500000] + optim/total_grad_norm=6.274 + train/ActionL1Loss=0.0723 + throughput/total_tokens=1,182,720,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6161/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,182,912,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6162/500000] + train/ActionL1Loss=0.0678 + throughput/total_tokens=1,183,104,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6163/500000] + train/ActionL1Loss=0.0611 + throughput/total_tokens=1,183,296,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6164/500000] + train/ActionL1Loss=0.0919 + throughput/total_tokens=1,183,488,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6165/500000] + train/ActionL1Loss=0.0721 + throughput/total_tokens=1,183,680,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6166/500000] + train/ActionL1Loss=0.0579 + throughput/total_tokens=1,183,872,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6167/500000] + train/ActionL1Loss=0.0791 + throughput/total_tokens=1,184,064,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6168/500000] + train/ActionL1Loss=0.0732 + throughput/total_tokens=1,184,256,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6169/500000] + train/ActionL1Loss=0.0576 + throughput/total_tokens=1,184,448,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6170/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=1,184,640,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6171/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=1,184,832,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6172/500000] + train/ActionL1Loss=0.0681 + throughput/total_tokens=1,185,024,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6173/500000] + train/ActionL1Loss=0.0626 + throughput/total_tokens=1,185,216,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6174/500000] + train/ActionL1Loss=0.0749 + throughput/total_tokens=1,185,408,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6175/500000] + train/ActionL1Loss=0.0806 + throughput/total_tokens=1,185,600,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6176/500000] + train/ActionL1Loss=0.0610 + throughput/total_tokens=1,185,792,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6177/500000] + train/ActionL1Loss=0.0894 + throughput/total_tokens=1,185,984,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6178/500000] + train/ActionL1Loss=0.0552 + throughput/total_tokens=1,186,176,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6179/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=1,186,368,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6180/500000] + optim/total_grad_norm=6.037 + train/ActionL1Loss=0.0566 + throughput/total_tokens=1,186,560,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6181/500000] + train/ActionL1Loss=0.0546 + throughput/total_tokens=1,186,752,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6182/500000] + train/ActionL1Loss=0.0549 + throughput/total_tokens=1,186,944,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6183/500000] + train/ActionL1Loss=0.0713 + throughput/total_tokens=1,187,136,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6184/500000] + train/ActionL1Loss=0.0617 + throughput/total_tokens=1,187,328,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6185/500000] + train/ActionL1Loss=0.0662 + throughput/total_tokens=1,187,520,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6186/500000] + train/ActionL1Loss=0.0592 + throughput/total_tokens=1,187,712,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6187/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=1,187,904,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6188/500000] + train/ActionL1Loss=0.0472 + throughput/total_tokens=1,188,096,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6189/500000] + train/ActionL1Loss=0.0555 + throughput/total_tokens=1,188,288,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6190/500000] + train/ActionL1Loss=0.0639 + throughput/total_tokens=1,188,480,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6191/500000] + train/ActionL1Loss=0.0676 + throughput/total_tokens=1,188,672,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6192/500000] + train/ActionL1Loss=0.0686 + throughput/total_tokens=1,188,864,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6193/500000] + train/ActionL1Loss=0.0724 + throughput/total_tokens=1,189,056,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6194/500000] + train/ActionL1Loss=0.0819 + throughput/total_tokens=1,189,248,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6195/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=1,189,440,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6196/500000] + train/ActionL1Loss=0.0660 + throughput/total_tokens=1,189,632,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6197/500000] + train/ActionL1Loss=0.0566 + throughput/total_tokens=1,189,824,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6198/500000] + train/ActionL1Loss=0.0634 + throughput/total_tokens=1,190,016,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6199/500000] + train/ActionL1Loss=0.0702 + throughput/total_tokens=1,190,208,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6200/500000] + optim/total_grad_norm=5.801 + train/ActionL1Loss=0.0550 + throughput/total_tokens=1,190,400,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6201/500000] + train/ActionL1Loss=0.0543 + throughput/total_tokens=1,190,592,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6202/500000] + train/ActionL1Loss=0.0499 + throughput/total_tokens=1,190,784,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6203/500000] + train/ActionL1Loss=0.0608 + throughput/total_tokens=1,190,976,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6204/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=1,191,168,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6205/500000] + train/ActionL1Loss=0.0601 + throughput/total_tokens=1,191,360,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6206/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,191,552,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6207/500000] + train/ActionL1Loss=0.0521 + throughput/total_tokens=1,191,744,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6208/500000] + train/ActionL1Loss=0.0520 + throughput/total_tokens=1,191,936,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6209/500000] + train/ActionL1Loss=0.0674 + throughput/total_tokens=1,192,128,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6210/500000] + train/ActionL1Loss=0.0873 + throughput/total_tokens=1,192,320,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6211/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,192,512,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6212/500000] + train/ActionL1Loss=0.0730 + throughput/total_tokens=1,192,704,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6213/500000] + train/ActionL1Loss=0.0605 + throughput/total_tokens=1,192,896,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6214/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=1,193,088,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6215/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=1,193,280,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6216/500000] + train/ActionL1Loss=0.0671 + throughput/total_tokens=1,193,472,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6217/500000] + train/ActionL1Loss=0.0541 + throughput/total_tokens=1,193,664,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6218/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=1,193,856,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6219/500000] + train/ActionL1Loss=0.0560 + throughput/total_tokens=1,194,048,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6220/500000] + optim/total_grad_norm=8.126 + train/ActionL1Loss=0.0589 + throughput/total_tokens=1,194,240,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6221/500000] + train/ActionL1Loss=0.0761 + throughput/total_tokens=1,194,432,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6222/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=1,194,624,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6223/500000] + train/ActionL1Loss=0.0775 + throughput/total_tokens=1,194,816,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6224/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,195,008,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6225/500000] + train/ActionL1Loss=0.0758 + throughput/total_tokens=1,195,200,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6226/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=1,195,392,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6227/500000] + train/ActionL1Loss=0.0558 + throughput/total_tokens=1,195,584,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6228/500000] + train/ActionL1Loss=0.0892 + throughput/total_tokens=1,195,776,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6229/500000] + train/ActionL1Loss=0.0785 + throughput/total_tokens=1,195,968,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6230/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,196,160,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6231/500000] + train/ActionL1Loss=0.0639 + throughput/total_tokens=1,196,352,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6232/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,196,544,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6233/500000] + train/ActionL1Loss=0.0552 + throughput/total_tokens=1,196,736,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6234/500000] + train/ActionL1Loss=0.0539 + throughput/total_tokens=1,196,928,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6235/500000] + train/ActionL1Loss=0.0786 + throughput/total_tokens=1,197,120,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6236/500000] + train/ActionL1Loss=0.0718 + throughput/total_tokens=1,197,312,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6237/500000] + train/ActionL1Loss=0.0607 + throughput/total_tokens=1,197,504,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6238/500000] + train/ActionL1Loss=0.0684 + throughput/total_tokens=1,197,696,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6239/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,197,888,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6240/500000] + optim/total_grad_norm=5.164 + train/ActionL1Loss=0.0684 + throughput/total_tokens=1,198,080,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6241/500000] + train/ActionL1Loss=0.0530 + throughput/total_tokens=1,198,272,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6242/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=1,198,464,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6243/500000] + train/ActionL1Loss=0.0710 + throughput/total_tokens=1,198,656,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6244/500000] + train/ActionL1Loss=0.0593 + throughput/total_tokens=1,198,848,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6245/500000] + train/ActionL1Loss=0.0484 + throughput/total_tokens=1,199,040,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6246/500000] + train/ActionL1Loss=0.0641 + throughput/total_tokens=1,199,232,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6247/500000] + train/ActionL1Loss=0.0556 + throughput/total_tokens=1,199,424,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6248/500000] + train/ActionL1Loss=0.0508 + throughput/total_tokens=1,199,616,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6249/500000] + train/ActionL1Loss=0.0633 + throughput/total_tokens=1,199,808,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6250/500000] + train/ActionL1Loss=0.0527 + throughput/total_tokens=1,200,000,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6251/500000] + train/ActionL1Loss=0.0590 + throughput/total_tokens=1,200,192,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6252/500000] + train/ActionL1Loss=0.0443 + throughput/total_tokens=1,200,384,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6253/500000] + train/ActionL1Loss=0.0691 + throughput/total_tokens=1,200,576,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6254/500000] + train/ActionL1Loss=0.0727 + throughput/total_tokens=1,200,768,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6255/500000] + train/ActionL1Loss=0.0608 + throughput/total_tokens=1,200,960,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6256/500000] + train/ActionL1Loss=0.0818 + throughput/total_tokens=1,201,152,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6257/500000] + train/ActionL1Loss=0.0751 + throughput/total_tokens=1,201,344,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6258/500000] + train/ActionL1Loss=0.0611 + throughput/total_tokens=1,201,536,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6259/500000] + train/ActionL1Loss=0.0576 + throughput/total_tokens=1,201,728,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6260/500000] + optim/total_grad_norm=6.735 + train/ActionL1Loss=0.0520 + throughput/total_tokens=1,201,920,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6261/500000] + train/ActionL1Loss=0.0683 + throughput/total_tokens=1,202,112,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6262/500000] + train/ActionL1Loss=0.0660 + throughput/total_tokens=1,202,304,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6263/500000] + train/ActionL1Loss=0.0690 + throughput/total_tokens=1,202,496,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6264/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=1,202,688,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6265/500000] + train/ActionL1Loss=0.0585 + throughput/total_tokens=1,202,880,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6266/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,203,072,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6267/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,203,264,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6268/500000] + train/ActionL1Loss=0.0607 + throughput/total_tokens=1,203,456,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6269/500000] + train/ActionL1Loss=0.0858 + throughput/total_tokens=1,203,648,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6270/500000] + train/ActionL1Loss=0.0528 + throughput/total_tokens=1,203,840,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6271/500000] + train/ActionL1Loss=0.0637 + throughput/total_tokens=1,204,032,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6272/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=1,204,224,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6273/500000] + train/ActionL1Loss=0.0585 + throughput/total_tokens=1,204,416,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6274/500000] + train/ActionL1Loss=0.0534 + throughput/total_tokens=1,204,608,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6275/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=1,204,800,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6276/500000] + train/ActionL1Loss=0.0416 + throughput/total_tokens=1,204,992,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6277/500000] + train/ActionL1Loss=0.0626 + throughput/total_tokens=1,205,184,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6278/500000] + train/ActionL1Loss=0.0508 + throughput/total_tokens=1,205,376,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6279/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=1,205,568,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6280/500000] + optim/total_grad_norm=6.073 + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,205,760,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6281/500000] + train/ActionL1Loss=0.0666 + throughput/total_tokens=1,205,952,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6282/500000] + train/ActionL1Loss=0.0718 + throughput/total_tokens=1,206,144,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6283/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=1,206,336,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6284/500000] + train/ActionL1Loss=0.0698 + throughput/total_tokens=1,206,528,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6285/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,206,720,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6286/500000] + train/ActionL1Loss=0.0757 + throughput/total_tokens=1,206,912,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6287/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=1,207,104,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6288/500000] + train/ActionL1Loss=0.0580 + throughput/total_tokens=1,207,296,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6289/500000] + train/ActionL1Loss=0.0683 + throughput/total_tokens=1,207,488,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6290/500000] + train/ActionL1Loss=0.0820 + throughput/total_tokens=1,207,680,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6291/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=1,207,872,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6292/500000] + train/ActionL1Loss=0.0731 + throughput/total_tokens=1,208,064,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6293/500000] + train/ActionL1Loss=0.0639 + throughput/total_tokens=1,208,256,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6294/500000] + train/ActionL1Loss=0.0658 + throughput/total_tokens=1,208,448,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6295/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=1,208,640,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6296/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=1,208,832,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6297/500000] + train/ActionL1Loss=0.0771 + throughput/total_tokens=1,209,024,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6298/500000] + train/ActionL1Loss=0.0730 + throughput/total_tokens=1,209,216,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6299/500000] + train/ActionL1Loss=0.0784 + throughput/total_tokens=1,209,408,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6300/500000] + optim/total_grad_norm=6.784 + train/ActionL1Loss=0.0741 + throughput/total_tokens=1,209,600,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6301/500000] + train/ActionL1Loss=0.0720 + throughput/total_tokens=1,209,792,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6302/500000] + train/ActionL1Loss=0.0608 + throughput/total_tokens=1,209,984,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=6303/500000] + train/ActionL1Loss=0.0634 + throughput/total_tokens=1,210,176,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=6304/500000] + train/ActionL1Loss=0.0722 + throughput/total_tokens=1,210,368,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6305/500000] + train/ActionL1Loss=0.0673 + throughput/total_tokens=1,210,560,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6306/500000] + train/ActionL1Loss=0.0734 + throughput/total_tokens=1,210,752,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6307/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=1,210,944,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6308/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,211,136,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6309/500000] + train/ActionL1Loss=0.1023 + throughput/total_tokens=1,211,328,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6310/500000] + train/ActionL1Loss=0.0904 + throughput/total_tokens=1,211,520,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=6311/500000] + train/ActionL1Loss=0.0653 + throughput/total_tokens=1,211,712,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6312/500000] + train/ActionL1Loss=0.0741 + throughput/total_tokens=1,211,904,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6313/500000] + train/ActionL1Loss=0.0545 + throughput/total_tokens=1,212,096,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=6314/500000] + train/ActionL1Loss=0.0609 + throughput/total_tokens=1,212,288,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=6315/500000] + train/ActionL1Loss=0.0629 + throughput/total_tokens=1,212,480,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=6316/500000] + train/ActionL1Loss=0.0641 + throughput/total_tokens=1,212,672,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=6317/500000] + train/ActionL1Loss=0.0684 + throughput/total_tokens=1,212,864,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6318/500000] + train/ActionL1Loss=0.0726 + throughput/total_tokens=1,213,056,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6319/500000] + train/ActionL1Loss=0.0820 + throughput/total_tokens=1,213,248,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6320/500000] + optim/total_grad_norm=7.189 + train/ActionL1Loss=0.0550 + throughput/total_tokens=1,213,440,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6321/500000] + train/ActionL1Loss=0.0641 + throughput/total_tokens=1,213,632,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6322/500000] + train/ActionL1Loss=0.0593 + throughput/total_tokens=1,213,824,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6323/500000] + train/ActionL1Loss=0.0518 + throughput/total_tokens=1,214,016,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6324/500000] + train/ActionL1Loss=0.0534 + throughput/total_tokens=1,214,208,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6325/500000] + train/ActionL1Loss=0.0551 + throughput/total_tokens=1,214,400,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6326/500000] + train/ActionL1Loss=0.0707 + throughput/total_tokens=1,214,592,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6327/500000] + train/ActionL1Loss=0.0587 + throughput/total_tokens=1,214,784,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6328/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=1,214,976,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6329/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=1,215,168,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6330/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=1,215,360,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=6331/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,215,552,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6332/500000] + train/ActionL1Loss=0.0601 + throughput/total_tokens=1,215,744,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6333/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=1,215,936,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6334/500000] + train/ActionL1Loss=0.0478 + throughput/total_tokens=1,216,128,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6335/500000] + train/ActionL1Loss=0.0610 + throughput/total_tokens=1,216,320,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6336/500000] + train/ActionL1Loss=0.0813 + throughput/total_tokens=1,216,512,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6337/500000] + train/ActionL1Loss=0.0520 + throughput/total_tokens=1,216,704,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6338/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,216,896,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6339/500000] + train/ActionL1Loss=0.0830 + throughput/total_tokens=1,217,088,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6340/500000] + optim/total_grad_norm=6.386 + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,217,280,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6341/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=1,217,472,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6342/500000] + train/ActionL1Loss=0.0535 + throughput/total_tokens=1,217,664,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6343/500000] + train/ActionL1Loss=0.0617 + throughput/total_tokens=1,217,856,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6344/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,218,048,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6345/500000] + train/ActionL1Loss=0.0949 + throughput/total_tokens=1,218,240,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6346/500000] + train/ActionL1Loss=0.0883 + throughput/total_tokens=1,218,432,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6347/500000] + train/ActionL1Loss=0.0556 + throughput/total_tokens=1,218,624,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6348/500000] + train/ActionL1Loss=0.0499 + throughput/total_tokens=1,218,816,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6349/500000] + train/ActionL1Loss=0.0627 + throughput/total_tokens=1,219,008,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6350/500000] + train/ActionL1Loss=0.0596 + throughput/total_tokens=1,219,200,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6351/500000] + train/ActionL1Loss=0.0634 + throughput/total_tokens=1,219,392,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6352/500000] + train/ActionL1Loss=0.0614 + throughput/total_tokens=1,219,584,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6353/500000] + train/ActionL1Loss=0.0703 + throughput/total_tokens=1,219,776,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6354/500000] + train/ActionL1Loss=0.0539 + throughput/total_tokens=1,219,968,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6355/500000] + train/ActionL1Loss=0.0512 + throughput/total_tokens=1,220,160,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6356/500000] + train/ActionL1Loss=0.0564 + throughput/total_tokens=1,220,352,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6357/500000] + train/ActionL1Loss=0.0703 + throughput/total_tokens=1,220,544,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6358/500000] + train/ActionL1Loss=0.0450 + throughput/total_tokens=1,220,736,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6359/500000] + train/ActionL1Loss=0.0598 + throughput/total_tokens=1,220,928,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6360/500000] + optim/total_grad_norm=5.073 + train/ActionL1Loss=0.0735 + throughput/total_tokens=1,221,120,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6361/500000] + train/ActionL1Loss=0.0559 + throughput/total_tokens=1,221,312,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6362/500000] + train/ActionL1Loss=0.0677 + throughput/total_tokens=1,221,504,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6363/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=1,221,696,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6364/500000] + train/ActionL1Loss=0.0676 + throughput/total_tokens=1,221,888,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6365/500000] + train/ActionL1Loss=0.0517 + throughput/total_tokens=1,222,080,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6366/500000] + train/ActionL1Loss=0.0519 + throughput/total_tokens=1,222,272,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6367/500000] + train/ActionL1Loss=0.0530 + throughput/total_tokens=1,222,464,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6368/500000] + train/ActionL1Loss=0.0616 + throughput/total_tokens=1,222,656,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6369/500000] + train/ActionL1Loss=0.0607 + throughput/total_tokens=1,222,848,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6370/500000] + train/ActionL1Loss=0.0771 + throughput/total_tokens=1,223,040,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6371/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=1,223,232,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6372/500000] + train/ActionL1Loss=0.0528 + throughput/total_tokens=1,223,424,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6373/500000] + train/ActionL1Loss=0.0478 + throughput/total_tokens=1,223,616,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6374/500000] + train/ActionL1Loss=0.0858 + throughput/total_tokens=1,223,808,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6375/500000] + train/ActionL1Loss=0.0454 + throughput/total_tokens=1,224,000,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6376/500000] + train/ActionL1Loss=0.0562 + throughput/total_tokens=1,224,192,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6377/500000] + train/ActionL1Loss=0.0574 + throughput/total_tokens=1,224,384,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6378/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=1,224,576,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6379/500000] + train/ActionL1Loss=0.0518 + throughput/total_tokens=1,224,768,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6380/500000] + optim/total_grad_norm=6.973 + train/ActionL1Loss=0.0576 + throughput/total_tokens=1,224,960,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=6381/500000] + train/ActionL1Loss=0.0652 + throughput/total_tokens=1,225,152,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6382/500000] + train/ActionL1Loss=0.0537 + throughput/total_tokens=1,225,344,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6383/500000] + train/ActionL1Loss=0.0556 + throughput/total_tokens=1,225,536,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6384/500000] + train/ActionL1Loss=0.0485 + throughput/total_tokens=1,225,728,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6385/500000] + train/ActionL1Loss=0.0697 + throughput/total_tokens=1,225,920,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6386/500000] + train/ActionL1Loss=0.0597 + throughput/total_tokens=1,226,112,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6387/500000] + train/ActionL1Loss=0.0690 + throughput/total_tokens=1,226,304,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6388/500000] + train/ActionL1Loss=0.0630 + throughput/total_tokens=1,226,496,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6389/500000] + train/ActionL1Loss=0.0780 + throughput/total_tokens=1,226,688,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6390/500000] + train/ActionL1Loss=0.0684 + throughput/total_tokens=1,226,880,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=6391/500000] + train/ActionL1Loss=0.0476 + throughput/total_tokens=1,227,072,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6392/500000] + train/ActionL1Loss=0.0721 + throughput/total_tokens=1,227,264,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6393/500000] + train/ActionL1Loss=0.0577 + throughput/total_tokens=1,227,456,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6394/500000] + train/ActionL1Loss=0.0512 + throughput/total_tokens=1,227,648,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6395/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,227,840,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6396/500000] + train/ActionL1Loss=0.0748 + throughput/total_tokens=1,228,032,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6397/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,228,224,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6398/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=1,228,416,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6399/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=1,228,608,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6400/500000] + optim/total_grad_norm=6.286 + train/ActionL1Loss=0.1102 + throughput/total_tokens=1,228,800,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6401/500000] + train/ActionL1Loss=0.0675 + throughput/total_tokens=1,228,992,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6402/500000] + train/ActionL1Loss=0.0584 + throughput/total_tokens=1,229,184,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6403/500000] + train/ActionL1Loss=0.0794 + throughput/total_tokens=1,229,376,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6404/500000] + train/ActionL1Loss=0.0614 + throughput/total_tokens=1,229,568,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6405/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=1,229,760,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6406/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=1,229,952,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6407/500000] + train/ActionL1Loss=0.0505 + throughput/total_tokens=1,230,144,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6408/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=1,230,336,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6409/500000] + train/ActionL1Loss=0.0634 + throughput/total_tokens=1,230,528,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6410/500000] + train/ActionL1Loss=0.0692 + throughput/total_tokens=1,230,720,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6411/500000] + train/ActionL1Loss=0.0623 + throughput/total_tokens=1,230,912,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6412/500000] + train/ActionL1Loss=0.0573 + throughput/total_tokens=1,231,104,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6413/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=1,231,296,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6414/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=1,231,488,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6415/500000] + train/ActionL1Loss=0.0758 + throughput/total_tokens=1,231,680,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6416/500000] + train/ActionL1Loss=0.0703 + throughput/total_tokens=1,231,872,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6417/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=1,232,064,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6418/500000] + train/ActionL1Loss=0.0587 + throughput/total_tokens=1,232,256,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6419/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,232,448,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6420/500000] + optim/total_grad_norm=5.885 + train/ActionL1Loss=0.0706 + throughput/total_tokens=1,232,640,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6421/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=1,232,832,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6422/500000] + train/ActionL1Loss=0.0819 + throughput/total_tokens=1,233,024,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6423/500000] + train/ActionL1Loss=0.0655 + throughput/total_tokens=1,233,216,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6424/500000] + train/ActionL1Loss=0.0584 + throughput/total_tokens=1,233,408,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6425/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=1,233,600,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6426/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=1,233,792,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6427/500000] + train/ActionL1Loss=0.0558 + throughput/total_tokens=1,233,984,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6428/500000] + train/ActionL1Loss=0.0861 + throughput/total_tokens=1,234,176,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6429/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,234,368,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6430/500000] + train/ActionL1Loss=0.0620 + throughput/total_tokens=1,234,560,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=6431/500000] + train/ActionL1Loss=0.0515 + throughput/total_tokens=1,234,752,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6432/500000] + train/ActionL1Loss=0.0780 + throughput/total_tokens=1,234,944,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6433/500000] + train/ActionL1Loss=0.0584 + throughput/total_tokens=1,235,136,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6434/500000] + train/ActionL1Loss=0.0516 + throughput/total_tokens=1,235,328,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6435/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=1,235,520,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6436/500000] + train/ActionL1Loss=0.0501 + throughput/total_tokens=1,235,712,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6437/500000] + train/ActionL1Loss=0.0467 + throughput/total_tokens=1,235,904,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6438/500000] + train/ActionL1Loss=0.0839 + throughput/total_tokens=1,236,096,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6439/500000] + train/ActionL1Loss=0.0808 + throughput/total_tokens=1,236,288,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6440/500000] + optim/total_grad_norm=5.990 + train/ActionL1Loss=0.0786 + throughput/total_tokens=1,236,480,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=6441/500000] + train/ActionL1Loss=0.0487 + throughput/total_tokens=1,236,672,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6442/500000] + train/ActionL1Loss=0.0699 + throughput/total_tokens=1,236,864,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6443/500000] + train/ActionL1Loss=0.0637 + throughput/total_tokens=1,237,056,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6444/500000] + train/ActionL1Loss=0.0687 + throughput/total_tokens=1,237,248,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6445/500000] + train/ActionL1Loss=0.0540 + throughput/total_tokens=1,237,440,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6446/500000] + train/ActionL1Loss=0.0593 + throughput/total_tokens=1,237,632,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6447/500000] + train/ActionL1Loss=0.0630 + throughput/total_tokens=1,237,824,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6448/500000] + train/ActionL1Loss=0.0562 + throughput/total_tokens=1,238,016,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6449/500000] + train/ActionL1Loss=0.0504 + throughput/total_tokens=1,238,208,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6450/500000] + train/ActionL1Loss=0.0550 + throughput/total_tokens=1,238,400,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6451/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=1,238,592,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6452/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=1,238,784,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6453/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,238,976,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6454/500000] + train/ActionL1Loss=0.0564 + throughput/total_tokens=1,239,168,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6455/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=1,239,360,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6456/500000] + train/ActionL1Loss=0.0598 + throughput/total_tokens=1,239,552,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6457/500000] + train/ActionL1Loss=0.0585 + throughput/total_tokens=1,239,744,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6458/500000] + train/ActionL1Loss=0.0439 + throughput/total_tokens=1,239,936,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6459/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,240,128,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6460/500000] + optim/total_grad_norm=7.166 + train/ActionL1Loss=0.0901 + throughput/total_tokens=1,240,320,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6461/500000] + train/ActionL1Loss=0.0656 + throughput/total_tokens=1,240,512,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6462/500000] + train/ActionL1Loss=0.0710 + throughput/total_tokens=1,240,704,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6463/500000] + train/ActionL1Loss=0.0686 + throughput/total_tokens=1,240,896,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6464/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,241,088,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6465/500000] + train/ActionL1Loss=0.0662 + throughput/total_tokens=1,241,280,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6466/500000] + train/ActionL1Loss=0.0578 + throughput/total_tokens=1,241,472,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6467/500000] + train/ActionL1Loss=0.0673 + throughput/total_tokens=1,241,664,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6468/500000] + train/ActionL1Loss=0.0811 + throughput/total_tokens=1,241,856,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6469/500000] + train/ActionL1Loss=0.0528 + throughput/total_tokens=1,242,048,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6470/500000] + train/ActionL1Loss=0.0510 + throughput/total_tokens=1,242,240,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6471/500000] + train/ActionL1Loss=0.0740 + throughput/total_tokens=1,242,432,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6472/500000] + train/ActionL1Loss=0.0648 + throughput/total_tokens=1,242,624,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6473/500000] + train/ActionL1Loss=0.0678 + throughput/total_tokens=1,242,816,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6474/500000] + train/ActionL1Loss=0.0702 + throughput/total_tokens=1,243,008,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6475/500000] + train/ActionL1Loss=0.0630 + throughput/total_tokens=1,243,200,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6476/500000] + train/ActionL1Loss=0.0655 + throughput/total_tokens=1,243,392,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6477/500000] + train/ActionL1Loss=0.0771 + throughput/total_tokens=1,243,584,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6478/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,243,776,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6479/500000] + train/ActionL1Loss=0.0840 + throughput/total_tokens=1,243,968,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6480/500000] + optim/total_grad_norm=6.535 + train/ActionL1Loss=0.0690 + throughput/total_tokens=1,244,160,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=6481/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,244,352,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6482/500000] + train/ActionL1Loss=0.0666 + throughput/total_tokens=1,244,544,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6483/500000] + train/ActionL1Loss=0.0518 + throughput/total_tokens=1,244,736,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6484/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=1,244,928,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6485/500000] + train/ActionL1Loss=0.0591 + throughput/total_tokens=1,245,120,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6486/500000] + train/ActionL1Loss=0.0660 + throughput/total_tokens=1,245,312,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6487/500000] + train/ActionL1Loss=0.0802 + throughput/total_tokens=1,245,504,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6488/500000] + train/ActionL1Loss=0.0630 + throughput/total_tokens=1,245,696,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6489/500000] + train/ActionL1Loss=0.0438 + throughput/total_tokens=1,245,888,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6490/500000] + train/ActionL1Loss=0.0593 + throughput/total_tokens=1,246,080,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=6491/500000] + train/ActionL1Loss=0.0706 + throughput/total_tokens=1,246,272,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6492/500000] + train/ActionL1Loss=0.0910 + throughput/total_tokens=1,246,464,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6493/500000] + train/ActionL1Loss=0.0797 + throughput/total_tokens=1,246,656,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6494/500000] + train/ActionL1Loss=0.0670 + throughput/total_tokens=1,246,848,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6495/500000] + train/ActionL1Loss=0.0611 + throughput/total_tokens=1,247,040,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6496/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=1,247,232,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6497/500000] + train/ActionL1Loss=0.0835 + throughput/total_tokens=1,247,424,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6498/500000] + train/ActionL1Loss=0.0686 + throughput/total_tokens=1,247,616,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6499/500000] + train/ActionL1Loss=0.0772 + throughput/total_tokens=1,247,808,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6500/500000] + optim/total_grad_norm=9.072 + train/ActionL1Loss=0.0601 + throughput/total_tokens=1,248,000,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +10/02 [03:29:54] INFO | >> Saving config... checkpoint.py:608 +10/02 [03:30:22] INFO | >> Saving model state... checkpoint.py:796 +10/02 [03:31:34] INFO | >> Saving optim state... checkpoint.py:811 +10/02 [03:33:08] INFO | >> Saving trainer state... checkpoint.py:670 + INFO | >> Saving config... checkpoint.py:608 +[step=6501/500000] + train/ActionL1Loss=0.0564 + throughput/total_tokens=1,248,192,000 + throughput/device/tokens_per_second=1,209 + throughput/device/batches_per_second=0.0504 +[step=6502/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=1,248,384,000 + throughput/device/tokens_per_second=1,185 + throughput/device/batches_per_second=0.0494 +[step=6503/500000] + train/ActionL1Loss=0.0470 + throughput/total_tokens=1,248,576,000 + throughput/device/tokens_per_second=1,179 + throughput/device/batches_per_second=0.0492 +[step=6504/500000] + train/ActionL1Loss=0.0714 + throughput/total_tokens=1,248,768,000 + throughput/device/tokens_per_second=1,176 + throughput/device/batches_per_second=0.0490 +[step=6505/500000] + train/ActionL1Loss=0.0480 + throughput/total_tokens=1,248,960,000 + throughput/device/tokens_per_second=1,174 + throughput/device/batches_per_second=0.0489 +[step=6506/500000] + train/ActionL1Loss=0.0561 + throughput/total_tokens=1,249,152,000 + throughput/device/tokens_per_second=1,172 + throughput/device/batches_per_second=0.0489 +[step=6507/500000] + train/ActionL1Loss=0.0633 + throughput/total_tokens=1,249,344,000 + throughput/device/tokens_per_second=1,171 + throughput/device/batches_per_second=0.0488 +[step=6508/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=1,249,536,000 + throughput/device/tokens_per_second=1,170 + throughput/device/batches_per_second=0.0488 +[step=6509/500000] + train/ActionL1Loss=0.0850 + throughput/total_tokens=1,249,728,000 + throughput/device/tokens_per_second=1,170 + throughput/device/batches_per_second=0.0488 +[step=6510/500000] + train/ActionL1Loss=0.0509 + throughput/total_tokens=1,249,920,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=6511/500000] + train/ActionL1Loss=0.0649 + throughput/total_tokens=1,250,112,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=6512/500000] + train/ActionL1Loss=0.0722 + throughput/total_tokens=1,250,304,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6513/500000] + train/ActionL1Loss=0.0579 + throughput/total_tokens=1,250,496,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6514/500000] + train/ActionL1Loss=0.0605 + throughput/total_tokens=1,250,688,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6515/500000] + train/ActionL1Loss=0.0658 + throughput/total_tokens=1,250,880,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6516/500000] + train/ActionL1Loss=0.0554 + throughput/total_tokens=1,251,072,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6517/500000] + train/ActionL1Loss=0.0732 + throughput/total_tokens=1,251,264,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6518/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,251,456,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6519/500000] + train/ActionL1Loss=0.0709 + throughput/total_tokens=1,251,648,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6520/500000] + optim/total_grad_norm=7.341 + train/ActionL1Loss=0.0531 + throughput/total_tokens=1,251,840,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6521/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=1,252,032,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6522/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,252,224,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6523/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=1,252,416,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6524/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=1,252,608,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6525/500000] + train/ActionL1Loss=0.0728 + throughput/total_tokens=1,252,800,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6526/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=1,252,992,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6527/500000] + train/ActionL1Loss=0.0754 + throughput/total_tokens=1,253,184,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6528/500000] + train/ActionL1Loss=0.0549 + throughput/total_tokens=1,253,376,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6529/500000] + train/ActionL1Loss=0.0592 + throughput/total_tokens=1,253,568,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6530/500000] + train/ActionL1Loss=0.0500 + throughput/total_tokens=1,253,760,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6531/500000] + train/ActionL1Loss=0.0518 + throughput/total_tokens=1,253,952,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6532/500000] + train/ActionL1Loss=0.0551 + throughput/total_tokens=1,254,144,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6533/500000] + train/ActionL1Loss=0.0670 + throughput/total_tokens=1,254,336,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6534/500000] + train/ActionL1Loss=0.0541 + throughput/total_tokens=1,254,528,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6535/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=1,254,720,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6536/500000] + train/ActionL1Loss=0.0827 + throughput/total_tokens=1,254,912,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6537/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=1,255,104,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6538/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,255,296,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6539/500000] + train/ActionL1Loss=0.0558 + throughput/total_tokens=1,255,488,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6540/500000] + optim/total_grad_norm=5.022 + train/ActionL1Loss=0.0612 + throughput/total_tokens=1,255,680,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6541/500000] + train/ActionL1Loss=0.0516 + throughput/total_tokens=1,255,872,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6542/500000] + train/ActionL1Loss=0.0629 + throughput/total_tokens=1,256,064,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6543/500000] + train/ActionL1Loss=0.0633 + throughput/total_tokens=1,256,256,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6544/500000] + train/ActionL1Loss=0.0499 + throughput/total_tokens=1,256,448,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6545/500000] + train/ActionL1Loss=0.0543 + throughput/total_tokens=1,256,640,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6546/500000] + train/ActionL1Loss=0.0686 + throughput/total_tokens=1,256,832,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6547/500000] + train/ActionL1Loss=0.0630 + throughput/total_tokens=1,257,024,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6548/500000] + train/ActionL1Loss=0.0603 + throughput/total_tokens=1,257,216,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6549/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=1,257,408,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6550/500000] + train/ActionL1Loss=0.0796 + throughput/total_tokens=1,257,600,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6551/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=1,257,792,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6552/500000] + train/ActionL1Loss=0.0483 + throughput/total_tokens=1,257,984,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6553/500000] + train/ActionL1Loss=0.0739 + throughput/total_tokens=1,258,176,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6554/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=1,258,368,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6555/500000] + train/ActionL1Loss=0.0569 + throughput/total_tokens=1,258,560,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6556/500000] + train/ActionL1Loss=0.0713 + throughput/total_tokens=1,258,752,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6557/500000] + train/ActionL1Loss=0.0543 + throughput/total_tokens=1,258,944,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6558/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=1,259,136,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6559/500000] + train/ActionL1Loss=0.0886 + throughput/total_tokens=1,259,328,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6560/500000] + optim/total_grad_norm=6.346 + train/ActionL1Loss=0.0742 + throughput/total_tokens=1,259,520,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6561/500000] + train/ActionL1Loss=0.0541 + throughput/total_tokens=1,259,712,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6562/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,259,904,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6563/500000] + train/ActionL1Loss=0.0590 + throughput/total_tokens=1,260,096,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6564/500000] + train/ActionL1Loss=0.0847 + throughput/total_tokens=1,260,288,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6565/500000] + train/ActionL1Loss=0.0733 + throughput/total_tokens=1,260,480,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6566/500000] + train/ActionL1Loss=0.0875 + throughput/total_tokens=1,260,672,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6567/500000] + train/ActionL1Loss=0.0594 + throughput/total_tokens=1,260,864,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6568/500000] + train/ActionL1Loss=0.0585 + throughput/total_tokens=1,261,056,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6569/500000] + train/ActionL1Loss=0.0649 + throughput/total_tokens=1,261,248,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6570/500000] + train/ActionL1Loss=0.0440 + throughput/total_tokens=1,261,440,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6571/500000] + train/ActionL1Loss=0.0609 + throughput/total_tokens=1,261,632,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6572/500000] + train/ActionL1Loss=0.0842 + throughput/total_tokens=1,261,824,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6573/500000] + train/ActionL1Loss=0.0646 + throughput/total_tokens=1,262,016,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6574/500000] + train/ActionL1Loss=0.0937 + throughput/total_tokens=1,262,208,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6575/500000] + train/ActionL1Loss=0.0582 + throughput/total_tokens=1,262,400,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6576/500000] + train/ActionL1Loss=0.0578 + throughput/total_tokens=1,262,592,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6577/500000] + train/ActionL1Loss=0.0643 + throughput/total_tokens=1,262,784,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6578/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=1,262,976,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6579/500000] + train/ActionL1Loss=0.0534 + throughput/total_tokens=1,263,168,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6580/500000] + optim/total_grad_norm=6.072 + train/ActionL1Loss=0.0692 + throughput/total_tokens=1,263,360,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6581/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,263,552,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6582/500000] + train/ActionL1Loss=0.0794 + throughput/total_tokens=1,263,744,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6583/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=1,263,936,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6584/500000] + train/ActionL1Loss=0.0571 + throughput/total_tokens=1,264,128,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6585/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=1,264,320,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6586/500000] + train/ActionL1Loss=0.0568 + throughput/total_tokens=1,264,512,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6587/500000] + train/ActionL1Loss=0.0676 + throughput/total_tokens=1,264,704,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6588/500000] + train/ActionL1Loss=0.0611 + throughput/total_tokens=1,264,896,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6589/500000] + train/ActionL1Loss=0.0489 + throughput/total_tokens=1,265,088,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6590/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=1,265,280,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6591/500000] + train/ActionL1Loss=0.0450 + throughput/total_tokens=1,265,472,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6592/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=1,265,664,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6593/500000] + train/ActionL1Loss=0.0533 + throughput/total_tokens=1,265,856,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6594/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=1,266,048,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6595/500000] + train/ActionL1Loss=0.0729 + throughput/total_tokens=1,266,240,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6596/500000] + train/ActionL1Loss=0.0568 + throughput/total_tokens=1,266,432,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6597/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,266,624,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6598/500000] + train/ActionL1Loss=0.0541 + throughput/total_tokens=1,266,816,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6599/500000] + train/ActionL1Loss=0.0750 + throughput/total_tokens=1,267,008,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6600/500000] + optim/total_grad_norm=5.483 + train/ActionL1Loss=0.0717 + throughput/total_tokens=1,267,200,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6601/500000] + train/ActionL1Loss=0.0752 + throughput/total_tokens=1,267,392,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6602/500000] + train/ActionL1Loss=0.0652 + throughput/total_tokens=1,267,584,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6603/500000] + train/ActionL1Loss=0.0597 + throughput/total_tokens=1,267,776,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6604/500000] + train/ActionL1Loss=0.0610 + throughput/total_tokens=1,267,968,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6605/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,268,160,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6606/500000] + train/ActionL1Loss=0.0658 + throughput/total_tokens=1,268,352,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6607/500000] + train/ActionL1Loss=0.0488 + throughput/total_tokens=1,268,544,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6608/500000] + train/ActionL1Loss=0.0924 + throughput/total_tokens=1,268,736,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6609/500000] + train/ActionL1Loss=0.0614 + throughput/total_tokens=1,268,928,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6610/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,269,120,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6611/500000] + train/ActionL1Loss=0.0512 + throughput/total_tokens=1,269,312,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6612/500000] + train/ActionL1Loss=0.0561 + throughput/total_tokens=1,269,504,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6613/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=1,269,696,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6614/500000] + train/ActionL1Loss=0.0674 + throughput/total_tokens=1,269,888,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6615/500000] + train/ActionL1Loss=0.0937 + throughput/total_tokens=1,270,080,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6616/500000] + train/ActionL1Loss=0.0771 + throughput/total_tokens=1,270,272,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6617/500000] + train/ActionL1Loss=0.0656 + throughput/total_tokens=1,270,464,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6618/500000] + train/ActionL1Loss=0.0635 + throughput/total_tokens=1,270,656,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6619/500000] + train/ActionL1Loss=0.0497 + throughput/total_tokens=1,270,848,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6620/500000] + optim/total_grad_norm=4.335 + train/ActionL1Loss=0.0600 + throughput/total_tokens=1,271,040,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6621/500000] + train/ActionL1Loss=0.0707 + throughput/total_tokens=1,271,232,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6622/500000] + train/ActionL1Loss=0.0637 + throughput/total_tokens=1,271,424,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6623/500000] + train/ActionL1Loss=0.0594 + throughput/total_tokens=1,271,616,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6624/500000] + train/ActionL1Loss=0.0559 + throughput/total_tokens=1,271,808,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6625/500000] + train/ActionL1Loss=0.0716 + throughput/total_tokens=1,272,000,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6626/500000] + train/ActionL1Loss=0.0806 + throughput/total_tokens=1,272,192,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6627/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,272,384,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6628/500000] + train/ActionL1Loss=0.0681 + throughput/total_tokens=1,272,576,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6629/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,272,768,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6630/500000] + train/ActionL1Loss=0.0821 + throughput/total_tokens=1,272,960,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6631/500000] + train/ActionL1Loss=0.0676 + throughput/total_tokens=1,273,152,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6632/500000] + train/ActionL1Loss=0.0750 + throughput/total_tokens=1,273,344,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6633/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,273,536,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6634/500000] + train/ActionL1Loss=0.0580 + throughput/total_tokens=1,273,728,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6635/500000] + train/ActionL1Loss=0.0561 + throughput/total_tokens=1,273,920,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6636/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=1,274,112,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6637/500000] + train/ActionL1Loss=0.0553 + throughput/total_tokens=1,274,304,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6638/500000] + train/ActionL1Loss=0.0626 + throughput/total_tokens=1,274,496,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6639/500000] + train/ActionL1Loss=0.0748 + throughput/total_tokens=1,274,688,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6640/500000] + optim/total_grad_norm=6.942 + train/ActionL1Loss=0.0466 + throughput/total_tokens=1,274,880,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6641/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,275,072,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6642/500000] + train/ActionL1Loss=0.0837 + throughput/total_tokens=1,275,264,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6643/500000] + train/ActionL1Loss=0.0595 + throughput/total_tokens=1,275,456,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6644/500000] + train/ActionL1Loss=0.0544 + throughput/total_tokens=1,275,648,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6645/500000] + train/ActionL1Loss=0.0486 + throughput/total_tokens=1,275,840,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6646/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=1,276,032,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6647/500000] + train/ActionL1Loss=0.0984 + throughput/total_tokens=1,276,224,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6648/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=1,276,416,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6649/500000] + train/ActionL1Loss=0.0678 + throughput/total_tokens=1,276,608,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6650/500000] + train/ActionL1Loss=0.0592 + throughput/total_tokens=1,276,800,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6651/500000] + train/ActionL1Loss=0.0491 + throughput/total_tokens=1,276,992,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6652/500000] + train/ActionL1Loss=0.0633 + throughput/total_tokens=1,277,184,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6653/500000] + train/ActionL1Loss=0.0760 + throughput/total_tokens=1,277,376,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6654/500000] + train/ActionL1Loss=0.0607 + throughput/total_tokens=1,277,568,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6655/500000] + train/ActionL1Loss=0.0578 + throughput/total_tokens=1,277,760,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6656/500000] + train/ActionL1Loss=0.0493 + throughput/total_tokens=1,277,952,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6657/500000] + train/ActionL1Loss=0.0538 + throughput/total_tokens=1,278,144,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6658/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=1,278,336,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6659/500000] + train/ActionL1Loss=0.0741 + throughput/total_tokens=1,278,528,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6660/500000] + optim/total_grad_norm=8.431 + train/ActionL1Loss=0.0617 + throughput/total_tokens=1,278,720,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6661/500000] + train/ActionL1Loss=0.0662 + throughput/total_tokens=1,278,912,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6662/500000] + train/ActionL1Loss=0.0717 + throughput/total_tokens=1,279,104,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6663/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=1,279,296,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6664/500000] + train/ActionL1Loss=0.0497 + throughput/total_tokens=1,279,488,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6665/500000] + train/ActionL1Loss=0.0748 + throughput/total_tokens=1,279,680,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6666/500000] + train/ActionL1Loss=0.0518 + throughput/total_tokens=1,279,872,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6667/500000] + train/ActionL1Loss=0.0617 + throughput/total_tokens=1,280,064,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6668/500000] + train/ActionL1Loss=0.0626 + throughput/total_tokens=1,280,256,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6669/500000] + train/ActionL1Loss=0.0583 + throughput/total_tokens=1,280,448,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=6670/500000] + train/ActionL1Loss=0.0670 + throughput/total_tokens=1,280,640,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=6671/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,280,832,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6672/500000] + train/ActionL1Loss=0.0673 + throughput/total_tokens=1,281,024,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6673/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=1,281,216,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6674/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,281,408,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6675/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=1,281,600,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6676/500000] + train/ActionL1Loss=0.0753 + throughput/total_tokens=1,281,792,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6677/500000] + train/ActionL1Loss=0.0451 + throughput/total_tokens=1,281,984,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6678/500000] + train/ActionL1Loss=0.0658 + throughput/total_tokens=1,282,176,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6679/500000] + train/ActionL1Loss=0.0621 + throughput/total_tokens=1,282,368,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6680/500000] + optim/total_grad_norm=5.009 + train/ActionL1Loss=0.0749 + throughput/total_tokens=1,282,560,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=6681/500000] + train/ActionL1Loss=0.0523 + throughput/total_tokens=1,282,752,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6682/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=1,282,944,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6683/500000] + train/ActionL1Loss=0.0603 + throughput/total_tokens=1,283,136,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6684/500000] + train/ActionL1Loss=0.0561 + throughput/total_tokens=1,283,328,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6685/500000] + train/ActionL1Loss=0.0486 + throughput/total_tokens=1,283,520,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6686/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,283,712,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6687/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=1,283,904,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6688/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=1,284,096,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6689/500000] + train/ActionL1Loss=0.0583 + throughput/total_tokens=1,284,288,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6690/500000] + train/ActionL1Loss=0.0556 + throughput/total_tokens=1,284,480,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=6691/500000] + train/ActionL1Loss=0.0488 + throughput/total_tokens=1,284,672,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6692/500000] + train/ActionL1Loss=0.0532 + throughput/total_tokens=1,284,864,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6693/500000] + train/ActionL1Loss=0.0520 + throughput/total_tokens=1,285,056,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6694/500000] + train/ActionL1Loss=0.0641 + throughput/total_tokens=1,285,248,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6695/500000] + train/ActionL1Loss=0.0637 + throughput/total_tokens=1,285,440,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6696/500000] + train/ActionL1Loss=0.0740 + throughput/total_tokens=1,285,632,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6697/500000] + train/ActionL1Loss=0.0810 + throughput/total_tokens=1,285,824,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6698/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=1,286,016,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6699/500000] + train/ActionL1Loss=0.0733 + throughput/total_tokens=1,286,208,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6700/500000] + optim/total_grad_norm=5.388 + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,286,400,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6701/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=1,286,592,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=6702/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=1,286,784,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=6703/500000] + train/ActionL1Loss=0.0653 + throughput/total_tokens=1,286,976,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=6704/500000] + train/ActionL1Loss=0.0569 + throughput/total_tokens=1,287,168,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=6705/500000] + train/ActionL1Loss=0.0509 + throughput/total_tokens=1,287,360,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=6706/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=1,287,552,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=6707/500000] + train/ActionL1Loss=0.0869 + throughput/total_tokens=1,287,744,000 + throughput/device/tokens_per_second=1,150 + throughput/device/batches_per_second=0.0480 +[step=6708/500000] + train/ActionL1Loss=0.0650 + throughput/total_tokens=1,287,936,000 + throughput/device/tokens_per_second=1,150 + throughput/device/batches_per_second=0.0479 +[step=6709/500000] + train/ActionL1Loss=0.0498 + throughput/total_tokens=1,288,128,000 + throughput/device/tokens_per_second=1,150 + throughput/device/batches_per_second=0.0479 +[step=6710/500000] + train/ActionL1Loss=0.0578 + throughput/total_tokens=1,288,320,000 + throughput/device/tokens_per_second=1,150 + throughput/device/batches_per_second=0.0479 + System/Peak GPU Memory (MB)=46,917 +[step=6711/500000] + train/ActionL1Loss=0.0559 + throughput/total_tokens=1,288,512,000 + throughput/device/tokens_per_second=1,150 + throughput/device/batches_per_second=0.0479 +[step=6712/500000] + train/ActionL1Loss=0.0593 + throughput/total_tokens=1,288,704,000 + throughput/device/tokens_per_second=1,150 + throughput/device/batches_per_second=0.0479 +[step=6713/500000] + train/ActionL1Loss=0.0627 + throughput/total_tokens=1,288,896,000 + throughput/device/tokens_per_second=1,150 + throughput/device/batches_per_second=0.0480 +[step=6714/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=1,289,088,000 + throughput/device/tokens_per_second=1,150 + throughput/device/batches_per_second=0.0480 +[step=6715/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,289,280,000 + throughput/device/tokens_per_second=1,150 + throughput/device/batches_per_second=0.0480 +[step=6716/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=1,289,472,000 + throughput/device/tokens_per_second=1,150 + throughput/device/batches_per_second=0.0480 +[step=6717/500000] + train/ActionL1Loss=0.0760 + throughput/total_tokens=1,289,664,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=6718/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=1,289,856,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=6719/500000] + train/ActionL1Loss=0.0562 + throughput/total_tokens=1,290,048,000 + throughput/device/tokens_per_second=1,151 + throughput/device/batches_per_second=0.0480 +[step=6720/500000] + optim/total_grad_norm=5.734 + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,290,240,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6721/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=1,290,432,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6722/500000] + train/ActionL1Loss=0.0653 + throughput/total_tokens=1,290,624,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6723/500000] + train/ActionL1Loss=0.0653 + throughput/total_tokens=1,290,816,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6724/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=1,291,008,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6725/500000] + train/ActionL1Loss=0.0702 + throughput/total_tokens=1,291,200,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6726/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=1,291,392,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6727/500000] + train/ActionL1Loss=0.0532 + throughput/total_tokens=1,291,584,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6728/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=1,291,776,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6729/500000] + train/ActionL1Loss=0.0496 + throughput/total_tokens=1,291,968,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6730/500000] + train/ActionL1Loss=0.0720 + throughput/total_tokens=1,292,160,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6731/500000] + train/ActionL1Loss=0.0576 + throughput/total_tokens=1,292,352,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6732/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,292,544,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6733/500000] + train/ActionL1Loss=0.0704 + throughput/total_tokens=1,292,736,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6734/500000] + train/ActionL1Loss=0.0568 + throughput/total_tokens=1,292,928,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6735/500000] + train/ActionL1Loss=0.0655 + throughput/total_tokens=1,293,120,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6736/500000] + train/ActionL1Loss=0.0728 + throughput/total_tokens=1,293,312,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6737/500000] + train/ActionL1Loss=0.0598 + throughput/total_tokens=1,293,504,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6738/500000] + train/ActionL1Loss=0.0610 + throughput/total_tokens=1,293,696,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6739/500000] + train/ActionL1Loss=0.0838 + throughput/total_tokens=1,293,888,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6740/500000] + optim/total_grad_norm=7.610 + train/ActionL1Loss=0.0643 + throughput/total_tokens=1,294,080,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6741/500000] + train/ActionL1Loss=0.0909 + throughput/total_tokens=1,294,272,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6742/500000] + train/ActionL1Loss=0.0870 + throughput/total_tokens=1,294,464,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6743/500000] + train/ActionL1Loss=0.0505 + throughput/total_tokens=1,294,656,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6744/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=1,294,848,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6745/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=1,295,040,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6746/500000] + train/ActionL1Loss=0.0762 + throughput/total_tokens=1,295,232,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6747/500000] + train/ActionL1Loss=0.0571 + throughput/total_tokens=1,295,424,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6748/500000] + train/ActionL1Loss=0.0741 + throughput/total_tokens=1,295,616,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6749/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=1,295,808,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6750/500000] + train/ActionL1Loss=0.0552 + throughput/total_tokens=1,296,000,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=6751/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=1,296,192,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6752/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=1,296,384,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=6753/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,296,576,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6754/500000] + train/ActionL1Loss=0.0635 + throughput/total_tokens=1,296,768,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6755/500000] + train/ActionL1Loss=0.0678 + throughput/total_tokens=1,296,960,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6756/500000] + train/ActionL1Loss=0.0637 + throughput/total_tokens=1,297,152,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6757/500000] + train/ActionL1Loss=0.0823 + throughput/total_tokens=1,297,344,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6758/500000] + train/ActionL1Loss=0.0523 + throughput/total_tokens=1,297,536,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6759/500000] + train/ActionL1Loss=0.0555 + throughput/total_tokens=1,297,728,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6760/500000] + optim/total_grad_norm=7.223 + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,297,920,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6761/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=1,298,112,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6762/500000] + train/ActionL1Loss=0.0680 + throughput/total_tokens=1,298,304,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6763/500000] + train/ActionL1Loss=0.0530 + throughput/total_tokens=1,298,496,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6764/500000] + train/ActionL1Loss=0.0634 + throughput/total_tokens=1,298,688,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6765/500000] + train/ActionL1Loss=0.0617 + throughput/total_tokens=1,298,880,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6766/500000] + train/ActionL1Loss=0.0729 + throughput/total_tokens=1,299,072,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6767/500000] + train/ActionL1Loss=0.0843 + throughput/total_tokens=1,299,264,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=6768/500000] + train/ActionL1Loss=0.0559 + throughput/total_tokens=1,299,456,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=6769/500000] + train/ActionL1Loss=0.0535 + throughput/total_tokens=1,299,648,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6770/500000] + train/ActionL1Loss=0.0893 + throughput/total_tokens=1,299,840,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=6771/500000] + train/ActionL1Loss=0.0744 + throughput/total_tokens=1,300,032,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6772/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,300,224,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6773/500000] + train/ActionL1Loss=0.0712 + throughput/total_tokens=1,300,416,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6774/500000] + train/ActionL1Loss=0.0483 + throughput/total_tokens=1,300,608,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6775/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,300,800,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6776/500000] + train/ActionL1Loss=0.0510 + throughput/total_tokens=1,300,992,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6777/500000] + train/ActionL1Loss=0.0583 + throughput/total_tokens=1,301,184,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6778/500000] + train/ActionL1Loss=0.0670 + throughput/total_tokens=1,301,376,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6779/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,301,568,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6780/500000] + optim/total_grad_norm=6.110 + train/ActionL1Loss=0.0643 + throughput/total_tokens=1,301,760,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=6781/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,301,952,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6782/500000] + train/ActionL1Loss=0.0509 + throughput/total_tokens=1,302,144,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6783/500000] + train/ActionL1Loss=0.0675 + throughput/total_tokens=1,302,336,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6784/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=1,302,528,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6785/500000] + train/ActionL1Loss=0.0595 + throughput/total_tokens=1,302,720,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6786/500000] + train/ActionL1Loss=0.0542 + throughput/total_tokens=1,302,912,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6787/500000] + train/ActionL1Loss=0.0566 + throughput/total_tokens=1,303,104,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6788/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=1,303,296,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6789/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=1,303,488,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6790/500000] + train/ActionL1Loss=0.0757 + throughput/total_tokens=1,303,680,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6791/500000] + train/ActionL1Loss=0.0486 + throughput/total_tokens=1,303,872,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6792/500000] + train/ActionL1Loss=0.0635 + throughput/total_tokens=1,304,064,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6793/500000] + train/ActionL1Loss=0.0724 + throughput/total_tokens=1,304,256,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6794/500000] + train/ActionL1Loss=0.0758 + throughput/total_tokens=1,304,448,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6795/500000] + train/ActionL1Loss=0.0718 + throughput/total_tokens=1,304,640,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6796/500000] + train/ActionL1Loss=0.0624 + throughput/total_tokens=1,304,832,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6797/500000] + train/ActionL1Loss=0.0601 + throughput/total_tokens=1,305,024,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6798/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=1,305,216,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6799/500000] + train/ActionL1Loss=0.0507 + throughput/total_tokens=1,305,408,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6800/500000] + optim/total_grad_norm=6.770 + train/ActionL1Loss=0.0547 + throughput/total_tokens=1,305,600,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6801/500000] + train/ActionL1Loss=0.0727 + throughput/total_tokens=1,305,792,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6802/500000] + train/ActionL1Loss=0.0538 + throughput/total_tokens=1,305,984,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6803/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=1,306,176,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6804/500000] + train/ActionL1Loss=0.0771 + throughput/total_tokens=1,306,368,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6805/500000] + train/ActionL1Loss=0.0670 + throughput/total_tokens=1,306,560,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6806/500000] + train/ActionL1Loss=0.0762 + throughput/total_tokens=1,306,752,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6807/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=1,306,944,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6808/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=1,307,136,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6809/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,307,328,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6810/500000] + train/ActionL1Loss=0.0794 + throughput/total_tokens=1,307,520,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6811/500000] + train/ActionL1Loss=0.0751 + throughput/total_tokens=1,307,712,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6812/500000] + train/ActionL1Loss=0.0544 + throughput/total_tokens=1,307,904,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6813/500000] + train/ActionL1Loss=0.0607 + throughput/total_tokens=1,308,096,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6814/500000] + train/ActionL1Loss=0.0629 + throughput/total_tokens=1,308,288,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6815/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=1,308,480,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6816/500000] + train/ActionL1Loss=0.0467 + throughput/total_tokens=1,308,672,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6817/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=1,308,864,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6818/500000] + train/ActionL1Loss=0.0657 + throughput/total_tokens=1,309,056,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6819/500000] + train/ActionL1Loss=0.0555 + throughput/total_tokens=1,309,248,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6820/500000] + optim/total_grad_norm=6.410 + train/ActionL1Loss=0.0633 + throughput/total_tokens=1,309,440,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6821/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=1,309,632,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6822/500000] + train/ActionL1Loss=0.0741 + throughput/total_tokens=1,309,824,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6823/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,310,016,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6824/500000] + train/ActionL1Loss=0.0670 + throughput/total_tokens=1,310,208,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6825/500000] + train/ActionL1Loss=0.0728 + throughput/total_tokens=1,310,400,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6826/500000] + train/ActionL1Loss=0.0422 + throughput/total_tokens=1,310,592,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6827/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=1,310,784,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6828/500000] + train/ActionL1Loss=0.0637 + throughput/total_tokens=1,310,976,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6829/500000] + train/ActionL1Loss=0.0726 + throughput/total_tokens=1,311,168,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6830/500000] + train/ActionL1Loss=0.0579 + throughput/total_tokens=1,311,360,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6831/500000] + train/ActionL1Loss=0.0564 + throughput/total_tokens=1,311,552,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6832/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,311,744,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6833/500000] + train/ActionL1Loss=0.0801 + throughput/total_tokens=1,311,936,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6834/500000] + train/ActionL1Loss=0.0545 + throughput/total_tokens=1,312,128,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6835/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=1,312,320,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6836/500000] + train/ActionL1Loss=0.0708 + throughput/total_tokens=1,312,512,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6837/500000] + train/ActionL1Loss=0.0748 + throughput/total_tokens=1,312,704,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6838/500000] + train/ActionL1Loss=0.0799 + throughput/total_tokens=1,312,896,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6839/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=1,313,088,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6840/500000] + optim/total_grad_norm=6.454 + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,313,280,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6841/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=1,313,472,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6842/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,313,664,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6843/500000] + train/ActionL1Loss=0.0652 + throughput/total_tokens=1,313,856,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6844/500000] + train/ActionL1Loss=0.0553 + throughput/total_tokens=1,314,048,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6845/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=1,314,240,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6846/500000] + train/ActionL1Loss=0.0565 + throughput/total_tokens=1,314,432,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6847/500000] + train/ActionL1Loss=0.0439 + throughput/total_tokens=1,314,624,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6848/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=1,314,816,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6849/500000] + train/ActionL1Loss=0.0523 + throughput/total_tokens=1,315,008,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6850/500000] + train/ActionL1Loss=0.0674 + throughput/total_tokens=1,315,200,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6851/500000] + train/ActionL1Loss=0.0831 + throughput/total_tokens=1,315,392,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6852/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=1,315,584,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6853/500000] + train/ActionL1Loss=0.0784 + throughput/total_tokens=1,315,776,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6854/500000] + train/ActionL1Loss=0.0702 + throughput/total_tokens=1,315,968,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6855/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=1,316,160,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6856/500000] + train/ActionL1Loss=0.0673 + throughput/total_tokens=1,316,352,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6857/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=1,316,544,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6858/500000] + train/ActionL1Loss=0.0768 + throughput/total_tokens=1,316,736,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6859/500000] + train/ActionL1Loss=0.0540 + throughput/total_tokens=1,316,928,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6860/500000] + optim/total_grad_norm=5.439 + train/ActionL1Loss=0.0938 + throughput/total_tokens=1,317,120,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6861/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,317,312,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6862/500000] + train/ActionL1Loss=0.0519 + throughput/total_tokens=1,317,504,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6863/500000] + train/ActionL1Loss=0.0681 + throughput/total_tokens=1,317,696,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6864/500000] + train/ActionL1Loss=0.0617 + throughput/total_tokens=1,317,888,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6865/500000] + train/ActionL1Loss=0.0686 + throughput/total_tokens=1,318,080,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6866/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=1,318,272,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6867/500000] + train/ActionL1Loss=0.0634 + throughput/total_tokens=1,318,464,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6868/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=1,318,656,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6869/500000] + train/ActionL1Loss=0.0650 + throughput/total_tokens=1,318,848,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=6870/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,319,040,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6871/500000] + train/ActionL1Loss=0.0825 + throughput/total_tokens=1,319,232,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6872/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=1,319,424,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6873/500000] + train/ActionL1Loss=0.0559 + throughput/total_tokens=1,319,616,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6874/500000] + train/ActionL1Loss=0.0617 + throughput/total_tokens=1,319,808,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6875/500000] + train/ActionL1Loss=0.0727 + throughput/total_tokens=1,320,000,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6876/500000] + train/ActionL1Loss=0.0506 + throughput/total_tokens=1,320,192,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6877/500000] + train/ActionL1Loss=0.0554 + throughput/total_tokens=1,320,384,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6878/500000] + train/ActionL1Loss=0.0706 + throughput/total_tokens=1,320,576,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6879/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=1,320,768,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6880/500000] + optim/total_grad_norm=6.151 + train/ActionL1Loss=0.0857 + throughput/total_tokens=1,320,960,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6881/500000] + train/ActionL1Loss=0.0582 + throughput/total_tokens=1,321,152,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6882/500000] + train/ActionL1Loss=0.0758 + throughput/total_tokens=1,321,344,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6883/500000] + train/ActionL1Loss=0.0595 + throughput/total_tokens=1,321,536,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6884/500000] + train/ActionL1Loss=0.0469 + throughput/total_tokens=1,321,728,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6885/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=1,321,920,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6886/500000] + train/ActionL1Loss=0.0832 + throughput/total_tokens=1,322,112,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6887/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=1,322,304,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6888/500000] + train/ActionL1Loss=0.0838 + throughput/total_tokens=1,322,496,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6889/500000] + train/ActionL1Loss=0.0684 + throughput/total_tokens=1,322,688,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6890/500000] + train/ActionL1Loss=0.0683 + throughput/total_tokens=1,322,880,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6891/500000] + train/ActionL1Loss=0.0541 + throughput/total_tokens=1,323,072,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6892/500000] + train/ActionL1Loss=0.0583 + throughput/total_tokens=1,323,264,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6893/500000] + train/ActionL1Loss=0.0386 + throughput/total_tokens=1,323,456,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6894/500000] + train/ActionL1Loss=0.0568 + throughput/total_tokens=1,323,648,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6895/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=1,323,840,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6896/500000] + train/ActionL1Loss=0.0606 + throughput/total_tokens=1,324,032,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6897/500000] + train/ActionL1Loss=0.0525 + throughput/total_tokens=1,324,224,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6898/500000] + train/ActionL1Loss=0.0535 + throughput/total_tokens=1,324,416,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6899/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=1,324,608,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=6900/500000] + optim/total_grad_norm=6.920 + train/ActionL1Loss=0.0623 + throughput/total_tokens=1,324,800,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=6901/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=1,324,992,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6902/500000] + train/ActionL1Loss=0.0453 + throughput/total_tokens=1,325,184,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6903/500000] + train/ActionL1Loss=0.0599 + throughput/total_tokens=1,325,376,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6904/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,325,568,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6905/500000] + train/ActionL1Loss=0.0529 + throughput/total_tokens=1,325,760,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=6906/500000] + train/ActionL1Loss=0.0583 + throughput/total_tokens=1,325,952,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=6907/500000] + train/ActionL1Loss=0.0591 + throughput/total_tokens=1,326,144,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=6908/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=1,326,336,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=6909/500000] + train/ActionL1Loss=0.0707 + throughput/total_tokens=1,326,528,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=6910/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=1,326,720,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6911/500000] + train/ActionL1Loss=0.0551 + throughput/total_tokens=1,326,912,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=6912/500000] + train/ActionL1Loss=0.0791 + throughput/total_tokens=1,327,104,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=6913/500000] + train/ActionL1Loss=0.0831 + throughput/total_tokens=1,327,296,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=6914/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,327,488,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=6915/500000] + train/ActionL1Loss=0.0523 + throughput/total_tokens=1,327,680,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=6916/500000] + train/ActionL1Loss=0.0598 + throughput/total_tokens=1,327,872,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=6917/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=1,328,064,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6918/500000] + train/ActionL1Loss=0.0718 + throughput/total_tokens=1,328,256,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6919/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,328,448,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6920/500000] + optim/total_grad_norm=6.979 + train/ActionL1Loss=0.0516 + throughput/total_tokens=1,328,640,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=6921/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,328,832,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6922/500000] + train/ActionL1Loss=0.0629 + throughput/total_tokens=1,329,024,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6923/500000] + train/ActionL1Loss=0.0566 + throughput/total_tokens=1,329,216,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6924/500000] + train/ActionL1Loss=0.0603 + throughput/total_tokens=1,329,408,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=6925/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=1,329,600,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6926/500000] + train/ActionL1Loss=0.0617 + throughput/total_tokens=1,329,792,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6927/500000] + train/ActionL1Loss=0.0840 + throughput/total_tokens=1,329,984,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6928/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,330,176,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6929/500000] + train/ActionL1Loss=0.0739 + throughput/total_tokens=1,330,368,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6930/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=1,330,560,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6931/500000] + train/ActionL1Loss=0.0850 + throughput/total_tokens=1,330,752,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6932/500000] + train/ActionL1Loss=0.0614 + throughput/total_tokens=1,330,944,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6933/500000] + train/ActionL1Loss=0.0714 + throughput/total_tokens=1,331,136,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6934/500000] + train/ActionL1Loss=0.0636 + throughput/total_tokens=1,331,328,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6935/500000] + train/ActionL1Loss=0.0747 + throughput/total_tokens=1,331,520,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6936/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,331,712,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6937/500000] + train/ActionL1Loss=0.0782 + throughput/total_tokens=1,331,904,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6938/500000] + train/ActionL1Loss=0.0827 + throughput/total_tokens=1,332,096,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6939/500000] + train/ActionL1Loss=0.0722 + throughput/total_tokens=1,332,288,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6940/500000] + optim/total_grad_norm=7.339 + train/ActionL1Loss=0.0710 + throughput/total_tokens=1,332,480,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6941/500000] + train/ActionL1Loss=0.0578 + throughput/total_tokens=1,332,672,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6942/500000] + train/ActionL1Loss=0.0594 + throughput/total_tokens=1,332,864,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6943/500000] + train/ActionL1Loss=0.0608 + throughput/total_tokens=1,333,056,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6944/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,333,248,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6945/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=1,333,440,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6946/500000] + train/ActionL1Loss=0.0565 + throughput/total_tokens=1,333,632,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6947/500000] + train/ActionL1Loss=0.0514 + throughput/total_tokens=1,333,824,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6948/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=1,334,016,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6949/500000] + train/ActionL1Loss=0.0761 + throughput/total_tokens=1,334,208,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=6950/500000] + train/ActionL1Loss=0.0551 + throughput/total_tokens=1,334,400,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6951/500000] + train/ActionL1Loss=0.0699 + throughput/total_tokens=1,334,592,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6952/500000] + train/ActionL1Loss=0.0419 + throughput/total_tokens=1,334,784,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6953/500000] + train/ActionL1Loss=0.0586 + throughput/total_tokens=1,334,976,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6954/500000] + train/ActionL1Loss=0.0748 + throughput/total_tokens=1,335,168,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6955/500000] + train/ActionL1Loss=0.0703 + throughput/total_tokens=1,335,360,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6956/500000] + train/ActionL1Loss=0.0698 + throughput/total_tokens=1,335,552,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6957/500000] + train/ActionL1Loss=0.0738 + throughput/total_tokens=1,335,744,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6958/500000] + train/ActionL1Loss=0.0648 + throughput/total_tokens=1,335,936,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6959/500000] + train/ActionL1Loss=0.0641 + throughput/total_tokens=1,336,128,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=6960/500000] + optim/total_grad_norm=4.608 + train/ActionL1Loss=0.0699 + throughput/total_tokens=1,336,320,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=6961/500000] + train/ActionL1Loss=0.0732 + throughput/total_tokens=1,336,512,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6962/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,336,704,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6963/500000] + train/ActionL1Loss=0.0550 + throughput/total_tokens=1,336,896,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6964/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=1,337,088,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6965/500000] + train/ActionL1Loss=0.0634 + throughput/total_tokens=1,337,280,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6966/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,337,472,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6967/500000] + train/ActionL1Loss=0.0581 + throughput/total_tokens=1,337,664,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6968/500000] + train/ActionL1Loss=0.0670 + throughput/total_tokens=1,337,856,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6969/500000] + train/ActionL1Loss=0.0727 + throughput/total_tokens=1,338,048,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=6970/500000] + train/ActionL1Loss=0.0650 + throughput/total_tokens=1,338,240,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6971/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=1,338,432,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6972/500000] + train/ActionL1Loss=0.0493 + throughput/total_tokens=1,338,624,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6973/500000] + train/ActionL1Loss=0.0832 + throughput/total_tokens=1,338,816,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6974/500000] + train/ActionL1Loss=0.0653 + throughput/total_tokens=1,339,008,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6975/500000] + train/ActionL1Loss=0.0957 + throughput/total_tokens=1,339,200,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6976/500000] + train/ActionL1Loss=0.0597 + throughput/total_tokens=1,339,392,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6977/500000] + train/ActionL1Loss=0.0566 + throughput/total_tokens=1,339,584,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6978/500000] + train/ActionL1Loss=0.0591 + throughput/total_tokens=1,339,776,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6979/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,339,968,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6980/500000] + optim/total_grad_norm=4.978 + train/ActionL1Loss=0.0819 + throughput/total_tokens=1,340,160,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6981/500000] + train/ActionL1Loss=0.0624 + throughput/total_tokens=1,340,352,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6982/500000] + train/ActionL1Loss=0.0488 + throughput/total_tokens=1,340,544,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6983/500000] + train/ActionL1Loss=0.0472 + throughput/total_tokens=1,340,736,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6984/500000] + train/ActionL1Loss=0.0819 + throughput/total_tokens=1,340,928,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6985/500000] + train/ActionL1Loss=0.0611 + throughput/total_tokens=1,341,120,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6986/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,341,312,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6987/500000] + train/ActionL1Loss=0.0788 + throughput/total_tokens=1,341,504,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6988/500000] + train/ActionL1Loss=0.0678 + throughput/total_tokens=1,341,696,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6989/500000] + train/ActionL1Loss=0.0687 + throughput/total_tokens=1,341,888,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6990/500000] + train/ActionL1Loss=0.0582 + throughput/total_tokens=1,342,080,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=6991/500000] + train/ActionL1Loss=0.0489 + throughput/total_tokens=1,342,272,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6992/500000] + train/ActionL1Loss=0.0758 + throughput/total_tokens=1,342,464,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6993/500000] + train/ActionL1Loss=0.0553 + throughput/total_tokens=1,342,656,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6994/500000] + train/ActionL1Loss=0.0581 + throughput/total_tokens=1,342,848,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6995/500000] + train/ActionL1Loss=0.0532 + throughput/total_tokens=1,343,040,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6996/500000] + train/ActionL1Loss=0.0528 + throughput/total_tokens=1,343,232,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6997/500000] + train/ActionL1Loss=0.0538 + throughput/total_tokens=1,343,424,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6998/500000] + train/ActionL1Loss=0.0626 + throughput/total_tokens=1,343,616,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=6999/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=1,343,808,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7000/500000] + optim/total_grad_norm=4.266 + train/ActionL1Loss=0.0685 + throughput/total_tokens=1,344,000,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +10/02 [06:26:18] INFO | >> Saving config... checkpoint.py:608 +10/02 [06:26:49] INFO | >> Saving model state... checkpoint.py:796 +10/02 [06:28:02] INFO | >> Saving optim state... checkpoint.py:811 +10/02 [06:29:37] INFO | >> Saving trainer state... checkpoint.py:670 + INFO | >> Saving config... checkpoint.py:608 +[step=7001/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,344,192,000 + throughput/device/tokens_per_second=1,211 + throughput/device/batches_per_second=0.0505 +[step=7002/500000] + train/ActionL1Loss=0.0569 + throughput/total_tokens=1,344,384,000 + throughput/device/tokens_per_second=1,174 + throughput/device/batches_per_second=0.0489 +[step=7003/500000] + train/ActionL1Loss=0.0747 + throughput/total_tokens=1,344,576,000 + throughput/device/tokens_per_second=1,146 + throughput/device/batches_per_second=0.0478 +[step=7004/500000] + train/ActionL1Loss=0.0511 + throughput/total_tokens=1,344,768,000 + throughput/device/tokens_per_second=1,145 + throughput/device/batches_per_second=0.0477 +[step=7005/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=1,344,960,000 + throughput/device/tokens_per_second=1,149 + throughput/device/batches_per_second=0.0479 +[step=7006/500000] + train/ActionL1Loss=0.0616 + throughput/total_tokens=1,345,152,000 + throughput/device/tokens_per_second=1,152 + throughput/device/batches_per_second=0.0480 +[step=7007/500000] + train/ActionL1Loss=0.0788 + throughput/total_tokens=1,345,344,000 + throughput/device/tokens_per_second=1,154 + throughput/device/batches_per_second=0.0481 +[step=7008/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=1,345,536,000 + throughput/device/tokens_per_second=1,156 + throughput/device/batches_per_second=0.0482 +[step=7009/500000] + train/ActionL1Loss=0.0671 + throughput/total_tokens=1,345,728,000 + throughput/device/tokens_per_second=1,157 + throughput/device/batches_per_second=0.0482 +[step=7010/500000] + train/ActionL1Loss=0.0507 + throughput/total_tokens=1,345,920,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 + System/Peak GPU Memory (MB)=46,917 +[step=7011/500000] + train/ActionL1Loss=0.0803 + throughput/total_tokens=1,346,112,000 + throughput/device/tokens_per_second=1,158 + throughput/device/batches_per_second=0.0483 +[step=7012/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=1,346,304,000 + throughput/device/tokens_per_second=1,159 + throughput/device/batches_per_second=0.0483 +[step=7013/500000] + train/ActionL1Loss=0.0504 + throughput/total_tokens=1,346,496,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0483 +[step=7014/500000] + train/ActionL1Loss=0.0669 + throughput/total_tokens=1,346,688,000 + throughput/device/tokens_per_second=1,160 + throughput/device/batches_per_second=0.0484 +[step=7015/500000] + train/ActionL1Loss=0.0789 + throughput/total_tokens=1,346,880,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=7016/500000] + train/ActionL1Loss=0.0686 + throughput/total_tokens=1,347,072,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=7017/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,347,264,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=7018/500000] + train/ActionL1Loss=0.0532 + throughput/total_tokens=1,347,456,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7019/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=1,347,648,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7020/500000] + optim/total_grad_norm=6.866 + train/ActionL1Loss=0.0729 + throughput/total_tokens=1,347,840,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=7021/500000] + train/ActionL1Loss=0.0691 + throughput/total_tokens=1,348,032,000 + throughput/device/tokens_per_second=1,161 + throughput/device/batches_per_second=0.0484 +[step=7022/500000] + train/ActionL1Loss=0.0678 + throughput/total_tokens=1,348,224,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=7023/500000] + train/ActionL1Loss=0.0801 + throughput/total_tokens=1,348,416,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7024/500000] + train/ActionL1Loss=0.0830 + throughput/total_tokens=1,348,608,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7025/500000] + train/ActionL1Loss=0.0734 + throughput/total_tokens=1,348,800,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7026/500000] + train/ActionL1Loss=0.0557 + throughput/total_tokens=1,348,992,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7027/500000] + train/ActionL1Loss=0.0432 + throughput/total_tokens=1,349,184,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7028/500000] + train/ActionL1Loss=0.0976 + throughput/total_tokens=1,349,376,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7029/500000] + train/ActionL1Loss=0.0545 + throughput/total_tokens=1,349,568,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7030/500000] + train/ActionL1Loss=0.0664 + throughput/total_tokens=1,349,760,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7031/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=1,349,952,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7032/500000] + train/ActionL1Loss=0.0569 + throughput/total_tokens=1,350,144,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7033/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=1,350,336,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7034/500000] + train/ActionL1Loss=0.0572 + throughput/total_tokens=1,350,528,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7035/500000] + train/ActionL1Loss=0.0624 + throughput/total_tokens=1,350,720,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7036/500000] + train/ActionL1Loss=0.0724 + throughput/total_tokens=1,350,912,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7037/500000] + train/ActionL1Loss=0.0780 + throughput/total_tokens=1,351,104,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7038/500000] + train/ActionL1Loss=0.0599 + throughput/total_tokens=1,351,296,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7039/500000] + train/ActionL1Loss=0.0623 + throughput/total_tokens=1,351,488,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7040/500000] + optim/total_grad_norm=5.971 + train/ActionL1Loss=0.0612 + throughput/total_tokens=1,351,680,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7041/500000] + train/ActionL1Loss=0.0738 + throughput/total_tokens=1,351,872,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7042/500000] + train/ActionL1Loss=0.0541 + throughput/total_tokens=1,352,064,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7043/500000] + train/ActionL1Loss=0.0669 + throughput/total_tokens=1,352,256,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7044/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=1,352,448,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7045/500000] + train/ActionL1Loss=0.0435 + throughput/total_tokens=1,352,640,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7046/500000] + train/ActionL1Loss=0.0596 + throughput/total_tokens=1,352,832,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7047/500000] + train/ActionL1Loss=0.0609 + throughput/total_tokens=1,353,024,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7048/500000] + train/ActionL1Loss=0.0518 + throughput/total_tokens=1,353,216,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7049/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=1,353,408,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7050/500000] + train/ActionL1Loss=0.0598 + throughput/total_tokens=1,353,600,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7051/500000] + train/ActionL1Loss=0.0493 + throughput/total_tokens=1,353,792,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7052/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=1,353,984,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7053/500000] + train/ActionL1Loss=0.0571 + throughput/total_tokens=1,354,176,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7054/500000] + train/ActionL1Loss=0.0422 + throughput/total_tokens=1,354,368,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7055/500000] + train/ActionL1Loss=0.0486 + throughput/total_tokens=1,354,560,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7056/500000] + train/ActionL1Loss=0.0568 + throughput/total_tokens=1,354,752,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7057/500000] + train/ActionL1Loss=0.0473 + throughput/total_tokens=1,354,944,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7058/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=1,355,136,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7059/500000] + train/ActionL1Loss=0.0491 + throughput/total_tokens=1,355,328,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7060/500000] + optim/total_grad_norm=5.626 + train/ActionL1Loss=0.0621 + throughput/total_tokens=1,355,520,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=7061/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=1,355,712,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7062/500000] + train/ActionL1Loss=0.0596 + throughput/total_tokens=1,355,904,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7063/500000] + train/ActionL1Loss=0.0519 + throughput/total_tokens=1,356,096,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7064/500000] + train/ActionL1Loss=0.0636 + throughput/total_tokens=1,356,288,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7065/500000] + train/ActionL1Loss=0.0556 + throughput/total_tokens=1,356,480,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7066/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,356,672,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7067/500000] + train/ActionL1Loss=0.0794 + throughput/total_tokens=1,356,864,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7068/500000] + train/ActionL1Loss=0.0572 + throughput/total_tokens=1,357,056,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7069/500000] + train/ActionL1Loss=0.0636 + throughput/total_tokens=1,357,248,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7070/500000] + train/ActionL1Loss=0.0503 + throughput/total_tokens=1,357,440,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7071/500000] + train/ActionL1Loss=0.0437 + throughput/total_tokens=1,357,632,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7072/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,357,824,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7073/500000] + train/ActionL1Loss=0.0758 + throughput/total_tokens=1,358,016,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7074/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,358,208,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7075/500000] + train/ActionL1Loss=0.0721 + throughput/total_tokens=1,358,400,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7076/500000] + train/ActionL1Loss=0.0531 + throughput/total_tokens=1,358,592,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7077/500000] + train/ActionL1Loss=0.0603 + throughput/total_tokens=1,358,784,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7078/500000] + train/ActionL1Loss=0.0510 + throughput/total_tokens=1,358,976,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7079/500000] + train/ActionL1Loss=0.0730 + throughput/total_tokens=1,359,168,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7080/500000] + optim/total_grad_norm=7.207 + train/ActionL1Loss=0.0558 + throughput/total_tokens=1,359,360,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7081/500000] + train/ActionL1Loss=0.0714 + throughput/total_tokens=1,359,552,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7082/500000] + train/ActionL1Loss=0.0580 + throughput/total_tokens=1,359,744,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7083/500000] + train/ActionL1Loss=0.0555 + throughput/total_tokens=1,359,936,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7084/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=1,360,128,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7085/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=1,360,320,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7086/500000] + train/ActionL1Loss=0.0582 + throughput/total_tokens=1,360,512,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7087/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,360,704,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7088/500000] + train/ActionL1Loss=0.0665 + throughput/total_tokens=1,360,896,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7089/500000] + train/ActionL1Loss=0.0480 + throughput/total_tokens=1,361,088,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7090/500000] + train/ActionL1Loss=0.0808 + throughput/total_tokens=1,361,280,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7091/500000] + train/ActionL1Loss=0.0671 + throughput/total_tokens=1,361,472,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7092/500000] + train/ActionL1Loss=0.0727 + throughput/total_tokens=1,361,664,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7093/500000] + train/ActionL1Loss=0.0573 + throughput/total_tokens=1,361,856,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7094/500000] + train/ActionL1Loss=0.0527 + throughput/total_tokens=1,362,048,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7095/500000] + train/ActionL1Loss=0.0630 + throughput/total_tokens=1,362,240,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7096/500000] + train/ActionL1Loss=0.0493 + throughput/total_tokens=1,362,432,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7097/500000] + train/ActionL1Loss=0.0834 + throughput/total_tokens=1,362,624,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7098/500000] + train/ActionL1Loss=0.0546 + throughput/total_tokens=1,362,816,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7099/500000] + train/ActionL1Loss=0.0627 + throughput/total_tokens=1,363,008,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7100/500000] + optim/total_grad_norm=5.959 + train/ActionL1Loss=0.0496 + throughput/total_tokens=1,363,200,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7101/500000] + train/ActionL1Loss=0.0706 + throughput/total_tokens=1,363,392,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7102/500000] + train/ActionL1Loss=0.0767 + throughput/total_tokens=1,363,584,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7103/500000] + train/ActionL1Loss=0.0527 + throughput/total_tokens=1,363,776,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7104/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=1,363,968,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=7105/500000] + train/ActionL1Loss=0.0804 + throughput/total_tokens=1,364,160,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=7106/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,364,352,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7107/500000] + train/ActionL1Loss=0.0518 + throughput/total_tokens=1,364,544,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=7108/500000] + train/ActionL1Loss=0.0771 + throughput/total_tokens=1,364,736,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=7109/500000] + train/ActionL1Loss=0.0804 + throughput/total_tokens=1,364,928,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7110/500000] + train/ActionL1Loss=0.0599 + throughput/total_tokens=1,365,120,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 + System/Peak GPU Memory (MB)=46,917 +[step=7111/500000] + train/ActionL1Loss=0.0620 + throughput/total_tokens=1,365,312,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7112/500000] + train/ActionL1Loss=0.0739 + throughput/total_tokens=1,365,504,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7113/500000] + train/ActionL1Loss=0.0649 + throughput/total_tokens=1,365,696,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7114/500000] + train/ActionL1Loss=0.0539 + throughput/total_tokens=1,365,888,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7115/500000] + train/ActionL1Loss=0.0627 + throughput/total_tokens=1,366,080,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7116/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,366,272,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7117/500000] + train/ActionL1Loss=0.0706 + throughput/total_tokens=1,366,464,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=7118/500000] + train/ActionL1Loss=0.0758 + throughput/total_tokens=1,366,656,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7119/500000] + train/ActionL1Loss=0.0608 + throughput/total_tokens=1,366,848,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7120/500000] + optim/total_grad_norm=5.819 + train/ActionL1Loss=0.0553 + throughput/total_tokens=1,367,040,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7121/500000] + train/ActionL1Loss=0.0601 + throughput/total_tokens=1,367,232,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7122/500000] + train/ActionL1Loss=0.0611 + throughput/total_tokens=1,367,424,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7123/500000] + train/ActionL1Loss=0.0551 + throughput/total_tokens=1,367,616,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7124/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,367,808,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7125/500000] + train/ActionL1Loss=0.0457 + throughput/total_tokens=1,368,000,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7126/500000] + train/ActionL1Loss=0.0630 + throughput/total_tokens=1,368,192,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7127/500000] + train/ActionL1Loss=0.0739 + throughput/total_tokens=1,368,384,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7128/500000] + train/ActionL1Loss=0.0785 + throughput/total_tokens=1,368,576,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7129/500000] + train/ActionL1Loss=0.0546 + throughput/total_tokens=1,368,768,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7130/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=1,368,960,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7131/500000] + train/ActionL1Loss=0.0533 + throughput/total_tokens=1,369,152,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7132/500000] + train/ActionL1Loss=0.0528 + throughput/total_tokens=1,369,344,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7133/500000] + train/ActionL1Loss=0.0607 + throughput/total_tokens=1,369,536,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7134/500000] + train/ActionL1Loss=0.0760 + throughput/total_tokens=1,369,728,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7135/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,369,920,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7136/500000] + train/ActionL1Loss=0.0596 + throughput/total_tokens=1,370,112,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7137/500000] + train/ActionL1Loss=0.0565 + throughput/total_tokens=1,370,304,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7138/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=1,370,496,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7139/500000] + train/ActionL1Loss=0.0787 + throughput/total_tokens=1,370,688,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7140/500000] + optim/total_grad_norm=5.508 + train/ActionL1Loss=0.0797 + throughput/total_tokens=1,370,880,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7141/500000] + train/ActionL1Loss=0.0738 + throughput/total_tokens=1,371,072,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7142/500000] + train/ActionL1Loss=0.0665 + throughput/total_tokens=1,371,264,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7143/500000] + train/ActionL1Loss=0.0540 + throughput/total_tokens=1,371,456,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7144/500000] + train/ActionL1Loss=0.0655 + throughput/total_tokens=1,371,648,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7145/500000] + train/ActionL1Loss=0.0743 + throughput/total_tokens=1,371,840,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7146/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=1,372,032,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7147/500000] + train/ActionL1Loss=0.0743 + throughput/total_tokens=1,372,224,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7148/500000] + train/ActionL1Loss=0.0703 + throughput/total_tokens=1,372,416,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7149/500000] + train/ActionL1Loss=0.0594 + throughput/total_tokens=1,372,608,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7150/500000] + train/ActionL1Loss=0.0438 + throughput/total_tokens=1,372,800,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7151/500000] + train/ActionL1Loss=0.0610 + throughput/total_tokens=1,372,992,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7152/500000] + train/ActionL1Loss=0.0606 + throughput/total_tokens=1,373,184,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7153/500000] + train/ActionL1Loss=0.0701 + throughput/total_tokens=1,373,376,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7154/500000] + train/ActionL1Loss=0.0569 + throughput/total_tokens=1,373,568,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7155/500000] + train/ActionL1Loss=0.0686 + throughput/total_tokens=1,373,760,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7156/500000] + train/ActionL1Loss=0.0607 + throughput/total_tokens=1,373,952,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7157/500000] + train/ActionL1Loss=0.0666 + throughput/total_tokens=1,374,144,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7158/500000] + train/ActionL1Loss=0.0576 + throughput/total_tokens=1,374,336,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7159/500000] + train/ActionL1Loss=0.0574 + throughput/total_tokens=1,374,528,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7160/500000] + optim/total_grad_norm=6.516 + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,374,720,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=7161/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=1,374,912,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7162/500000] + train/ActionL1Loss=0.0581 + throughput/total_tokens=1,375,104,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7163/500000] + train/ActionL1Loss=0.0583 + throughput/total_tokens=1,375,296,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7164/500000] + train/ActionL1Loss=0.0522 + throughput/total_tokens=1,375,488,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7165/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,375,680,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7166/500000] + train/ActionL1Loss=0.0646 + throughput/total_tokens=1,375,872,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7167/500000] + train/ActionL1Loss=0.0452 + throughput/total_tokens=1,376,064,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7168/500000] + train/ActionL1Loss=0.0629 + throughput/total_tokens=1,376,256,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7169/500000] + train/ActionL1Loss=0.0714 + throughput/total_tokens=1,376,448,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7170/500000] + train/ActionL1Loss=0.0680 + throughput/total_tokens=1,376,640,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7171/500000] + train/ActionL1Loss=0.0640 + throughput/total_tokens=1,376,832,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7172/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=1,377,024,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7173/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=1,377,216,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7174/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=1,377,408,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7175/500000] + train/ActionL1Loss=0.0500 + throughput/total_tokens=1,377,600,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7176/500000] + train/ActionL1Loss=0.0574 + throughput/total_tokens=1,377,792,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7177/500000] + train/ActionL1Loss=0.0607 + throughput/total_tokens=1,377,984,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7178/500000] + train/ActionL1Loss=0.0451 + throughput/total_tokens=1,378,176,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7179/500000] + train/ActionL1Loss=0.0583 + throughput/total_tokens=1,378,368,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7180/500000] + optim/total_grad_norm=7.958 + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,378,560,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7181/500000] + train/ActionL1Loss=0.0494 + throughput/total_tokens=1,378,752,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7182/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,378,944,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7183/500000] + train/ActionL1Loss=0.0728 + throughput/total_tokens=1,379,136,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7184/500000] + train/ActionL1Loss=0.0504 + throughput/total_tokens=1,379,328,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7185/500000] + train/ActionL1Loss=0.0455 + throughput/total_tokens=1,379,520,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7186/500000] + train/ActionL1Loss=0.0574 + throughput/total_tokens=1,379,712,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7187/500000] + train/ActionL1Loss=0.0533 + throughput/total_tokens=1,379,904,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7188/500000] + train/ActionL1Loss=0.0525 + throughput/total_tokens=1,380,096,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7189/500000] + train/ActionL1Loss=0.0491 + throughput/total_tokens=1,380,288,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7190/500000] + train/ActionL1Loss=0.0573 + throughput/total_tokens=1,380,480,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7191/500000] + train/ActionL1Loss=0.0485 + throughput/total_tokens=1,380,672,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7192/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,380,864,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7193/500000] + train/ActionL1Loss=0.0532 + throughput/total_tokens=1,381,056,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7194/500000] + train/ActionL1Loss=0.0491 + throughput/total_tokens=1,381,248,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7195/500000] + train/ActionL1Loss=0.0648 + throughput/total_tokens=1,381,440,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7196/500000] + train/ActionL1Loss=0.0535 + throughput/total_tokens=1,381,632,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7197/500000] + train/ActionL1Loss=0.0559 + throughput/total_tokens=1,381,824,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7198/500000] + train/ActionL1Loss=0.0820 + throughput/total_tokens=1,382,016,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7199/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=1,382,208,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7200/500000] + optim/total_grad_norm=5.271 + train/ActionL1Loss=0.0548 + throughput/total_tokens=1,382,400,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7201/500000] + train/ActionL1Loss=0.0577 + throughput/total_tokens=1,382,592,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7202/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=1,382,784,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=7203/500000] + train/ActionL1Loss=0.0573 + throughput/total_tokens=1,382,976,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7204/500000] + train/ActionL1Loss=0.0832 + throughput/total_tokens=1,383,168,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7205/500000] + train/ActionL1Loss=0.0484 + throughput/total_tokens=1,383,360,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7206/500000] + train/ActionL1Loss=0.0664 + throughput/total_tokens=1,383,552,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7207/500000] + train/ActionL1Loss=0.0496 + throughput/total_tokens=1,383,744,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=7208/500000] + train/ActionL1Loss=0.0554 + throughput/total_tokens=1,383,936,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7209/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,384,128,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7210/500000] + train/ActionL1Loss=0.0559 + throughput/total_tokens=1,384,320,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=7211/500000] + train/ActionL1Loss=0.0453 + throughput/total_tokens=1,384,512,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=7212/500000] + train/ActionL1Loss=0.0516 + throughput/total_tokens=1,384,704,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=7213/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,384,896,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=7214/500000] + train/ActionL1Loss=0.0524 + throughput/total_tokens=1,385,088,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7215/500000] + train/ActionL1Loss=0.0585 + throughput/total_tokens=1,385,280,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7216/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,385,472,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7217/500000] + train/ActionL1Loss=0.0652 + throughput/total_tokens=1,385,664,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7218/500000] + train/ActionL1Loss=0.0511 + throughput/total_tokens=1,385,856,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7219/500000] + train/ActionL1Loss=0.0565 + throughput/total_tokens=1,386,048,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=7220/500000] + optim/total_grad_norm=6.081 + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,386,240,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7221/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=1,386,432,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7222/500000] + train/ActionL1Loss=0.0524 + throughput/total_tokens=1,386,624,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7223/500000] + train/ActionL1Loss=0.0709 + throughput/total_tokens=1,386,816,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7224/500000] + train/ActionL1Loss=0.0650 + throughput/total_tokens=1,387,008,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7225/500000] + train/ActionL1Loss=0.0673 + throughput/total_tokens=1,387,200,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7226/500000] + train/ActionL1Loss=0.0678 + throughput/total_tokens=1,387,392,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7227/500000] + train/ActionL1Loss=0.0539 + throughput/total_tokens=1,387,584,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7228/500000] + train/ActionL1Loss=0.0793 + throughput/total_tokens=1,387,776,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7229/500000] + train/ActionL1Loss=0.0702 + throughput/total_tokens=1,387,968,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7230/500000] + train/ActionL1Loss=0.0617 + throughput/total_tokens=1,388,160,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7231/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=1,388,352,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7232/500000] + train/ActionL1Loss=0.0652 + throughput/total_tokens=1,388,544,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7233/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=1,388,736,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7234/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,388,928,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7235/500000] + train/ActionL1Loss=0.0547 + throughput/total_tokens=1,389,120,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7236/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=1,389,312,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7237/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,389,504,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7238/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=1,389,696,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7239/500000] + train/ActionL1Loss=0.0710 + throughput/total_tokens=1,389,888,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7240/500000] + optim/total_grad_norm=5.700 + train/ActionL1Loss=0.0711 + throughput/total_tokens=1,390,080,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7241/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=1,390,272,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7242/500000] + train/ActionL1Loss=0.0543 + throughput/total_tokens=1,390,464,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7243/500000] + train/ActionL1Loss=0.0787 + throughput/total_tokens=1,390,656,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7244/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,390,848,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7245/500000] + train/ActionL1Loss=0.0776 + throughput/total_tokens=1,391,040,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7246/500000] + train/ActionL1Loss=0.0445 + throughput/total_tokens=1,391,232,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7247/500000] + train/ActionL1Loss=0.0493 + throughput/total_tokens=1,391,424,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7248/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=1,391,616,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7249/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,391,808,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7250/500000] + train/ActionL1Loss=0.0536 + throughput/total_tokens=1,392,000,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7251/500000] + train/ActionL1Loss=0.0472 + throughput/total_tokens=1,392,192,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7252/500000] + train/ActionL1Loss=0.0546 + throughput/total_tokens=1,392,384,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7253/500000] + train/ActionL1Loss=0.0624 + throughput/total_tokens=1,392,576,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7254/500000] + train/ActionL1Loss=0.0576 + throughput/total_tokens=1,392,768,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7255/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,392,960,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7256/500000] + train/ActionL1Loss=0.0686 + throughput/total_tokens=1,393,152,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7257/500000] + train/ActionL1Loss=0.0572 + throughput/total_tokens=1,393,344,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7258/500000] + train/ActionL1Loss=0.0476 + throughput/total_tokens=1,393,536,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7259/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=1,393,728,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7260/500000] + optim/total_grad_norm=6.312 + train/ActionL1Loss=0.0416 + throughput/total_tokens=1,393,920,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=7261/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=1,394,112,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7262/500000] + train/ActionL1Loss=0.0401 + throughput/total_tokens=1,394,304,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7263/500000] + train/ActionL1Loss=0.0621 + throughput/total_tokens=1,394,496,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7264/500000] + train/ActionL1Loss=0.0466 + throughput/total_tokens=1,394,688,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7265/500000] + train/ActionL1Loss=0.0559 + throughput/total_tokens=1,394,880,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7266/500000] + train/ActionL1Loss=0.0721 + throughput/total_tokens=1,395,072,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7267/500000] + train/ActionL1Loss=0.0497 + throughput/total_tokens=1,395,264,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=7268/500000] + train/ActionL1Loss=0.0594 + throughput/total_tokens=1,395,456,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=7269/500000] + train/ActionL1Loss=0.0701 + throughput/total_tokens=1,395,648,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7270/500000] + train/ActionL1Loss=0.0529 + throughput/total_tokens=1,395,840,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7271/500000] + train/ActionL1Loss=0.0567 + throughput/total_tokens=1,396,032,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7272/500000] + train/ActionL1Loss=0.0798 + throughput/total_tokens=1,396,224,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7273/500000] + train/ActionL1Loss=0.0643 + throughput/total_tokens=1,396,416,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7274/500000] + train/ActionL1Loss=0.0595 + throughput/total_tokens=1,396,608,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7275/500000] + train/ActionL1Loss=0.0608 + throughput/total_tokens=1,396,800,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7276/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,396,992,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7277/500000] + train/ActionL1Loss=0.0596 + throughput/total_tokens=1,397,184,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7278/500000] + train/ActionL1Loss=0.0757 + throughput/total_tokens=1,397,376,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7279/500000] + train/ActionL1Loss=0.0713 + throughput/total_tokens=1,397,568,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7280/500000] + optim/total_grad_norm=7.779 + train/ActionL1Loss=0.0470 + throughput/total_tokens=1,397,760,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7281/500000] + train/ActionL1Loss=0.0671 + throughput/total_tokens=1,397,952,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7282/500000] + train/ActionL1Loss=0.0686 + throughput/total_tokens=1,398,144,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7283/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,398,336,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7284/500000] + train/ActionL1Loss=0.0559 + throughput/total_tokens=1,398,528,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7285/500000] + train/ActionL1Loss=0.0598 + throughput/total_tokens=1,398,720,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7286/500000] + train/ActionL1Loss=0.0811 + throughput/total_tokens=1,398,912,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7287/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=1,399,104,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7288/500000] + train/ActionL1Loss=0.0657 + throughput/total_tokens=1,399,296,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7289/500000] + train/ActionL1Loss=0.0580 + throughput/total_tokens=1,399,488,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7290/500000] + train/ActionL1Loss=0.0452 + throughput/total_tokens=1,399,680,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7291/500000] + train/ActionL1Loss=0.0660 + throughput/total_tokens=1,399,872,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7292/500000] + train/ActionL1Loss=0.0499 + throughput/total_tokens=1,400,064,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7293/500000] + train/ActionL1Loss=0.0746 + throughput/total_tokens=1,400,256,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7294/500000] + train/ActionL1Loss=0.0477 + throughput/total_tokens=1,400,448,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7295/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,400,640,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7296/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=1,400,832,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7297/500000] + train/ActionL1Loss=0.0503 + throughput/total_tokens=1,401,024,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7298/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,401,216,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7299/500000] + train/ActionL1Loss=0.0639 + throughput/total_tokens=1,401,408,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7300/500000] + optim/total_grad_norm=6.383 + train/ActionL1Loss=0.0517 + throughput/total_tokens=1,401,600,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7301/500000] + train/ActionL1Loss=0.0691 + throughput/total_tokens=1,401,792,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7302/500000] + train/ActionL1Loss=0.0539 + throughput/total_tokens=1,401,984,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7303/500000] + train/ActionL1Loss=0.0741 + throughput/total_tokens=1,402,176,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7304/500000] + train/ActionL1Loss=0.0572 + throughput/total_tokens=1,402,368,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7305/500000] + train/ActionL1Loss=0.0560 + throughput/total_tokens=1,402,560,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7306/500000] + train/ActionL1Loss=0.0760 + throughput/total_tokens=1,402,752,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7307/500000] + train/ActionL1Loss=0.0545 + throughput/total_tokens=1,402,944,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7308/500000] + train/ActionL1Loss=0.0605 + throughput/total_tokens=1,403,136,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7309/500000] + train/ActionL1Loss=0.0481 + throughput/total_tokens=1,403,328,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7310/500000] + train/ActionL1Loss=0.0571 + throughput/total_tokens=1,403,520,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=7311/500000] + train/ActionL1Loss=0.0536 + throughput/total_tokens=1,403,712,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7312/500000] + train/ActionL1Loss=0.0690 + throughput/total_tokens=1,403,904,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7313/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=1,404,096,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7314/500000] + train/ActionL1Loss=0.0493 + throughput/total_tokens=1,404,288,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7315/500000] + train/ActionL1Loss=0.0577 + throughput/total_tokens=1,404,480,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7316/500000] + train/ActionL1Loss=0.0633 + throughput/total_tokens=1,404,672,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7317/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=1,404,864,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7318/500000] + train/ActionL1Loss=0.0553 + throughput/total_tokens=1,405,056,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7319/500000] + train/ActionL1Loss=0.0582 + throughput/total_tokens=1,405,248,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7320/500000] + optim/total_grad_norm=7.936 + train/ActionL1Loss=0.0683 + throughput/total_tokens=1,405,440,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7321/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=1,405,632,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7322/500000] + train/ActionL1Loss=0.0760 + throughput/total_tokens=1,405,824,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7323/500000] + train/ActionL1Loss=0.0730 + throughput/total_tokens=1,406,016,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7324/500000] + train/ActionL1Loss=0.0634 + throughput/total_tokens=1,406,208,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7325/500000] + train/ActionL1Loss=0.0607 + throughput/total_tokens=1,406,400,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7326/500000] + train/ActionL1Loss=0.0541 + throughput/total_tokens=1,406,592,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7327/500000] + train/ActionL1Loss=0.0723 + throughput/total_tokens=1,406,784,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7328/500000] + train/ActionL1Loss=0.0525 + throughput/total_tokens=1,406,976,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7329/500000] + train/ActionL1Loss=0.0476 + throughput/total_tokens=1,407,168,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7330/500000] + train/ActionL1Loss=0.0657 + throughput/total_tokens=1,407,360,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7331/500000] + train/ActionL1Loss=0.0558 + throughput/total_tokens=1,407,552,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7332/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=1,407,744,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7333/500000] + train/ActionL1Loss=0.0562 + throughput/total_tokens=1,407,936,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7334/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=1,408,128,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7335/500000] + train/ActionL1Loss=0.0689 + throughput/total_tokens=1,408,320,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7336/500000] + train/ActionL1Loss=0.0656 + throughput/total_tokens=1,408,512,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7337/500000] + train/ActionL1Loss=0.0544 + throughput/total_tokens=1,408,704,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7338/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,408,896,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7339/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,409,088,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7340/500000] + optim/total_grad_norm=5.910 + train/ActionL1Loss=0.0640 + throughput/total_tokens=1,409,280,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7341/500000] + train/ActionL1Loss=0.0511 + throughput/total_tokens=1,409,472,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7342/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,409,664,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7343/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,409,856,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7344/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=1,410,048,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7345/500000] + train/ActionL1Loss=0.0541 + throughput/total_tokens=1,410,240,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7346/500000] + train/ActionL1Loss=0.0439 + throughput/total_tokens=1,410,432,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7347/500000] + train/ActionL1Loss=0.0571 + throughput/total_tokens=1,410,624,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7348/500000] + train/ActionL1Loss=0.0483 + throughput/total_tokens=1,410,816,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7349/500000] + train/ActionL1Loss=0.0597 + throughput/total_tokens=1,411,008,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7350/500000] + train/ActionL1Loss=0.0476 + throughput/total_tokens=1,411,200,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7351/500000] + train/ActionL1Loss=0.0735 + throughput/total_tokens=1,411,392,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7352/500000] + train/ActionL1Loss=0.0594 + throughput/total_tokens=1,411,584,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7353/500000] + train/ActionL1Loss=0.0637 + throughput/total_tokens=1,411,776,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7354/500000] + train/ActionL1Loss=0.0733 + throughput/total_tokens=1,411,968,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7355/500000] + train/ActionL1Loss=0.0495 + throughput/total_tokens=1,412,160,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7356/500000] + train/ActionL1Loss=0.0783 + throughput/total_tokens=1,412,352,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7357/500000] + train/ActionL1Loss=0.0549 + throughput/total_tokens=1,412,544,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7358/500000] + train/ActionL1Loss=0.0432 + throughput/total_tokens=1,412,736,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7359/500000] + train/ActionL1Loss=0.0691 + throughput/total_tokens=1,412,928,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7360/500000] + optim/total_grad_norm=4.925 + train/ActionL1Loss=0.0794 + throughput/total_tokens=1,413,120,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=7361/500000] + train/ActionL1Loss=0.0491 + throughput/total_tokens=1,413,312,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7362/500000] + train/ActionL1Loss=0.0534 + throughput/total_tokens=1,413,504,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7363/500000] + train/ActionL1Loss=0.0627 + throughput/total_tokens=1,413,696,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7364/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=1,413,888,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7365/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=1,414,080,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7366/500000] + train/ActionL1Loss=0.0780 + throughput/total_tokens=1,414,272,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7367/500000] + train/ActionL1Loss=0.0499 + throughput/total_tokens=1,414,464,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7368/500000] + train/ActionL1Loss=0.0749 + throughput/total_tokens=1,414,656,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7369/500000] + train/ActionL1Loss=0.0755 + throughput/total_tokens=1,414,848,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7370/500000] + train/ActionL1Loss=0.0700 + throughput/total_tokens=1,415,040,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7371/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,415,232,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7372/500000] + train/ActionL1Loss=0.0717 + throughput/total_tokens=1,415,424,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7373/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=1,415,616,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7374/500000] + train/ActionL1Loss=0.0666 + throughput/total_tokens=1,415,808,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7375/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,416,000,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7376/500000] + train/ActionL1Loss=0.0714 + throughput/total_tokens=1,416,192,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7377/500000] + train/ActionL1Loss=0.0510 + throughput/total_tokens=1,416,384,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7378/500000] + train/ActionL1Loss=0.0658 + throughput/total_tokens=1,416,576,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7379/500000] + train/ActionL1Loss=0.0555 + throughput/total_tokens=1,416,768,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7380/500000] + optim/total_grad_norm=5.884 + train/ActionL1Loss=0.0973 + throughput/total_tokens=1,416,960,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7381/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=1,417,152,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7382/500000] + train/ActionL1Loss=0.0684 + throughput/total_tokens=1,417,344,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7383/500000] + train/ActionL1Loss=0.0643 + throughput/total_tokens=1,417,536,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7384/500000] + train/ActionL1Loss=0.0556 + throughput/total_tokens=1,417,728,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7385/500000] + train/ActionL1Loss=0.0706 + throughput/total_tokens=1,417,920,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7386/500000] + train/ActionL1Loss=0.0586 + throughput/total_tokens=1,418,112,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7387/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,418,304,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7388/500000] + train/ActionL1Loss=0.0664 + throughput/total_tokens=1,418,496,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7389/500000] + train/ActionL1Loss=0.0553 + throughput/total_tokens=1,418,688,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7390/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=1,418,880,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7391/500000] + train/ActionL1Loss=0.0709 + throughput/total_tokens=1,419,072,000 + throughput/device/tokens_per_second=1,126 + throughput/device/batches_per_second=0.0470 +[step=7392/500000] + train/ActionL1Loss=0.0571 + throughput/total_tokens=1,419,264,000 + throughput/device/tokens_per_second=1,126 + throughput/device/batches_per_second=0.0470 +[step=7393/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=1,419,456,000 + throughput/device/tokens_per_second=1,126 + throughput/device/batches_per_second=0.0469 +[step=7394/500000] + train/ActionL1Loss=0.0572 + throughput/total_tokens=1,419,648,000 + throughput/device/tokens_per_second=1,126 + throughput/device/batches_per_second=0.0469 +[step=7395/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=1,419,840,000 + throughput/device/tokens_per_second=1,125 + throughput/device/batches_per_second=0.0469 +[step=7396/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=1,420,032,000 + throughput/device/tokens_per_second=1,124 + throughput/device/batches_per_second=0.0469 +[step=7397/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=1,420,224,000 + throughput/device/tokens_per_second=1,124 + throughput/device/batches_per_second=0.0469 +[step=7398/500000] + train/ActionL1Loss=0.0547 + throughput/total_tokens=1,420,416,000 + throughput/device/tokens_per_second=1,124 + throughput/device/batches_per_second=0.0469 +[step=7399/500000] + train/ActionL1Loss=0.0699 + throughput/total_tokens=1,420,608,000 + throughput/device/tokens_per_second=1,125 + throughput/device/batches_per_second=0.0469 +[step=7400/500000] + optim/total_grad_norm=5.472 + train/ActionL1Loss=0.0537 + throughput/total_tokens=1,420,800,000 + throughput/device/tokens_per_second=1,125 + throughput/device/batches_per_second=0.0469 + System/Peak GPU Memory (MB)=46,917 +[step=7401/500000] + train/ActionL1Loss=0.0491 + throughput/total_tokens=1,420,992,000 + throughput/device/tokens_per_second=1,123 + throughput/device/batches_per_second=0.0468 +[step=7402/500000] + train/ActionL1Loss=0.0499 + throughput/total_tokens=1,421,184,000 + throughput/device/tokens_per_second=1,123 + throughput/device/batches_per_second=0.0468 +[step=7403/500000] + train/ActionL1Loss=0.0702 + throughput/total_tokens=1,421,376,000 + throughput/device/tokens_per_second=1,123 + throughput/device/batches_per_second=0.0468 +[step=7404/500000] + train/ActionL1Loss=0.0408 + throughput/total_tokens=1,421,568,000 + throughput/device/tokens_per_second=1,123 + throughput/device/batches_per_second=0.0468 +[step=7405/500000] + train/ActionL1Loss=0.0532 + throughput/total_tokens=1,421,760,000 + throughput/device/tokens_per_second=1,123 + throughput/device/batches_per_second=0.0468 +[step=7406/500000] + train/ActionL1Loss=0.0570 + throughput/total_tokens=1,421,952,000 + throughput/device/tokens_per_second=1,123 + throughput/device/batches_per_second=0.0468 +[step=7407/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,422,144,000 + throughput/device/tokens_per_second=1,123 + throughput/device/batches_per_second=0.0468 +[step=7408/500000] + train/ActionL1Loss=0.0509 + throughput/total_tokens=1,422,336,000 + throughput/device/tokens_per_second=1,123 + throughput/device/batches_per_second=0.0468 +[step=7409/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,422,528,000 + throughput/device/tokens_per_second=1,122 + throughput/device/batches_per_second=0.0468 +[step=7410/500000] + train/ActionL1Loss=0.0512 + throughput/total_tokens=1,422,720,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=7411/500000] + train/ActionL1Loss=0.0550 + throughput/total_tokens=1,422,912,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7412/500000] + train/ActionL1Loss=0.0580 + throughput/total_tokens=1,423,104,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7413/500000] + train/ActionL1Loss=0.0555 + throughput/total_tokens=1,423,296,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7414/500000] + train/ActionL1Loss=0.0503 + throughput/total_tokens=1,423,488,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7415/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,423,680,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7416/500000] + train/ActionL1Loss=0.0606 + throughput/total_tokens=1,423,872,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7417/500000] + train/ActionL1Loss=0.0599 + throughput/total_tokens=1,424,064,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7418/500000] + train/ActionL1Loss=0.0629 + throughput/total_tokens=1,424,256,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7419/500000] + train/ActionL1Loss=0.0738 + throughput/total_tokens=1,424,448,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7420/500000] + optim/total_grad_norm=6.026 + train/ActionL1Loss=0.0700 + throughput/total_tokens=1,424,640,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7421/500000] + train/ActionL1Loss=0.0503 + throughput/total_tokens=1,424,832,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7422/500000] + train/ActionL1Loss=0.0507 + throughput/total_tokens=1,425,024,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7423/500000] + train/ActionL1Loss=0.0592 + throughput/total_tokens=1,425,216,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7424/500000] + train/ActionL1Loss=0.0606 + throughput/total_tokens=1,425,408,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7425/500000] + train/ActionL1Loss=0.0744 + throughput/total_tokens=1,425,600,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7426/500000] + train/ActionL1Loss=0.0592 + throughput/total_tokens=1,425,792,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7427/500000] + train/ActionL1Loss=0.0689 + throughput/total_tokens=1,425,984,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7428/500000] + train/ActionL1Loss=0.0774 + throughput/total_tokens=1,426,176,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7429/500000] + train/ActionL1Loss=0.0568 + throughput/total_tokens=1,426,368,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7430/500000] + train/ActionL1Loss=0.0506 + throughput/total_tokens=1,426,560,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7431/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=1,426,752,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7432/500000] + train/ActionL1Loss=0.0560 + throughput/total_tokens=1,426,944,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7433/500000] + train/ActionL1Loss=0.0698 + throughput/total_tokens=1,427,136,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7434/500000] + train/ActionL1Loss=0.0617 + throughput/total_tokens=1,427,328,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7435/500000] + train/ActionL1Loss=0.0714 + throughput/total_tokens=1,427,520,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7436/500000] + train/ActionL1Loss=0.0598 + throughput/total_tokens=1,427,712,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7437/500000] + train/ActionL1Loss=0.0627 + throughput/total_tokens=1,427,904,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7438/500000] + train/ActionL1Loss=0.0539 + throughput/total_tokens=1,428,096,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7439/500000] + train/ActionL1Loss=0.0603 + throughput/total_tokens=1,428,288,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7440/500000] + optim/total_grad_norm=4.235 + train/ActionL1Loss=0.0789 + throughput/total_tokens=1,428,480,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7441/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=1,428,672,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7442/500000] + train/ActionL1Loss=0.0666 + throughput/total_tokens=1,428,864,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7443/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,429,056,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7444/500000] + train/ActionL1Loss=0.0717 + throughput/total_tokens=1,429,248,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7445/500000] + train/ActionL1Loss=0.0862 + throughput/total_tokens=1,429,440,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7446/500000] + train/ActionL1Loss=0.0727 + throughput/total_tokens=1,429,632,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7447/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=1,429,824,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7448/500000] + train/ActionL1Loss=0.0713 + throughput/total_tokens=1,430,016,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7449/500000] + train/ActionL1Loss=0.0826 + throughput/total_tokens=1,430,208,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7450/500000] + train/ActionL1Loss=0.0581 + throughput/total_tokens=1,430,400,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7451/500000] + train/ActionL1Loss=0.0513 + throughput/total_tokens=1,430,592,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7452/500000] + train/ActionL1Loss=0.0477 + throughput/total_tokens=1,430,784,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=7453/500000] + train/ActionL1Loss=0.0706 + throughput/total_tokens=1,430,976,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7454/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,431,168,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7455/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,431,360,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=7456/500000] + train/ActionL1Loss=0.0635 + throughput/total_tokens=1,431,552,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7457/500000] + train/ActionL1Loss=0.0646 + throughput/total_tokens=1,431,744,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7458/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=1,431,936,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=7459/500000] + train/ActionL1Loss=0.0662 + throughput/total_tokens=1,432,128,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=7460/500000] + optim/total_grad_norm=6.887 + train/ActionL1Loss=0.0576 + throughput/total_tokens=1,432,320,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=7461/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,432,512,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=7462/500000] + train/ActionL1Loss=0.0565 + throughput/total_tokens=1,432,704,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=7463/500000] + train/ActionL1Loss=0.0821 + throughput/total_tokens=1,432,896,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=7464/500000] + train/ActionL1Loss=0.0551 + throughput/total_tokens=1,433,088,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=7465/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,433,280,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7466/500000] + train/ActionL1Loss=0.0754 + throughput/total_tokens=1,433,472,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7467/500000] + train/ActionL1Loss=0.0753 + throughput/total_tokens=1,433,664,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7468/500000] + train/ActionL1Loss=0.0691 + throughput/total_tokens=1,433,856,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7469/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=1,434,048,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7470/500000] + train/ActionL1Loss=0.0809 + throughput/total_tokens=1,434,240,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7471/500000] + train/ActionL1Loss=0.0745 + throughput/total_tokens=1,434,432,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7472/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=1,434,624,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7473/500000] + train/ActionL1Loss=0.0593 + throughput/total_tokens=1,434,816,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7474/500000] + train/ActionL1Loss=0.0650 + throughput/total_tokens=1,435,008,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7475/500000] + train/ActionL1Loss=0.0690 + throughput/total_tokens=1,435,200,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7476/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,435,392,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7477/500000] + train/ActionL1Loss=0.0580 + throughput/total_tokens=1,435,584,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7478/500000] + train/ActionL1Loss=0.0690 + throughput/total_tokens=1,435,776,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7479/500000] + train/ActionL1Loss=0.0709 + throughput/total_tokens=1,435,968,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7480/500000] + optim/total_grad_norm=5.998 + train/ActionL1Loss=0.0531 + throughput/total_tokens=1,436,160,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7481/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=1,436,352,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7482/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,436,544,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7483/500000] + train/ActionL1Loss=0.0521 + throughput/total_tokens=1,436,736,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7484/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,436,928,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7485/500000] + train/ActionL1Loss=0.0524 + throughput/total_tokens=1,437,120,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7486/500000] + train/ActionL1Loss=0.0577 + throughput/total_tokens=1,437,312,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7487/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=1,437,504,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7488/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=1,437,696,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7489/500000] + train/ActionL1Loss=0.0590 + throughput/total_tokens=1,437,888,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7490/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=1,438,080,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7491/500000] + train/ActionL1Loss=0.0590 + throughput/total_tokens=1,438,272,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7492/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,438,464,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7493/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=1,438,656,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7494/500000] + train/ActionL1Loss=0.0672 + throughput/total_tokens=1,438,848,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7495/500000] + train/ActionL1Loss=0.0459 + throughput/total_tokens=1,439,040,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7496/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=1,439,232,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7497/500000] + train/ActionL1Loss=0.0521 + throughput/total_tokens=1,439,424,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7498/500000] + train/ActionL1Loss=0.0471 + throughput/total_tokens=1,439,616,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7499/500000] + train/ActionL1Loss=0.0868 + throughput/total_tokens=1,439,808,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7500/500000] + optim/total_grad_norm=7.588 + train/ActionL1Loss=0.0482 + throughput/total_tokens=1,440,000,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +10/02 [09:22:51] INFO | >> Saving config... checkpoint.py:608 +10/02 [09:23:31] INFO | >> Saving model state... checkpoint.py:796 +10/02 [09:24:39] INFO | >> Saving optim state... checkpoint.py:811 +10/02 [09:26:13] INFO | >> Saving trainer state... checkpoint.py:670 + INFO | >> Saving config... checkpoint.py:608 +[step=7501/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=1,440,192,000 + throughput/device/tokens_per_second=1,213 + throughput/device/batches_per_second=0.0506 +[step=7502/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,440,384,000 + throughput/device/tokens_per_second=1,189 + throughput/device/batches_per_second=0.0496 +[step=7503/500000] + train/ActionL1Loss=0.0445 + throughput/total_tokens=1,440,576,000 + throughput/device/tokens_per_second=1,183 + throughput/device/batches_per_second=0.0493 +[step=7504/500000] + train/ActionL1Loss=0.0583 + throughput/total_tokens=1,440,768,000 + throughput/device/tokens_per_second=1,179 + throughput/device/batches_per_second=0.0492 +[step=7505/500000] + train/ActionL1Loss=0.0413 + throughput/total_tokens=1,440,960,000 + throughput/device/tokens_per_second=1,177 + throughput/device/batches_per_second=0.0491 +[step=7506/500000] + train/ActionL1Loss=0.0687 + throughput/total_tokens=1,441,152,000 + throughput/device/tokens_per_second=1,176 + throughput/device/batches_per_second=0.0490 +[step=7507/500000] + train/ActionL1Loss=0.0657 + throughput/total_tokens=1,441,344,000 + throughput/device/tokens_per_second=1,175 + throughput/device/batches_per_second=0.0490 +[step=7508/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,441,536,000 + throughput/device/tokens_per_second=1,174 + throughput/device/batches_per_second=0.0490 +[step=7509/500000] + train/ActionL1Loss=0.0599 + throughput/total_tokens=1,441,728,000 + throughput/device/tokens_per_second=1,173 + throughput/device/batches_per_second=0.0489 +[step=7510/500000] + train/ActionL1Loss=0.0512 + throughput/total_tokens=1,441,920,000 + throughput/device/tokens_per_second=1,173 + throughput/device/batches_per_second=0.0489 + System/Peak GPU Memory (MB)=46,917 +[step=7511/500000] + train/ActionL1Loss=0.0547 + throughput/total_tokens=1,442,112,000 + throughput/device/tokens_per_second=1,172 + throughput/device/batches_per_second=0.0489 +[step=7512/500000] + train/ActionL1Loss=0.0468 + throughput/total_tokens=1,442,304,000 + throughput/device/tokens_per_second=1,172 + throughput/device/batches_per_second=0.0489 +[step=7513/500000] + train/ActionL1Loss=0.0584 + throughput/total_tokens=1,442,496,000 + throughput/device/tokens_per_second=1,172 + throughput/device/batches_per_second=0.0488 +[step=7514/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,442,688,000 + throughput/device/tokens_per_second=1,171 + throughput/device/batches_per_second=0.0488 +[step=7515/500000] + train/ActionL1Loss=0.0709 + throughput/total_tokens=1,442,880,000 + throughput/device/tokens_per_second=1,171 + throughput/device/batches_per_second=0.0488 +[step=7516/500000] + train/ActionL1Loss=0.0578 + throughput/total_tokens=1,443,072,000 + throughput/device/tokens_per_second=1,171 + throughput/device/batches_per_second=0.0488 +[step=7517/500000] + train/ActionL1Loss=0.0410 + throughput/total_tokens=1,443,264,000 + throughput/device/tokens_per_second=1,171 + throughput/device/batches_per_second=0.0488 +[step=7518/500000] + train/ActionL1Loss=0.0570 + throughput/total_tokens=1,443,456,000 + throughput/device/tokens_per_second=1,171 + throughput/device/batches_per_second=0.0488 +[step=7519/500000] + train/ActionL1Loss=0.0582 + throughput/total_tokens=1,443,648,000 + throughput/device/tokens_per_second=1,171 + throughput/device/batches_per_second=0.0488 +[step=7520/500000] + optim/total_grad_norm=6.610 + train/ActionL1Loss=0.0537 + throughput/total_tokens=1,443,840,000 + throughput/device/tokens_per_second=1,170 + throughput/device/batches_per_second=0.0488 + System/Peak GPU Memory (MB)=46,917 +[step=7521/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=1,444,032,000 + throughput/device/tokens_per_second=1,170 + throughput/device/batches_per_second=0.0488 +[step=7522/500000] + train/ActionL1Loss=0.0676 + throughput/total_tokens=1,444,224,000 + throughput/device/tokens_per_second=1,170 + throughput/device/batches_per_second=0.0488 +[step=7523/500000] + train/ActionL1Loss=0.0630 + throughput/total_tokens=1,444,416,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7524/500000] + train/ActionL1Loss=0.0701 + throughput/total_tokens=1,444,608,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7525/500000] + train/ActionL1Loss=0.0736 + throughput/total_tokens=1,444,800,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7526/500000] + train/ActionL1Loss=0.0494 + throughput/total_tokens=1,444,992,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7527/500000] + train/ActionL1Loss=0.0507 + throughput/total_tokens=1,445,184,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7528/500000] + train/ActionL1Loss=0.0649 + throughput/total_tokens=1,445,376,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7529/500000] + train/ActionL1Loss=0.0620 + throughput/total_tokens=1,445,568,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7530/500000] + train/ActionL1Loss=0.0603 + throughput/total_tokens=1,445,760,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7531/500000] + train/ActionL1Loss=0.0596 + throughput/total_tokens=1,445,952,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7532/500000] + train/ActionL1Loss=0.0681 + throughput/total_tokens=1,446,144,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7533/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=1,446,336,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7534/500000] + train/ActionL1Loss=0.0568 + throughput/total_tokens=1,446,528,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7535/500000] + train/ActionL1Loss=0.0686 + throughput/total_tokens=1,446,720,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7536/500000] + train/ActionL1Loss=0.0522 + throughput/total_tokens=1,446,912,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7537/500000] + train/ActionL1Loss=0.0578 + throughput/total_tokens=1,447,104,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7538/500000] + train/ActionL1Loss=0.0501 + throughput/total_tokens=1,447,296,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7539/500000] + train/ActionL1Loss=0.0621 + throughput/total_tokens=1,447,488,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7540/500000] + optim/total_grad_norm=8.261 + train/ActionL1Loss=0.0635 + throughput/total_tokens=1,447,680,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7541/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,447,872,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7542/500000] + train/ActionL1Loss=0.0725 + throughput/total_tokens=1,448,064,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7543/500000] + train/ActionL1Loss=0.0542 + throughput/total_tokens=1,448,256,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7544/500000] + train/ActionL1Loss=0.0586 + throughput/total_tokens=1,448,448,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7545/500000] + train/ActionL1Loss=0.0674 + throughput/total_tokens=1,448,640,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7546/500000] + train/ActionL1Loss=0.0672 + throughput/total_tokens=1,448,832,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7547/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=1,449,024,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7548/500000] + train/ActionL1Loss=0.0753 + throughput/total_tokens=1,449,216,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7549/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=1,449,408,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7550/500000] + train/ActionL1Loss=0.0430 + throughput/total_tokens=1,449,600,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7551/500000] + train/ActionL1Loss=0.0591 + throughput/total_tokens=1,449,792,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7552/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=1,449,984,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7553/500000] + train/ActionL1Loss=0.0576 + throughput/total_tokens=1,450,176,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7554/500000] + train/ActionL1Loss=0.0549 + throughput/total_tokens=1,450,368,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7555/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,450,560,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7556/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=1,450,752,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7557/500000] + train/ActionL1Loss=0.0657 + throughput/total_tokens=1,450,944,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7558/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=1,451,136,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7559/500000] + train/ActionL1Loss=0.0635 + throughput/total_tokens=1,451,328,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7560/500000] + optim/total_grad_norm=6.637 + train/ActionL1Loss=0.0563 + throughput/total_tokens=1,451,520,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=7561/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=1,451,712,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7562/500000] + train/ActionL1Loss=0.0484 + throughput/total_tokens=1,451,904,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7563/500000] + train/ActionL1Loss=0.0510 + throughput/total_tokens=1,452,096,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7564/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=1,452,288,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7565/500000] + train/ActionL1Loss=0.1020 + throughput/total_tokens=1,452,480,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7566/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,452,672,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7567/500000] + train/ActionL1Loss=0.0517 + throughput/total_tokens=1,452,864,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7568/500000] + train/ActionL1Loss=0.0717 + throughput/total_tokens=1,453,056,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7569/500000] + train/ActionL1Loss=0.0574 + throughput/total_tokens=1,453,248,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7570/500000] + train/ActionL1Loss=0.0721 + throughput/total_tokens=1,453,440,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7571/500000] + train/ActionL1Loss=0.0817 + throughput/total_tokens=1,453,632,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7572/500000] + train/ActionL1Loss=0.0703 + throughput/total_tokens=1,453,824,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7573/500000] + train/ActionL1Loss=0.0660 + throughput/total_tokens=1,454,016,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7574/500000] + train/ActionL1Loss=0.0809 + throughput/total_tokens=1,454,208,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7575/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=1,454,400,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7576/500000] + train/ActionL1Loss=0.0573 + throughput/total_tokens=1,454,592,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7577/500000] + train/ActionL1Loss=0.0703 + throughput/total_tokens=1,454,784,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7578/500000] + train/ActionL1Loss=0.0730 + throughput/total_tokens=1,454,976,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7579/500000] + train/ActionL1Loss=0.0820 + throughput/total_tokens=1,455,168,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7580/500000] + optim/total_grad_norm=6.462 + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,455,360,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7581/500000] + train/ActionL1Loss=0.0586 + throughput/total_tokens=1,455,552,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7582/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=1,455,744,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7583/500000] + train/ActionL1Loss=0.0853 + throughput/total_tokens=1,455,936,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7584/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=1,456,128,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7585/500000] + train/ActionL1Loss=0.0508 + throughput/total_tokens=1,456,320,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7586/500000] + train/ActionL1Loss=0.0545 + throughput/total_tokens=1,456,512,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7587/500000] + train/ActionL1Loss=0.0732 + throughput/total_tokens=1,456,704,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7588/500000] + train/ActionL1Loss=0.0876 + throughput/total_tokens=1,456,896,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7589/500000] + train/ActionL1Loss=0.0570 + throughput/total_tokens=1,457,088,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7590/500000] + train/ActionL1Loss=0.0744 + throughput/total_tokens=1,457,280,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7591/500000] + train/ActionL1Loss=0.0608 + throughput/total_tokens=1,457,472,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7592/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=1,457,664,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7593/500000] + train/ActionL1Loss=0.0403 + throughput/total_tokens=1,457,856,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7594/500000] + train/ActionL1Loss=0.0516 + throughput/total_tokens=1,458,048,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7595/500000] + train/ActionL1Loss=0.0603 + throughput/total_tokens=1,458,240,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7596/500000] + train/ActionL1Loss=0.0520 + throughput/total_tokens=1,458,432,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7597/500000] + train/ActionL1Loss=0.0756 + throughput/total_tokens=1,458,624,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7598/500000] + train/ActionL1Loss=0.0653 + throughput/total_tokens=1,458,816,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7599/500000] + train/ActionL1Loss=0.0653 + throughput/total_tokens=1,459,008,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7600/500000] + optim/total_grad_norm=7.400 + train/ActionL1Loss=0.0516 + throughput/total_tokens=1,459,200,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7601/500000] + train/ActionL1Loss=0.0710 + throughput/total_tokens=1,459,392,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7602/500000] + train/ActionL1Loss=0.0507 + throughput/total_tokens=1,459,584,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7603/500000] + train/ActionL1Loss=0.0435 + throughput/total_tokens=1,459,776,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7604/500000] + train/ActionL1Loss=0.0606 + throughput/total_tokens=1,459,968,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7605/500000] + train/ActionL1Loss=0.0697 + throughput/total_tokens=1,460,160,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7606/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=1,460,352,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7607/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=1,460,544,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7608/500000] + train/ActionL1Loss=0.0800 + throughput/total_tokens=1,460,736,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7609/500000] + train/ActionL1Loss=0.0532 + throughput/total_tokens=1,460,928,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7610/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=1,461,120,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7611/500000] + train/ActionL1Loss=0.0539 + throughput/total_tokens=1,461,312,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7612/500000] + train/ActionL1Loss=0.0560 + throughput/total_tokens=1,461,504,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7613/500000] + train/ActionL1Loss=0.0768 + throughput/total_tokens=1,461,696,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=7614/500000] + train/ActionL1Loss=0.0886 + throughput/total_tokens=1,461,888,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7615/500000] + train/ActionL1Loss=0.0716 + throughput/total_tokens=1,462,080,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7616/500000] + train/ActionL1Loss=0.0491 + throughput/total_tokens=1,462,272,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7617/500000] + train/ActionL1Loss=0.0537 + throughput/total_tokens=1,462,464,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7618/500000] + train/ActionL1Loss=0.0606 + throughput/total_tokens=1,462,656,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7619/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=1,462,848,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7620/500000] + optim/total_grad_norm=6.446 + train/ActionL1Loss=0.0516 + throughput/total_tokens=1,463,040,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7621/500000] + train/ActionL1Loss=0.0559 + throughput/total_tokens=1,463,232,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7622/500000] + train/ActionL1Loss=0.0704 + throughput/total_tokens=1,463,424,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7623/500000] + train/ActionL1Loss=0.0672 + throughput/total_tokens=1,463,616,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7624/500000] + train/ActionL1Loss=0.0572 + throughput/total_tokens=1,463,808,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7625/500000] + train/ActionL1Loss=0.0568 + throughput/total_tokens=1,464,000,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7626/500000] + train/ActionL1Loss=0.0687 + throughput/total_tokens=1,464,192,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7627/500000] + train/ActionL1Loss=0.0469 + throughput/total_tokens=1,464,384,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7628/500000] + train/ActionL1Loss=0.0617 + throughput/total_tokens=1,464,576,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7629/500000] + train/ActionL1Loss=0.0717 + throughput/total_tokens=1,464,768,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7630/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,464,960,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7631/500000] + train/ActionL1Loss=0.0639 + throughput/total_tokens=1,465,152,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7632/500000] + train/ActionL1Loss=0.0911 + throughput/total_tokens=1,465,344,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7633/500000] + train/ActionL1Loss=0.0605 + throughput/total_tokens=1,465,536,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7634/500000] + train/ActionL1Loss=0.0649 + throughput/total_tokens=1,465,728,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7635/500000] + train/ActionL1Loss=0.0489 + throughput/total_tokens=1,465,920,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7636/500000] + train/ActionL1Loss=0.0686 + throughput/total_tokens=1,466,112,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7637/500000] + train/ActionL1Loss=0.0648 + throughput/total_tokens=1,466,304,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7638/500000] + train/ActionL1Loss=0.0482 + throughput/total_tokens=1,466,496,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7639/500000] + train/ActionL1Loss=0.0707 + throughput/total_tokens=1,466,688,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7640/500000] + optim/total_grad_norm=6.145 + train/ActionL1Loss=0.0471 + throughput/total_tokens=1,466,880,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7641/500000] + train/ActionL1Loss=0.0590 + throughput/total_tokens=1,467,072,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7642/500000] + train/ActionL1Loss=0.0662 + throughput/total_tokens=1,467,264,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7643/500000] + train/ActionL1Loss=0.0586 + throughput/total_tokens=1,467,456,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7644/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=1,467,648,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7645/500000] + train/ActionL1Loss=0.0626 + throughput/total_tokens=1,467,840,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7646/500000] + train/ActionL1Loss=0.0656 + throughput/total_tokens=1,468,032,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7647/500000] + train/ActionL1Loss=0.0595 + throughput/total_tokens=1,468,224,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7648/500000] + train/ActionL1Loss=0.0520 + throughput/total_tokens=1,468,416,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7649/500000] + train/ActionL1Loss=0.0681 + throughput/total_tokens=1,468,608,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7650/500000] + train/ActionL1Loss=0.0493 + throughput/total_tokens=1,468,800,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7651/500000] + train/ActionL1Loss=0.0629 + throughput/total_tokens=1,468,992,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7652/500000] + train/ActionL1Loss=0.0788 + throughput/total_tokens=1,469,184,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7653/500000] + train/ActionL1Loss=0.0714 + throughput/total_tokens=1,469,376,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7654/500000] + train/ActionL1Loss=0.0702 + throughput/total_tokens=1,469,568,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=7655/500000] + train/ActionL1Loss=0.0478 + throughput/total_tokens=1,469,760,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=7656/500000] + train/ActionL1Loss=0.0596 + throughput/total_tokens=1,469,952,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=7657/500000] + train/ActionL1Loss=0.0641 + throughput/total_tokens=1,470,144,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7658/500000] + train/ActionL1Loss=0.0727 + throughput/total_tokens=1,470,336,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7659/500000] + train/ActionL1Loss=0.0690 + throughput/total_tokens=1,470,528,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7660/500000] + optim/total_grad_norm=6.801 + train/ActionL1Loss=0.0614 + throughput/total_tokens=1,470,720,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=7661/500000] + train/ActionL1Loss=0.0421 + throughput/total_tokens=1,470,912,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7662/500000] + train/ActionL1Loss=0.0465 + throughput/total_tokens=1,471,104,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7663/500000] + train/ActionL1Loss=0.0729 + throughput/total_tokens=1,471,296,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7664/500000] + train/ActionL1Loss=0.0478 + throughput/total_tokens=1,471,488,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7665/500000] + train/ActionL1Loss=0.0568 + throughput/total_tokens=1,471,680,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7666/500000] + train/ActionL1Loss=0.0549 + throughput/total_tokens=1,471,872,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7667/500000] + train/ActionL1Loss=0.0499 + throughput/total_tokens=1,472,064,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7668/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=1,472,256,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7669/500000] + train/ActionL1Loss=0.0559 + throughput/total_tokens=1,472,448,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7670/500000] + train/ActionL1Loss=0.0393 + throughput/total_tokens=1,472,640,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7671/500000] + train/ActionL1Loss=0.0521 + throughput/total_tokens=1,472,832,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7672/500000] + train/ActionL1Loss=0.0557 + throughput/total_tokens=1,473,024,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7673/500000] + train/ActionL1Loss=0.0596 + throughput/total_tokens=1,473,216,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7674/500000] + train/ActionL1Loss=0.0689 + throughput/total_tokens=1,473,408,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7675/500000] + train/ActionL1Loss=0.0675 + throughput/total_tokens=1,473,600,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7676/500000] + train/ActionL1Loss=0.0552 + throughput/total_tokens=1,473,792,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7677/500000] + train/ActionL1Loss=0.0570 + throughput/total_tokens=1,473,984,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7678/500000] + train/ActionL1Loss=0.0626 + throughput/total_tokens=1,474,176,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7679/500000] + train/ActionL1Loss=0.0556 + throughput/total_tokens=1,474,368,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7680/500000] + optim/total_grad_norm=8.626 + train/ActionL1Loss=0.0616 + throughput/total_tokens=1,474,560,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7681/500000] + train/ActionL1Loss=0.0683 + throughput/total_tokens=1,474,752,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7682/500000] + train/ActionL1Loss=0.0620 + throughput/total_tokens=1,474,944,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7683/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,475,136,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7684/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=1,475,328,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7685/500000] + train/ActionL1Loss=0.0753 + throughput/total_tokens=1,475,520,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7686/500000] + train/ActionL1Loss=0.0629 + throughput/total_tokens=1,475,712,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7687/500000] + train/ActionL1Loss=0.0472 + throughput/total_tokens=1,475,904,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7688/500000] + train/ActionL1Loss=0.0409 + throughput/total_tokens=1,476,096,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7689/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,476,288,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7690/500000] + train/ActionL1Loss=0.0518 + throughput/total_tokens=1,476,480,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7691/500000] + train/ActionL1Loss=0.0513 + throughput/total_tokens=1,476,672,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7692/500000] + train/ActionL1Loss=0.0677 + throughput/total_tokens=1,476,864,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7693/500000] + train/ActionL1Loss=0.0485 + throughput/total_tokens=1,477,056,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7694/500000] + train/ActionL1Loss=0.0474 + throughput/total_tokens=1,477,248,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7695/500000] + train/ActionL1Loss=0.0491 + throughput/total_tokens=1,477,440,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7696/500000] + train/ActionL1Loss=0.0614 + throughput/total_tokens=1,477,632,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7697/500000] + train/ActionL1Loss=0.0582 + throughput/total_tokens=1,477,824,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7698/500000] + train/ActionL1Loss=0.0498 + throughput/total_tokens=1,478,016,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7699/500000] + train/ActionL1Loss=0.0457 + throughput/total_tokens=1,478,208,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7700/500000] + optim/total_grad_norm=5.611 + train/ActionL1Loss=0.0608 + throughput/total_tokens=1,478,400,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7701/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,478,592,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7702/500000] + train/ActionL1Loss=0.0760 + throughput/total_tokens=1,478,784,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7703/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=1,478,976,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7704/500000] + train/ActionL1Loss=0.0508 + throughput/total_tokens=1,479,168,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7705/500000] + train/ActionL1Loss=0.0594 + throughput/total_tokens=1,479,360,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7706/500000] + train/ActionL1Loss=0.0585 + throughput/total_tokens=1,479,552,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7707/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=1,479,744,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7708/500000] + train/ActionL1Loss=0.0596 + throughput/total_tokens=1,479,936,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7709/500000] + train/ActionL1Loss=0.0585 + throughput/total_tokens=1,480,128,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7710/500000] + train/ActionL1Loss=0.0606 + throughput/total_tokens=1,480,320,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7711/500000] + train/ActionL1Loss=0.0529 + throughput/total_tokens=1,480,512,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7712/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=1,480,704,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7713/500000] + train/ActionL1Loss=0.0516 + throughput/total_tokens=1,480,896,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7714/500000] + train/ActionL1Loss=0.0528 + throughput/total_tokens=1,481,088,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7715/500000] + train/ActionL1Loss=0.0806 + throughput/total_tokens=1,481,280,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7716/500000] + train/ActionL1Loss=0.0709 + throughput/total_tokens=1,481,472,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7717/500000] + train/ActionL1Loss=0.0666 + throughput/total_tokens=1,481,664,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7718/500000] + train/ActionL1Loss=0.0859 + throughput/total_tokens=1,481,856,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7719/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,482,048,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7720/500000] + optim/total_grad_norm=4.346 + train/ActionL1Loss=0.0449 + throughput/total_tokens=1,482,240,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7721/500000] + train/ActionL1Loss=0.0656 + throughput/total_tokens=1,482,432,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7722/500000] + train/ActionL1Loss=0.0875 + throughput/total_tokens=1,482,624,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7723/500000] + train/ActionL1Loss=0.0573 + throughput/total_tokens=1,482,816,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7724/500000] + train/ActionL1Loss=0.0586 + throughput/total_tokens=1,483,008,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7725/500000] + train/ActionL1Loss=0.0546 + throughput/total_tokens=1,483,200,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7726/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,483,392,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7727/500000] + train/ActionL1Loss=0.0598 + throughput/total_tokens=1,483,584,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7728/500000] + train/ActionL1Loss=0.0745 + throughput/total_tokens=1,483,776,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7729/500000] + train/ActionL1Loss=0.0795 + throughput/total_tokens=1,483,968,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7730/500000] + train/ActionL1Loss=0.0641 + throughput/total_tokens=1,484,160,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7731/500000] + train/ActionL1Loss=0.0664 + throughput/total_tokens=1,484,352,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7732/500000] + train/ActionL1Loss=0.0543 + throughput/total_tokens=1,484,544,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7733/500000] + train/ActionL1Loss=0.0419 + throughput/total_tokens=1,484,736,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7734/500000] + train/ActionL1Loss=0.0578 + throughput/total_tokens=1,484,928,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7735/500000] + train/ActionL1Loss=0.0655 + throughput/total_tokens=1,485,120,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7736/500000] + train/ActionL1Loss=0.0881 + throughput/total_tokens=1,485,312,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7737/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,485,504,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7738/500000] + train/ActionL1Loss=0.0504 + throughput/total_tokens=1,485,696,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7739/500000] + train/ActionL1Loss=0.0471 + throughput/total_tokens=1,485,888,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7740/500000] + optim/total_grad_norm=5.096 + train/ActionL1Loss=0.0940 + throughput/total_tokens=1,486,080,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7741/500000] + train/ActionL1Loss=0.0560 + throughput/total_tokens=1,486,272,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7742/500000] + train/ActionL1Loss=0.0656 + throughput/total_tokens=1,486,464,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7743/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,486,656,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7744/500000] + train/ActionL1Loss=0.0582 + throughput/total_tokens=1,486,848,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7745/500000] + train/ActionL1Loss=0.0572 + throughput/total_tokens=1,487,040,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7746/500000] + train/ActionL1Loss=0.0690 + throughput/total_tokens=1,487,232,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7747/500000] + train/ActionL1Loss=0.0481 + throughput/total_tokens=1,487,424,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7748/500000] + train/ActionL1Loss=0.0548 + throughput/total_tokens=1,487,616,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7749/500000] + train/ActionL1Loss=0.0579 + throughput/total_tokens=1,487,808,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7750/500000] + train/ActionL1Loss=0.0545 + throughput/total_tokens=1,488,000,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7751/500000] + train/ActionL1Loss=0.0652 + throughput/total_tokens=1,488,192,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=7752/500000] + train/ActionL1Loss=0.0873 + throughput/total_tokens=1,488,384,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=7753/500000] + train/ActionL1Loss=0.0504 + throughput/total_tokens=1,488,576,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7754/500000] + train/ActionL1Loss=0.0529 + throughput/total_tokens=1,488,768,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7755/500000] + train/ActionL1Loss=0.0528 + throughput/total_tokens=1,488,960,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7756/500000] + train/ActionL1Loss=0.0538 + throughput/total_tokens=1,489,152,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7757/500000] + train/ActionL1Loss=0.0552 + throughput/total_tokens=1,489,344,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7758/500000] + train/ActionL1Loss=0.0534 + throughput/total_tokens=1,489,536,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7759/500000] + train/ActionL1Loss=0.0522 + throughput/total_tokens=1,489,728,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7760/500000] + optim/total_grad_norm=7.822 + train/ActionL1Loss=0.0570 + throughput/total_tokens=1,489,920,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=7761/500000] + train/ActionL1Loss=0.0830 + throughput/total_tokens=1,490,112,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7762/500000] + train/ActionL1Loss=0.0686 + throughput/total_tokens=1,490,304,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7763/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,490,496,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7764/500000] + train/ActionL1Loss=0.0601 + throughput/total_tokens=1,490,688,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7765/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=1,490,880,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7766/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,491,072,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7767/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=1,491,264,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7768/500000] + train/ActionL1Loss=0.0468 + throughput/total_tokens=1,491,456,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7769/500000] + train/ActionL1Loss=0.0456 + throughput/total_tokens=1,491,648,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7770/500000] + train/ActionL1Loss=0.0517 + throughput/total_tokens=1,491,840,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7771/500000] + train/ActionL1Loss=0.0501 + throughput/total_tokens=1,492,032,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7772/500000] + train/ActionL1Loss=0.0736 + throughput/total_tokens=1,492,224,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7773/500000] + train/ActionL1Loss=0.0419 + throughput/total_tokens=1,492,416,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7774/500000] + train/ActionL1Loss=0.0560 + throughput/total_tokens=1,492,608,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7775/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,492,800,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7776/500000] + train/ActionL1Loss=0.0639 + throughput/total_tokens=1,492,992,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7777/500000] + train/ActionL1Loss=0.0652 + throughput/total_tokens=1,493,184,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7778/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,493,376,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7779/500000] + train/ActionL1Loss=0.0556 + throughput/total_tokens=1,493,568,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7780/500000] + optim/total_grad_norm=6.279 + train/ActionL1Loss=0.0447 + throughput/total_tokens=1,493,760,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7781/500000] + train/ActionL1Loss=0.0787 + throughput/total_tokens=1,493,952,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7782/500000] + train/ActionL1Loss=0.0723 + throughput/total_tokens=1,494,144,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7783/500000] + train/ActionL1Loss=0.0611 + throughput/total_tokens=1,494,336,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7784/500000] + train/ActionL1Loss=0.0448 + throughput/total_tokens=1,494,528,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7785/500000] + train/ActionL1Loss=0.0565 + throughput/total_tokens=1,494,720,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7786/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=1,494,912,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7787/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=1,495,104,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7788/500000] + train/ActionL1Loss=0.0637 + throughput/total_tokens=1,495,296,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7789/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=1,495,488,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7790/500000] + train/ActionL1Loss=0.0565 + throughput/total_tokens=1,495,680,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7791/500000] + train/ActionL1Loss=0.0498 + throughput/total_tokens=1,495,872,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7792/500000] + train/ActionL1Loss=0.0592 + throughput/total_tokens=1,496,064,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7793/500000] + train/ActionL1Loss=0.0529 + throughput/total_tokens=1,496,256,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7794/500000] + train/ActionL1Loss=0.0698 + throughput/total_tokens=1,496,448,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7795/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=1,496,640,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7796/500000] + train/ActionL1Loss=0.0598 + throughput/total_tokens=1,496,832,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7797/500000] + train/ActionL1Loss=0.0501 + throughput/total_tokens=1,497,024,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7798/500000] + train/ActionL1Loss=0.0530 + throughput/total_tokens=1,497,216,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7799/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=1,497,408,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7800/500000] + optim/total_grad_norm=5.378 + train/ActionL1Loss=0.0637 + throughput/total_tokens=1,497,600,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7801/500000] + train/ActionL1Loss=0.0505 + throughput/total_tokens=1,497,792,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7802/500000] + train/ActionL1Loss=0.0592 + throughput/total_tokens=1,497,984,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7803/500000] + train/ActionL1Loss=0.0646 + throughput/total_tokens=1,498,176,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7804/500000] + train/ActionL1Loss=0.0635 + throughput/total_tokens=1,498,368,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7805/500000] + train/ActionL1Loss=0.0549 + throughput/total_tokens=1,498,560,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7806/500000] + train/ActionL1Loss=0.0553 + throughput/total_tokens=1,498,752,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7807/500000] + train/ActionL1Loss=0.0574 + throughput/total_tokens=1,498,944,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7808/500000] + train/ActionL1Loss=0.0784 + throughput/total_tokens=1,499,136,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7809/500000] + train/ActionL1Loss=0.0496 + throughput/total_tokens=1,499,328,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7810/500000] + train/ActionL1Loss=0.0552 + throughput/total_tokens=1,499,520,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7811/500000] + train/ActionL1Loss=0.0499 + throughput/total_tokens=1,499,712,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7812/500000] + train/ActionL1Loss=0.0871 + throughput/total_tokens=1,499,904,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7813/500000] + train/ActionL1Loss=0.0440 + throughput/total_tokens=1,500,096,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7814/500000] + train/ActionL1Loss=0.0489 + throughput/total_tokens=1,500,288,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7815/500000] + train/ActionL1Loss=0.0623 + throughput/total_tokens=1,500,480,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7816/500000] + train/ActionL1Loss=0.0530 + throughput/total_tokens=1,500,672,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7817/500000] + train/ActionL1Loss=0.0770 + throughput/total_tokens=1,500,864,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7818/500000] + train/ActionL1Loss=0.0888 + throughput/total_tokens=1,501,056,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7819/500000] + train/ActionL1Loss=0.0620 + throughput/total_tokens=1,501,248,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=7820/500000] + optim/total_grad_norm=6.834 + train/ActionL1Loss=0.0408 + throughput/total_tokens=1,501,440,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7821/500000] + train/ActionL1Loss=0.0805 + throughput/total_tokens=1,501,632,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7822/500000] + train/ActionL1Loss=0.0665 + throughput/total_tokens=1,501,824,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7823/500000] + train/ActionL1Loss=0.0671 + throughput/total_tokens=1,502,016,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7824/500000] + train/ActionL1Loss=0.0571 + throughput/total_tokens=1,502,208,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7825/500000] + train/ActionL1Loss=0.0671 + throughput/total_tokens=1,502,400,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7826/500000] + train/ActionL1Loss=0.0526 + throughput/total_tokens=1,502,592,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7827/500000] + train/ActionL1Loss=0.0673 + throughput/total_tokens=1,502,784,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7828/500000] + train/ActionL1Loss=0.0680 + throughput/total_tokens=1,502,976,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7829/500000] + train/ActionL1Loss=0.0535 + throughput/total_tokens=1,503,168,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7830/500000] + train/ActionL1Loss=0.0538 + throughput/total_tokens=1,503,360,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7831/500000] + train/ActionL1Loss=0.0754 + throughput/total_tokens=1,503,552,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7832/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,503,744,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7833/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=1,503,936,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7834/500000] + train/ActionL1Loss=0.0720 + throughput/total_tokens=1,504,128,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7835/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,504,320,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7836/500000] + train/ActionL1Loss=0.0593 + throughput/total_tokens=1,504,512,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7837/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=1,504,704,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7838/500000] + train/ActionL1Loss=0.0540 + throughput/total_tokens=1,504,896,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7839/500000] + train/ActionL1Loss=0.0630 + throughput/total_tokens=1,505,088,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7840/500000] + optim/total_grad_norm=5.781 + train/ActionL1Loss=0.0528 + throughput/total_tokens=1,505,280,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7841/500000] + train/ActionL1Loss=0.0530 + throughput/total_tokens=1,505,472,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7842/500000] + train/ActionL1Loss=0.0784 + throughput/total_tokens=1,505,664,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7843/500000] + train/ActionL1Loss=0.0550 + throughput/total_tokens=1,505,856,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7844/500000] + train/ActionL1Loss=0.0636 + throughput/total_tokens=1,506,048,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7845/500000] + train/ActionL1Loss=0.0698 + throughput/total_tokens=1,506,240,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7846/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,506,432,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7847/500000] + train/ActionL1Loss=0.0579 + throughput/total_tokens=1,506,624,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7848/500000] + train/ActionL1Loss=0.0635 + throughput/total_tokens=1,506,816,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7849/500000] + train/ActionL1Loss=0.0964 + throughput/total_tokens=1,507,008,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7850/500000] + train/ActionL1Loss=0.0550 + throughput/total_tokens=1,507,200,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7851/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=1,507,392,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7852/500000] + train/ActionL1Loss=0.0504 + throughput/total_tokens=1,507,584,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7853/500000] + train/ActionL1Loss=0.0761 + throughput/total_tokens=1,507,776,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7854/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=1,507,968,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=7855/500000] + train/ActionL1Loss=0.0672 + throughput/total_tokens=1,508,160,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7856/500000] + train/ActionL1Loss=0.0554 + throughput/total_tokens=1,508,352,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7857/500000] + train/ActionL1Loss=0.0665 + throughput/total_tokens=1,508,544,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7858/500000] + train/ActionL1Loss=0.0672 + throughput/total_tokens=1,508,736,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7859/500000] + train/ActionL1Loss=0.0484 + throughput/total_tokens=1,508,928,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7860/500000] + optim/total_grad_norm=6.353 + train/ActionL1Loss=0.0461 + throughput/total_tokens=1,509,120,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7861/500000] + train/ActionL1Loss=0.0655 + throughput/total_tokens=1,509,312,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7862/500000] + train/ActionL1Loss=0.0536 + throughput/total_tokens=1,509,504,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7863/500000] + train/ActionL1Loss=0.0579 + throughput/total_tokens=1,509,696,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7864/500000] + train/ActionL1Loss=0.0550 + throughput/total_tokens=1,509,888,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7865/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,510,080,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7866/500000] + train/ActionL1Loss=0.0528 + throughput/total_tokens=1,510,272,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7867/500000] + train/ActionL1Loss=0.0791 + throughput/total_tokens=1,510,464,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7868/500000] + train/ActionL1Loss=0.0581 + throughput/total_tokens=1,510,656,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7869/500000] + train/ActionL1Loss=0.0532 + throughput/total_tokens=1,510,848,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7870/500000] + train/ActionL1Loss=0.0513 + throughput/total_tokens=1,511,040,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7871/500000] + train/ActionL1Loss=0.0772 + throughput/total_tokens=1,511,232,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7872/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=1,511,424,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7873/500000] + train/ActionL1Loss=0.0723 + throughput/total_tokens=1,511,616,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7874/500000] + train/ActionL1Loss=0.0485 + throughput/total_tokens=1,511,808,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7875/500000] + train/ActionL1Loss=0.0816 + throughput/total_tokens=1,512,000,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7876/500000] + train/ActionL1Loss=0.0565 + throughput/total_tokens=1,512,192,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7877/500000] + train/ActionL1Loss=0.0497 + throughput/total_tokens=1,512,384,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7878/500000] + train/ActionL1Loss=0.0564 + throughput/total_tokens=1,512,576,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7879/500000] + train/ActionL1Loss=0.0564 + throughput/total_tokens=1,512,768,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7880/500000] + optim/total_grad_norm=6.621 + train/ActionL1Loss=0.0720 + throughput/total_tokens=1,512,960,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7881/500000] + train/ActionL1Loss=0.0721 + throughput/total_tokens=1,513,152,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7882/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,513,344,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7883/500000] + train/ActionL1Loss=0.0608 + throughput/total_tokens=1,513,536,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7884/500000] + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,513,728,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7885/500000] + train/ActionL1Loss=0.0717 + throughput/total_tokens=1,513,920,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7886/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=1,514,112,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7887/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=1,514,304,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7888/500000] + train/ActionL1Loss=0.0605 + throughput/total_tokens=1,514,496,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7889/500000] + train/ActionL1Loss=0.0586 + throughput/total_tokens=1,514,688,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7890/500000] + train/ActionL1Loss=0.0572 + throughput/total_tokens=1,514,880,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7891/500000] + train/ActionL1Loss=0.0513 + throughput/total_tokens=1,515,072,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7892/500000] + train/ActionL1Loss=0.0793 + throughput/total_tokens=1,515,264,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7893/500000] + train/ActionL1Loss=0.0532 + throughput/total_tokens=1,515,456,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7894/500000] + train/ActionL1Loss=0.0494 + throughput/total_tokens=1,515,648,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7895/500000] + train/ActionL1Loss=0.0599 + throughput/total_tokens=1,515,840,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7896/500000] + train/ActionL1Loss=0.0584 + throughput/total_tokens=1,516,032,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=7897/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,516,224,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=7898/500000] + train/ActionL1Loss=0.0701 + throughput/total_tokens=1,516,416,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7899/500000] + train/ActionL1Loss=0.0795 + throughput/total_tokens=1,516,608,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7900/500000] + optim/total_grad_norm=7.112 + train/ActionL1Loss=0.0620 + throughput/total_tokens=1,516,800,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7901/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,516,992,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7902/500000] + train/ActionL1Loss=0.0639 + throughput/total_tokens=1,517,184,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7903/500000] + train/ActionL1Loss=0.0521 + throughput/total_tokens=1,517,376,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7904/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,517,568,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7905/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=1,517,760,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7906/500000] + train/ActionL1Loss=0.0498 + throughput/total_tokens=1,517,952,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7907/500000] + train/ActionL1Loss=0.0768 + throughput/total_tokens=1,518,144,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7908/500000] + train/ActionL1Loss=0.0572 + throughput/total_tokens=1,518,336,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=7909/500000] + train/ActionL1Loss=0.0431 + throughput/total_tokens=1,518,528,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7910/500000] + train/ActionL1Loss=0.0652 + throughput/total_tokens=1,518,720,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=7911/500000] + train/ActionL1Loss=0.0541 + throughput/total_tokens=1,518,912,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7912/500000] + train/ActionL1Loss=0.0470 + throughput/total_tokens=1,519,104,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7913/500000] + train/ActionL1Loss=0.0494 + throughput/total_tokens=1,519,296,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7914/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=1,519,488,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7915/500000] + train/ActionL1Loss=0.0616 + throughput/total_tokens=1,519,680,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7916/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=1,519,872,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=7917/500000] + train/ActionL1Loss=0.0634 + throughput/total_tokens=1,520,064,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7918/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=1,520,256,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7919/500000] + train/ActionL1Loss=0.0400 + throughput/total_tokens=1,520,448,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=7920/500000] + optim/total_grad_norm=6.966 + train/ActionL1Loss=0.0573 + throughput/total_tokens=1,520,640,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7921/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=1,520,832,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7922/500000] + train/ActionL1Loss=0.0563 + throughput/total_tokens=1,521,024,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7923/500000] + train/ActionL1Loss=0.0497 + throughput/total_tokens=1,521,216,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7924/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,521,408,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7925/500000] + train/ActionL1Loss=0.0818 + throughput/total_tokens=1,521,600,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7926/500000] + train/ActionL1Loss=0.0678 + throughput/total_tokens=1,521,792,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7927/500000] + train/ActionL1Loss=0.0573 + throughput/total_tokens=1,521,984,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7928/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,522,176,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7929/500000] + train/ActionL1Loss=0.0751 + throughput/total_tokens=1,522,368,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7930/500000] + train/ActionL1Loss=0.0519 + throughput/total_tokens=1,522,560,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7931/500000] + train/ActionL1Loss=0.0554 + throughput/total_tokens=1,522,752,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7932/500000] + train/ActionL1Loss=0.0493 + throughput/total_tokens=1,522,944,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7933/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,523,136,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7934/500000] + train/ActionL1Loss=0.0727 + throughput/total_tokens=1,523,328,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7935/500000] + train/ActionL1Loss=0.0643 + throughput/total_tokens=1,523,520,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7936/500000] + train/ActionL1Loss=0.0501 + throughput/total_tokens=1,523,712,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7937/500000] + train/ActionL1Loss=0.0641 + throughput/total_tokens=1,523,904,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7938/500000] + train/ActionL1Loss=0.0554 + throughput/total_tokens=1,524,096,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7939/500000] + train/ActionL1Loss=0.0557 + throughput/total_tokens=1,524,288,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7940/500000] + optim/total_grad_norm=5.282 + train/ActionL1Loss=0.0517 + throughput/total_tokens=1,524,480,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7941/500000] + train/ActionL1Loss=0.0579 + throughput/total_tokens=1,524,672,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7942/500000] + train/ActionL1Loss=0.0557 + throughput/total_tokens=1,524,864,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7943/500000] + train/ActionL1Loss=0.0460 + throughput/total_tokens=1,525,056,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7944/500000] + train/ActionL1Loss=0.0873 + throughput/total_tokens=1,525,248,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7945/500000] + train/ActionL1Loss=0.0474 + throughput/total_tokens=1,525,440,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7946/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=1,525,632,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7947/500000] + train/ActionL1Loss=0.0706 + throughput/total_tokens=1,525,824,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7948/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,526,016,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7949/500000] + train/ActionL1Loss=0.0590 + throughput/total_tokens=1,526,208,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7950/500000] + train/ActionL1Loss=0.0728 + throughput/total_tokens=1,526,400,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7951/500000] + train/ActionL1Loss=0.0670 + throughput/total_tokens=1,526,592,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7952/500000] + train/ActionL1Loss=0.0745 + throughput/total_tokens=1,526,784,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7953/500000] + train/ActionL1Loss=0.0643 + throughput/total_tokens=1,526,976,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7954/500000] + train/ActionL1Loss=0.0789 + throughput/total_tokens=1,527,168,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7955/500000] + train/ActionL1Loss=0.0477 + throughput/total_tokens=1,527,360,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7956/500000] + train/ActionL1Loss=0.0466 + throughput/total_tokens=1,527,552,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7957/500000] + train/ActionL1Loss=0.0678 + throughput/total_tokens=1,527,744,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=7958/500000] + train/ActionL1Loss=0.0924 + throughput/total_tokens=1,527,936,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=7959/500000] + train/ActionL1Loss=0.0565 + throughput/total_tokens=1,528,128,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=7960/500000] + optim/total_grad_norm=6.486 + train/ActionL1Loss=0.0578 + throughput/total_tokens=1,528,320,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=7961/500000] + train/ActionL1Loss=0.0674 + throughput/total_tokens=1,528,512,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7962/500000] + train/ActionL1Loss=0.0718 + throughput/total_tokens=1,528,704,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7963/500000] + train/ActionL1Loss=0.0754 + throughput/total_tokens=1,528,896,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7964/500000] + train/ActionL1Loss=0.0523 + throughput/total_tokens=1,529,088,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7965/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,529,280,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7966/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=1,529,472,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7967/500000] + train/ActionL1Loss=0.0524 + throughput/total_tokens=1,529,664,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7968/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=1,529,856,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7969/500000] + train/ActionL1Loss=0.0558 + throughput/total_tokens=1,530,048,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=7970/500000] + train/ActionL1Loss=0.0585 + throughput/total_tokens=1,530,240,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7971/500000] + train/ActionL1Loss=0.0609 + throughput/total_tokens=1,530,432,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7972/500000] + train/ActionL1Loss=0.0757 + throughput/total_tokens=1,530,624,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7973/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=1,530,816,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7974/500000] + train/ActionL1Loss=0.0545 + throughput/total_tokens=1,531,008,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7975/500000] + train/ActionL1Loss=0.0533 + throughput/total_tokens=1,531,200,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7976/500000] + train/ActionL1Loss=0.0624 + throughput/total_tokens=1,531,392,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7977/500000] + train/ActionL1Loss=0.0732 + throughput/total_tokens=1,531,584,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7978/500000] + train/ActionL1Loss=0.0922 + throughput/total_tokens=1,531,776,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7979/500000] + train/ActionL1Loss=0.0487 + throughput/total_tokens=1,531,968,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7980/500000] + optim/total_grad_norm=3.987 + train/ActionL1Loss=0.0579 + throughput/total_tokens=1,532,160,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7981/500000] + train/ActionL1Loss=0.0623 + throughput/total_tokens=1,532,352,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7982/500000] + train/ActionL1Loss=0.0606 + throughput/total_tokens=1,532,544,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7983/500000] + train/ActionL1Loss=0.0754 + throughput/total_tokens=1,532,736,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=7984/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,532,928,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7985/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=1,533,120,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7986/500000] + train/ActionL1Loss=0.0563 + throughput/total_tokens=1,533,312,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7987/500000] + train/ActionL1Loss=0.0459 + throughput/total_tokens=1,533,504,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7988/500000] + train/ActionL1Loss=0.0484 + throughput/total_tokens=1,533,696,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7989/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,533,888,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7990/500000] + train/ActionL1Loss=0.0577 + throughput/total_tokens=1,534,080,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=7991/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=1,534,272,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7992/500000] + train/ActionL1Loss=0.0641 + throughput/total_tokens=1,534,464,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7993/500000] + train/ActionL1Loss=0.0570 + throughput/total_tokens=1,534,656,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7994/500000] + train/ActionL1Loss=0.0785 + throughput/total_tokens=1,534,848,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7995/500000] + train/ActionL1Loss=0.0819 + throughput/total_tokens=1,535,040,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7996/500000] + train/ActionL1Loss=0.0869 + throughput/total_tokens=1,535,232,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7997/500000] + train/ActionL1Loss=0.0501 + throughput/total_tokens=1,535,424,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7998/500000] + train/ActionL1Loss=0.0808 + throughput/total_tokens=1,535,616,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=7999/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=1,535,808,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8000/500000] + optim/total_grad_norm=6.170 + train/ActionL1Loss=0.0549 + throughput/total_tokens=1,536,000,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +10/02 [12:18:58] INFO | >> Saving config... checkpoint.py:608 +10/02 [12:19:40] INFO | >> Saving model state... checkpoint.py:796 +10/02 [12:20:52] INFO | >> Saving optim state... checkpoint.py:811 +10/02 [12:22:25] INFO | >> Saving trainer state... checkpoint.py:670 + INFO | >> Saving config... checkpoint.py:608 +[step=8001/500000] + train/ActionL1Loss=0.0490 + throughput/total_tokens=1,536,192,000 + throughput/device/tokens_per_second=1,213 + throughput/device/batches_per_second=0.0506 +[step=8002/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,536,384,000 + throughput/device/tokens_per_second=1,186 + throughput/device/batches_per_second=0.0495 +[step=8003/500000] + train/ActionL1Loss=0.0507 + throughput/total_tokens=1,536,576,000 + throughput/device/tokens_per_second=1,175 + throughput/device/batches_per_second=0.0490 +[step=8004/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,536,768,000 + throughput/device/tokens_per_second=1,171 + throughput/device/batches_per_second=0.0488 +[step=8005/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,536,960,000 + throughput/device/tokens_per_second=1,170 + throughput/device/batches_per_second=0.0488 +[step=8006/500000] + train/ActionL1Loss=0.0658 + throughput/total_tokens=1,537,152,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8007/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=1,537,344,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8008/500000] + train/ActionL1Loss=0.0539 + throughput/total_tokens=1,537,536,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8009/500000] + train/ActionL1Loss=0.0597 + throughput/total_tokens=1,537,728,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8010/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=1,537,920,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8011/500000] + train/ActionL1Loss=0.0536 + throughput/total_tokens=1,538,112,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8012/500000] + train/ActionL1Loss=0.0511 + throughput/total_tokens=1,538,304,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8013/500000] + train/ActionL1Loss=0.0740 + throughput/total_tokens=1,538,496,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8014/500000] + train/ActionL1Loss=0.0597 + throughput/total_tokens=1,538,688,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8015/500000] + train/ActionL1Loss=0.0674 + throughput/total_tokens=1,538,880,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8016/500000] + train/ActionL1Loss=0.0646 + throughput/total_tokens=1,539,072,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8017/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=1,539,264,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8018/500000] + train/ActionL1Loss=0.0483 + throughput/total_tokens=1,539,456,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8019/500000] + train/ActionL1Loss=0.0587 + throughput/total_tokens=1,539,648,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8020/500000] + optim/total_grad_norm=6.306 + train/ActionL1Loss=0.0395 + throughput/total_tokens=1,539,840,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8021/500000] + train/ActionL1Loss=0.0640 + throughput/total_tokens=1,540,032,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8022/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=1,540,224,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8023/500000] + train/ActionL1Loss=0.0660 + throughput/total_tokens=1,540,416,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8024/500000] + train/ActionL1Loss=0.0577 + throughput/total_tokens=1,540,608,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8025/500000] + train/ActionL1Loss=0.0607 + throughput/total_tokens=1,540,800,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8026/500000] + train/ActionL1Loss=0.0542 + throughput/total_tokens=1,540,992,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8027/500000] + train/ActionL1Loss=0.0585 + throughput/total_tokens=1,541,184,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8028/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,541,376,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8029/500000] + train/ActionL1Loss=0.0497 + throughput/total_tokens=1,541,568,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8030/500000] + train/ActionL1Loss=0.0614 + throughput/total_tokens=1,541,760,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8031/500000] + train/ActionL1Loss=0.0691 + throughput/total_tokens=1,541,952,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8032/500000] + train/ActionL1Loss=0.0687 + throughput/total_tokens=1,542,144,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8033/500000] + train/ActionL1Loss=0.0593 + throughput/total_tokens=1,542,336,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8034/500000] + train/ActionL1Loss=0.0672 + throughput/total_tokens=1,542,528,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8035/500000] + train/ActionL1Loss=0.0470 + throughput/total_tokens=1,542,720,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8036/500000] + train/ActionL1Loss=0.0585 + throughput/total_tokens=1,542,912,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8037/500000] + train/ActionL1Loss=0.0590 + throughput/total_tokens=1,543,104,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8038/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,543,296,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8039/500000] + train/ActionL1Loss=0.0562 + throughput/total_tokens=1,543,488,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8040/500000] + optim/total_grad_norm=7.374 + train/ActionL1Loss=0.0648 + throughput/total_tokens=1,543,680,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8041/500000] + train/ActionL1Loss=0.0665 + throughput/total_tokens=1,543,872,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8042/500000] + train/ActionL1Loss=0.0596 + throughput/total_tokens=1,544,064,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8043/500000] + train/ActionL1Loss=0.0551 + throughput/total_tokens=1,544,256,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8044/500000] + train/ActionL1Loss=0.0527 + throughput/total_tokens=1,544,448,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8045/500000] + train/ActionL1Loss=0.0553 + throughput/total_tokens=1,544,640,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8046/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,544,832,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8047/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=1,545,024,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8048/500000] + train/ActionL1Loss=0.0665 + throughput/total_tokens=1,545,216,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8049/500000] + train/ActionL1Loss=0.0529 + throughput/total_tokens=1,545,408,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8050/500000] + train/ActionL1Loss=0.0497 + throughput/total_tokens=1,545,600,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8051/500000] + train/ActionL1Loss=0.0599 + throughput/total_tokens=1,545,792,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8052/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,545,984,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8053/500000] + train/ActionL1Loss=0.0726 + throughput/total_tokens=1,546,176,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8054/500000] + train/ActionL1Loss=0.0641 + throughput/total_tokens=1,546,368,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8055/500000] + train/ActionL1Loss=0.0520 + throughput/total_tokens=1,546,560,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8056/500000] + train/ActionL1Loss=0.0904 + throughput/total_tokens=1,546,752,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8057/500000] + train/ActionL1Loss=0.0626 + throughput/total_tokens=1,546,944,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8058/500000] + train/ActionL1Loss=0.0551 + throughput/total_tokens=1,547,136,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8059/500000] + train/ActionL1Loss=0.0820 + throughput/total_tokens=1,547,328,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8060/500000] + optim/total_grad_norm=7.330 + train/ActionL1Loss=0.0650 + throughput/total_tokens=1,547,520,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=8061/500000] + train/ActionL1Loss=0.0799 + throughput/total_tokens=1,547,712,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8062/500000] + train/ActionL1Loss=0.0793 + throughput/total_tokens=1,547,904,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8063/500000] + train/ActionL1Loss=0.0576 + throughput/total_tokens=1,548,096,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8064/500000] + train/ActionL1Loss=0.0603 + throughput/total_tokens=1,548,288,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8065/500000] + train/ActionL1Loss=0.0653 + throughput/total_tokens=1,548,480,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8066/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=1,548,672,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8067/500000] + train/ActionL1Loss=0.0591 + throughput/total_tokens=1,548,864,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8068/500000] + train/ActionL1Loss=0.0671 + throughput/total_tokens=1,549,056,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8069/500000] + train/ActionL1Loss=0.0495 + throughput/total_tokens=1,549,248,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8070/500000] + train/ActionL1Loss=0.0627 + throughput/total_tokens=1,549,440,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8071/500000] + train/ActionL1Loss=0.0574 + throughput/total_tokens=1,549,632,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8072/500000] + train/ActionL1Loss=0.0673 + throughput/total_tokens=1,549,824,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8073/500000] + train/ActionL1Loss=0.0685 + throughput/total_tokens=1,550,016,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8074/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=1,550,208,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8075/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=1,550,400,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8076/500000] + train/ActionL1Loss=0.0609 + throughput/total_tokens=1,550,592,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8077/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=1,550,784,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8078/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=1,550,976,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8079/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=1,551,168,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8080/500000] + optim/total_grad_norm=7.223 + train/ActionL1Loss=0.0516 + throughput/total_tokens=1,551,360,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8081/500000] + train/ActionL1Loss=0.0536 + throughput/total_tokens=1,551,552,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8082/500000] + train/ActionL1Loss=0.0565 + throughput/total_tokens=1,551,744,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8083/500000] + train/ActionL1Loss=0.0698 + throughput/total_tokens=1,551,936,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8084/500000] + train/ActionL1Loss=0.0511 + throughput/total_tokens=1,552,128,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8085/500000] + train/ActionL1Loss=0.0643 + throughput/total_tokens=1,552,320,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8086/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,552,512,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8087/500000] + train/ActionL1Loss=0.0698 + throughput/total_tokens=1,552,704,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8088/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,552,896,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8089/500000] + train/ActionL1Loss=0.0509 + throughput/total_tokens=1,553,088,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8090/500000] + train/ActionL1Loss=0.0630 + throughput/total_tokens=1,553,280,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8091/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,553,472,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8092/500000] + train/ActionL1Loss=0.0493 + throughput/total_tokens=1,553,664,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8093/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,553,856,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8094/500000] + train/ActionL1Loss=0.0673 + throughput/total_tokens=1,554,048,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8095/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=1,554,240,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8096/500000] + train/ActionL1Loss=0.0691 + throughput/total_tokens=1,554,432,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8097/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,554,624,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8098/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=1,554,816,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8099/500000] + train/ActionL1Loss=0.0847 + throughput/total_tokens=1,555,008,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8100/500000] + optim/total_grad_norm=5.448 + train/ActionL1Loss=0.0566 + throughput/total_tokens=1,555,200,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8101/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=1,555,392,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=8102/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=1,555,584,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=8103/500000] + train/ActionL1Loss=0.0773 + throughput/total_tokens=1,555,776,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=8104/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=1,555,968,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=8105/500000] + train/ActionL1Loss=0.0423 + throughput/total_tokens=1,556,160,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=8106/500000] + train/ActionL1Loss=0.0535 + throughput/total_tokens=1,556,352,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=8107/500000] + train/ActionL1Loss=0.0614 + throughput/total_tokens=1,556,544,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0485 +[step=8108/500000] + train/ActionL1Loss=0.0509 + throughput/total_tokens=1,556,736,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=8109/500000] + train/ActionL1Loss=0.0798 + throughput/total_tokens=1,556,928,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=8110/500000] + train/ActionL1Loss=0.0635 + throughput/total_tokens=1,557,120,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=8111/500000] + train/ActionL1Loss=0.0643 + throughput/total_tokens=1,557,312,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=8112/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=1,557,504,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=8113/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=1,557,696,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=8114/500000] + train/ActionL1Loss=0.0555 + throughput/total_tokens=1,557,888,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=8115/500000] + train/ActionL1Loss=0.0593 + throughput/total_tokens=1,558,080,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8116/500000] + train/ActionL1Loss=0.0729 + throughput/total_tokens=1,558,272,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=8117/500000] + train/ActionL1Loss=0.0673 + throughput/total_tokens=1,558,464,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=8118/500000] + train/ActionL1Loss=0.0579 + throughput/total_tokens=1,558,656,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8119/500000] + train/ActionL1Loss=0.0493 + throughput/total_tokens=1,558,848,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8120/500000] + optim/total_grad_norm=5.159 + train/ActionL1Loss=0.0542 + throughput/total_tokens=1,559,040,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8121/500000] + train/ActionL1Loss=0.0499 + throughput/total_tokens=1,559,232,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8122/500000] + train/ActionL1Loss=0.0640 + throughput/total_tokens=1,559,424,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8123/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,559,616,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8124/500000] + train/ActionL1Loss=0.0551 + throughput/total_tokens=1,559,808,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8125/500000] + train/ActionL1Loss=0.0610 + throughput/total_tokens=1,560,000,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8126/500000] + train/ActionL1Loss=0.0647 + throughput/total_tokens=1,560,192,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8127/500000] + train/ActionL1Loss=0.0501 + throughput/total_tokens=1,560,384,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8128/500000] + train/ActionL1Loss=0.0696 + throughput/total_tokens=1,560,576,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8129/500000] + train/ActionL1Loss=0.0450 + throughput/total_tokens=1,560,768,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8130/500000] + train/ActionL1Loss=0.0407 + throughput/total_tokens=1,560,960,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8131/500000] + train/ActionL1Loss=0.0737 + throughput/total_tokens=1,561,152,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8132/500000] + train/ActionL1Loss=0.0483 + throughput/total_tokens=1,561,344,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8133/500000] + train/ActionL1Loss=0.0538 + throughput/total_tokens=1,561,536,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8134/500000] + train/ActionL1Loss=0.0472 + throughput/total_tokens=1,561,728,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8135/500000] + train/ActionL1Loss=0.0480 + throughput/total_tokens=1,561,920,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8136/500000] + train/ActionL1Loss=0.0798 + throughput/total_tokens=1,562,112,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8137/500000] + train/ActionL1Loss=0.0479 + throughput/total_tokens=1,562,304,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8138/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=1,562,496,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8139/500000] + train/ActionL1Loss=0.0722 + throughput/total_tokens=1,562,688,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8140/500000] + optim/total_grad_norm=5.230 + train/ActionL1Loss=0.0517 + throughput/total_tokens=1,562,880,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8141/500000] + train/ActionL1Loss=0.0540 + throughput/total_tokens=1,563,072,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8142/500000] + train/ActionL1Loss=0.0538 + throughput/total_tokens=1,563,264,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8143/500000] + train/ActionL1Loss=0.0462 + throughput/total_tokens=1,563,456,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8144/500000] + train/ActionL1Loss=0.0676 + throughput/total_tokens=1,563,648,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8145/500000] + train/ActionL1Loss=0.0469 + throughput/total_tokens=1,563,840,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8146/500000] + train/ActionL1Loss=0.0607 + throughput/total_tokens=1,564,032,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8147/500000] + train/ActionL1Loss=0.0524 + throughput/total_tokens=1,564,224,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8148/500000] + train/ActionL1Loss=0.0515 + throughput/total_tokens=1,564,416,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8149/500000] + train/ActionL1Loss=0.0621 + throughput/total_tokens=1,564,608,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8150/500000] + train/ActionL1Loss=0.0643 + throughput/total_tokens=1,564,800,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8151/500000] + train/ActionL1Loss=0.0562 + throughput/total_tokens=1,564,992,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8152/500000] + train/ActionL1Loss=0.0681 + throughput/total_tokens=1,565,184,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8153/500000] + train/ActionL1Loss=0.0586 + throughput/total_tokens=1,565,376,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8154/500000] + train/ActionL1Loss=0.0512 + throughput/total_tokens=1,565,568,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8155/500000] + train/ActionL1Loss=0.0446 + throughput/total_tokens=1,565,760,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8156/500000] + train/ActionL1Loss=0.0480 + throughput/total_tokens=1,565,952,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8157/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=1,566,144,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8158/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,566,336,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8159/500000] + train/ActionL1Loss=0.0603 + throughput/total_tokens=1,566,528,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8160/500000] + optim/total_grad_norm=6.170 + train/ActionL1Loss=0.0658 + throughput/total_tokens=1,566,720,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=8161/500000] + train/ActionL1Loss=0.0660 + throughput/total_tokens=1,566,912,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8162/500000] + train/ActionL1Loss=0.0491 + throughput/total_tokens=1,567,104,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8163/500000] + train/ActionL1Loss=0.0601 + throughput/total_tokens=1,567,296,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8164/500000] + train/ActionL1Loss=0.0574 + throughput/total_tokens=1,567,488,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8165/500000] + train/ActionL1Loss=0.0543 + throughput/total_tokens=1,567,680,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8166/500000] + train/ActionL1Loss=0.0484 + throughput/total_tokens=1,567,872,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8167/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=1,568,064,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8168/500000] + train/ActionL1Loss=0.0666 + throughput/total_tokens=1,568,256,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8169/500000] + train/ActionL1Loss=0.0463 + throughput/total_tokens=1,568,448,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8170/500000] + train/ActionL1Loss=0.0798 + throughput/total_tokens=1,568,640,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8171/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,568,832,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8172/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=1,569,024,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8173/500000] + train/ActionL1Loss=0.0752 + throughput/total_tokens=1,569,216,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8174/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,569,408,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8175/500000] + train/ActionL1Loss=0.0550 + throughput/total_tokens=1,569,600,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8176/500000] + train/ActionL1Loss=0.0437 + throughput/total_tokens=1,569,792,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8177/500000] + train/ActionL1Loss=0.0429 + throughput/total_tokens=1,569,984,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8178/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=1,570,176,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8179/500000] + train/ActionL1Loss=0.0626 + throughput/total_tokens=1,570,368,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8180/500000] + optim/total_grad_norm=6.299 + train/ActionL1Loss=0.0547 + throughput/total_tokens=1,570,560,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8181/500000] + train/ActionL1Loss=0.0698 + throughput/total_tokens=1,570,752,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8182/500000] + train/ActionL1Loss=0.0530 + throughput/total_tokens=1,570,944,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8183/500000] + train/ActionL1Loss=0.0525 + throughput/total_tokens=1,571,136,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8184/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=1,571,328,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8185/500000] + train/ActionL1Loss=0.0556 + throughput/total_tokens=1,571,520,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8186/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=1,571,712,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8187/500000] + train/ActionL1Loss=0.0804 + throughput/total_tokens=1,571,904,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8188/500000] + train/ActionL1Loss=0.0530 + throughput/total_tokens=1,572,096,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8189/500000] + train/ActionL1Loss=0.0703 + throughput/total_tokens=1,572,288,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8190/500000] + train/ActionL1Loss=0.0483 + throughput/total_tokens=1,572,480,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8191/500000] + train/ActionL1Loss=0.0519 + throughput/total_tokens=1,572,672,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8192/500000] + train/ActionL1Loss=0.0554 + throughput/total_tokens=1,572,864,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8193/500000] + train/ActionL1Loss=0.0461 + throughput/total_tokens=1,573,056,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8194/500000] + train/ActionL1Loss=0.0548 + throughput/total_tokens=1,573,248,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8195/500000] + train/ActionL1Loss=0.0689 + throughput/total_tokens=1,573,440,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8196/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=1,573,632,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8197/500000] + train/ActionL1Loss=0.0620 + throughput/total_tokens=1,573,824,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8198/500000] + train/ActionL1Loss=0.0583 + throughput/total_tokens=1,574,016,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8199/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=1,574,208,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8200/500000] + optim/total_grad_norm=5.098 + train/ActionL1Loss=0.0653 + throughput/total_tokens=1,574,400,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8201/500000] + train/ActionL1Loss=0.0564 + throughput/total_tokens=1,574,592,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8202/500000] + train/ActionL1Loss=0.0794 + throughput/total_tokens=1,574,784,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8203/500000] + train/ActionL1Loss=0.0777 + throughput/total_tokens=1,574,976,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8204/500000] + train/ActionL1Loss=0.0601 + throughput/total_tokens=1,575,168,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8205/500000] + train/ActionL1Loss=0.0515 + throughput/total_tokens=1,575,360,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8206/500000] + train/ActionL1Loss=0.0707 + throughput/total_tokens=1,575,552,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8207/500000] + train/ActionL1Loss=0.0566 + throughput/total_tokens=1,575,744,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8208/500000] + train/ActionL1Loss=0.0731 + throughput/total_tokens=1,575,936,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8209/500000] + train/ActionL1Loss=0.0580 + throughput/total_tokens=1,576,128,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8210/500000] + train/ActionL1Loss=0.0603 + throughput/total_tokens=1,576,320,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8211/500000] + train/ActionL1Loss=0.0657 + throughput/total_tokens=1,576,512,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8212/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,576,704,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8213/500000] + train/ActionL1Loss=0.0591 + throughput/total_tokens=1,576,896,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8214/500000] + train/ActionL1Loss=0.0477 + throughput/total_tokens=1,577,088,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8215/500000] + train/ActionL1Loss=0.0658 + throughput/total_tokens=1,577,280,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8216/500000] + train/ActionL1Loss=0.0606 + throughput/total_tokens=1,577,472,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8217/500000] + train/ActionL1Loss=0.0597 + throughput/total_tokens=1,577,664,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8218/500000] + train/ActionL1Loss=0.0378 + throughput/total_tokens=1,577,856,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8219/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,578,048,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8220/500000] + optim/total_grad_norm=5.415 + train/ActionL1Loss=0.0666 + throughput/total_tokens=1,578,240,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8221/500000] + train/ActionL1Loss=0.0552 + throughput/total_tokens=1,578,432,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8222/500000] + train/ActionL1Loss=0.0406 + throughput/total_tokens=1,578,624,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8223/500000] + train/ActionL1Loss=0.0635 + throughput/total_tokens=1,578,816,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8224/500000] + train/ActionL1Loss=0.0543 + throughput/total_tokens=1,579,008,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8225/500000] + train/ActionL1Loss=0.0617 + throughput/total_tokens=1,579,200,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8226/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=1,579,392,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8227/500000] + train/ActionL1Loss=0.0609 + throughput/total_tokens=1,579,584,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8228/500000] + train/ActionL1Loss=0.0574 + throughput/total_tokens=1,579,776,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8229/500000] + train/ActionL1Loss=0.0702 + throughput/total_tokens=1,579,968,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8230/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,580,160,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8231/500000] + train/ActionL1Loss=0.0517 + throughput/total_tokens=1,580,352,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8232/500000] + train/ActionL1Loss=0.0526 + throughput/total_tokens=1,580,544,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8233/500000] + train/ActionL1Loss=0.0579 + throughput/total_tokens=1,580,736,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8234/500000] + train/ActionL1Loss=0.0631 + throughput/total_tokens=1,580,928,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8235/500000] + train/ActionL1Loss=0.0708 + throughput/total_tokens=1,581,120,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8236/500000] + train/ActionL1Loss=0.0502 + throughput/total_tokens=1,581,312,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8237/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=1,581,504,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8238/500000] + train/ActionL1Loss=0.0671 + throughput/total_tokens=1,581,696,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8239/500000] + train/ActionL1Loss=0.0383 + throughput/total_tokens=1,581,888,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8240/500000] + optim/total_grad_norm=5.537 + train/ActionL1Loss=0.0660 + throughput/total_tokens=1,582,080,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8241/500000] + train/ActionL1Loss=0.0538 + throughput/total_tokens=1,582,272,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8242/500000] + train/ActionL1Loss=0.0562 + throughput/total_tokens=1,582,464,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8243/500000] + train/ActionL1Loss=0.0811 + throughput/total_tokens=1,582,656,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8244/500000] + train/ActionL1Loss=0.0736 + throughput/total_tokens=1,582,848,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8245/500000] + train/ActionL1Loss=0.0500 + throughput/total_tokens=1,583,040,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8246/500000] + train/ActionL1Loss=0.0684 + throughput/total_tokens=1,583,232,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8247/500000] + train/ActionL1Loss=0.0809 + throughput/total_tokens=1,583,424,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8248/500000] + train/ActionL1Loss=0.0665 + throughput/total_tokens=1,583,616,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8249/500000] + train/ActionL1Loss=0.0542 + throughput/total_tokens=1,583,808,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8250/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=1,584,000,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8251/500000] + train/ActionL1Loss=0.0794 + throughput/total_tokens=1,584,192,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8252/500000] + train/ActionL1Loss=0.0553 + throughput/total_tokens=1,584,384,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8253/500000] + train/ActionL1Loss=0.0749 + throughput/total_tokens=1,584,576,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8254/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=1,584,768,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8255/500000] + train/ActionL1Loss=0.0534 + throughput/total_tokens=1,584,960,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8256/500000] + train/ActionL1Loss=0.0474 + throughput/total_tokens=1,585,152,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8257/500000] + train/ActionL1Loss=0.0418 + throughput/total_tokens=1,585,344,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8258/500000] + train/ActionL1Loss=0.0606 + throughput/total_tokens=1,585,536,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8259/500000] + train/ActionL1Loss=0.0596 + throughput/total_tokens=1,585,728,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8260/500000] + optim/total_grad_norm=5.470 + train/ActionL1Loss=0.0652 + throughput/total_tokens=1,585,920,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8261/500000] + train/ActionL1Loss=0.0552 + throughput/total_tokens=1,586,112,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8262/500000] + train/ActionL1Loss=0.0662 + throughput/total_tokens=1,586,304,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8263/500000] + train/ActionL1Loss=0.0545 + throughput/total_tokens=1,586,496,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8264/500000] + train/ActionL1Loss=0.0571 + throughput/total_tokens=1,586,688,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8265/500000] + train/ActionL1Loss=0.0514 + throughput/total_tokens=1,586,880,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8266/500000] + train/ActionL1Loss=0.0616 + throughput/total_tokens=1,587,072,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8267/500000] + train/ActionL1Loss=0.0427 + throughput/total_tokens=1,587,264,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=8268/500000] + train/ActionL1Loss=0.0633 + throughput/total_tokens=1,587,456,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=8269/500000] + train/ActionL1Loss=0.0734 + throughput/total_tokens=1,587,648,000 + throughput/device/tokens_per_second=1,163 + throughput/device/batches_per_second=0.0485 +[step=8270/500000] + train/ActionL1Loss=0.0545 + throughput/total_tokens=1,587,840,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8271/500000] + train/ActionL1Loss=0.0675 + throughput/total_tokens=1,588,032,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8272/500000] + train/ActionL1Loss=0.0517 + throughput/total_tokens=1,588,224,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8273/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=1,588,416,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8274/500000] + train/ActionL1Loss=0.0444 + throughput/total_tokens=1,588,608,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8275/500000] + train/ActionL1Loss=0.0476 + throughput/total_tokens=1,588,800,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8276/500000] + train/ActionL1Loss=0.0535 + throughput/total_tokens=1,588,992,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8277/500000] + train/ActionL1Loss=0.0394 + throughput/total_tokens=1,589,184,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8278/500000] + train/ActionL1Loss=0.0583 + throughput/total_tokens=1,589,376,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8279/500000] + train/ActionL1Loss=0.0723 + throughput/total_tokens=1,589,568,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8280/500000] + optim/total_grad_norm=6.555 + train/ActionL1Loss=0.0638 + throughput/total_tokens=1,589,760,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8281/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=1,589,952,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8282/500000] + train/ActionL1Loss=0.0553 + throughput/total_tokens=1,590,144,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8283/500000] + train/ActionL1Loss=0.0605 + throughput/total_tokens=1,590,336,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8284/500000] + train/ActionL1Loss=0.0568 + throughput/total_tokens=1,590,528,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8285/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,590,720,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8286/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,590,912,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8287/500000] + train/ActionL1Loss=0.0519 + throughput/total_tokens=1,591,104,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8288/500000] + train/ActionL1Loss=0.0464 + throughput/total_tokens=1,591,296,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8289/500000] + train/ActionL1Loss=0.0579 + throughput/total_tokens=1,591,488,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8290/500000] + train/ActionL1Loss=0.0492 + throughput/total_tokens=1,591,680,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8291/500000] + train/ActionL1Loss=0.0429 + throughput/total_tokens=1,591,872,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8292/500000] + train/ActionL1Loss=0.0527 + throughput/total_tokens=1,592,064,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8293/500000] + train/ActionL1Loss=0.0477 + throughput/total_tokens=1,592,256,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8294/500000] + train/ActionL1Loss=0.0665 + throughput/total_tokens=1,592,448,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8295/500000] + train/ActionL1Loss=0.0484 + throughput/total_tokens=1,592,640,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8296/500000] + train/ActionL1Loss=0.0502 + throughput/total_tokens=1,592,832,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8297/500000] + train/ActionL1Loss=0.0538 + throughput/total_tokens=1,593,024,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8298/500000] + train/ActionL1Loss=0.0559 + throughput/total_tokens=1,593,216,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8299/500000] + train/ActionL1Loss=0.0636 + throughput/total_tokens=1,593,408,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8300/500000] + optim/total_grad_norm=4.815 + train/ActionL1Loss=0.0630 + throughput/total_tokens=1,593,600,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8301/500000] + train/ActionL1Loss=0.0596 + throughput/total_tokens=1,593,792,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8302/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,593,984,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8303/500000] + train/ActionL1Loss=0.0496 + throughput/total_tokens=1,594,176,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8304/500000] + train/ActionL1Loss=0.0582 + throughput/total_tokens=1,594,368,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8305/500000] + train/ActionL1Loss=0.0566 + throughput/total_tokens=1,594,560,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8306/500000] + train/ActionL1Loss=0.0620 + throughput/total_tokens=1,594,752,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8307/500000] + train/ActionL1Loss=0.0646 + throughput/total_tokens=1,594,944,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8308/500000] + train/ActionL1Loss=0.0453 + throughput/total_tokens=1,595,136,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8309/500000] + train/ActionL1Loss=0.0520 + throughput/total_tokens=1,595,328,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8310/500000] + train/ActionL1Loss=0.0529 + throughput/total_tokens=1,595,520,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=8311/500000] + train/ActionL1Loss=0.0748 + throughput/total_tokens=1,595,712,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8312/500000] + train/ActionL1Loss=0.0812 + throughput/total_tokens=1,595,904,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8313/500000] + train/ActionL1Loss=0.0502 + throughput/total_tokens=1,596,096,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8314/500000] + train/ActionL1Loss=0.0374 + throughput/total_tokens=1,596,288,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8315/500000] + train/ActionL1Loss=0.0592 + throughput/total_tokens=1,596,480,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8316/500000] + train/ActionL1Loss=0.0563 + throughput/total_tokens=1,596,672,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8317/500000] + train/ActionL1Loss=0.0389 + throughput/total_tokens=1,596,864,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8318/500000] + train/ActionL1Loss=0.0460 + throughput/total_tokens=1,597,056,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8319/500000] + train/ActionL1Loss=0.0544 + throughput/total_tokens=1,597,248,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8320/500000] + optim/total_grad_norm=5.330 + train/ActionL1Loss=0.0771 + throughput/total_tokens=1,597,440,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8321/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,597,632,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8322/500000] + train/ActionL1Loss=0.0609 + throughput/total_tokens=1,597,824,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8323/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,598,016,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8324/500000] + train/ActionL1Loss=0.0605 + throughput/total_tokens=1,598,208,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8325/500000] + train/ActionL1Loss=0.0635 + throughput/total_tokens=1,598,400,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8326/500000] + train/ActionL1Loss=0.0724 + throughput/total_tokens=1,598,592,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8327/500000] + train/ActionL1Loss=0.0564 + throughput/total_tokens=1,598,784,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8328/500000] + train/ActionL1Loss=0.0642 + throughput/total_tokens=1,598,976,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8329/500000] + train/ActionL1Loss=0.0627 + throughput/total_tokens=1,599,168,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8330/500000] + train/ActionL1Loss=0.0541 + throughput/total_tokens=1,599,360,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8331/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,599,552,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8332/500000] + train/ActionL1Loss=0.0468 + throughput/total_tokens=1,599,744,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8333/500000] + train/ActionL1Loss=0.0678 + throughput/total_tokens=1,599,936,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8334/500000] + train/ActionL1Loss=0.0445 + throughput/total_tokens=1,600,128,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8335/500000] + train/ActionL1Loss=0.0578 + throughput/total_tokens=1,600,320,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8336/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=1,600,512,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8337/500000] + train/ActionL1Loss=0.0746 + throughput/total_tokens=1,600,704,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8338/500000] + train/ActionL1Loss=0.0583 + throughput/total_tokens=1,600,896,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8339/500000] + train/ActionL1Loss=0.0769 + throughput/total_tokens=1,601,088,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8340/500000] + optim/total_grad_norm=6.980 + train/ActionL1Loss=0.0569 + throughput/total_tokens=1,601,280,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8341/500000] + train/ActionL1Loss=0.0721 + throughput/total_tokens=1,601,472,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8342/500000] + train/ActionL1Loss=0.0532 + throughput/total_tokens=1,601,664,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8343/500000] + train/ActionL1Loss=0.0719 + throughput/total_tokens=1,601,856,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8344/500000] + train/ActionL1Loss=0.0434 + throughput/total_tokens=1,602,048,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8345/500000] + train/ActionL1Loss=0.0812 + throughput/total_tokens=1,602,240,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8346/500000] + train/ActionL1Loss=0.0538 + throughput/total_tokens=1,602,432,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8347/500000] + train/ActionL1Loss=0.0614 + throughput/total_tokens=1,602,624,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8348/500000] + train/ActionL1Loss=0.0577 + throughput/total_tokens=1,602,816,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8349/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=1,603,008,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8350/500000] + train/ActionL1Loss=0.0491 + throughput/total_tokens=1,603,200,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8351/500000] + train/ActionL1Loss=0.0623 + throughput/total_tokens=1,603,392,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8352/500000] + train/ActionL1Loss=0.0581 + throughput/total_tokens=1,603,584,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8353/500000] + train/ActionL1Loss=0.0733 + throughput/total_tokens=1,603,776,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8354/500000] + train/ActionL1Loss=0.0471 + throughput/total_tokens=1,603,968,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8355/500000] + train/ActionL1Loss=0.0629 + throughput/total_tokens=1,604,160,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8356/500000] + train/ActionL1Loss=0.0464 + throughput/total_tokens=1,604,352,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8357/500000] + train/ActionL1Loss=0.0551 + throughput/total_tokens=1,604,544,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8358/500000] + train/ActionL1Loss=0.0495 + throughput/total_tokens=1,604,736,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8359/500000] + train/ActionL1Loss=0.0463 + throughput/total_tokens=1,604,928,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8360/500000] + optim/total_grad_norm=6.697 + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,605,120,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8361/500000] + train/ActionL1Loss=0.0894 + throughput/total_tokens=1,605,312,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8362/500000] + train/ActionL1Loss=0.0624 + throughput/total_tokens=1,605,504,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8363/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,605,696,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8364/500000] + train/ActionL1Loss=0.0713 + throughput/total_tokens=1,605,888,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8365/500000] + train/ActionL1Loss=0.0617 + throughput/total_tokens=1,606,080,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8366/500000] + train/ActionL1Loss=0.0520 + throughput/total_tokens=1,606,272,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8367/500000] + train/ActionL1Loss=0.0472 + throughput/total_tokens=1,606,464,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8368/500000] + train/ActionL1Loss=0.0548 + throughput/total_tokens=1,606,656,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8369/500000] + train/ActionL1Loss=0.0616 + throughput/total_tokens=1,606,848,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8370/500000] + train/ActionL1Loss=0.0760 + throughput/total_tokens=1,607,040,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8371/500000] + train/ActionL1Loss=0.0471 + throughput/total_tokens=1,607,232,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8372/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,607,424,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8373/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=1,607,616,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8374/500000] + train/ActionL1Loss=0.0726 + throughput/total_tokens=1,607,808,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8375/500000] + train/ActionL1Loss=0.0808 + throughput/total_tokens=1,608,000,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8376/500000] + train/ActionL1Loss=0.0868 + throughput/total_tokens=1,608,192,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8377/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,608,384,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8378/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,608,576,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8379/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=1,608,768,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8380/500000] + optim/total_grad_norm=5.955 + train/ActionL1Loss=0.0429 + throughput/total_tokens=1,608,960,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8381/500000] + train/ActionL1Loss=0.0531 + throughput/total_tokens=1,609,152,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8382/500000] + train/ActionL1Loss=0.0734 + throughput/total_tokens=1,609,344,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8383/500000] + train/ActionL1Loss=0.0513 + throughput/total_tokens=1,609,536,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8384/500000] + train/ActionL1Loss=0.0533 + throughput/total_tokens=1,609,728,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8385/500000] + train/ActionL1Loss=0.0611 + throughput/total_tokens=1,609,920,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8386/500000] + train/ActionL1Loss=0.0626 + throughput/total_tokens=1,610,112,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8387/500000] + train/ActionL1Loss=0.0502 + throughput/total_tokens=1,610,304,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8388/500000] + train/ActionL1Loss=0.0531 + throughput/total_tokens=1,610,496,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8389/500000] + train/ActionL1Loss=0.0598 + throughput/total_tokens=1,610,688,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8390/500000] + train/ActionL1Loss=0.0577 + throughput/total_tokens=1,610,880,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8391/500000] + train/ActionL1Loss=0.0612 + throughput/total_tokens=1,611,072,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8392/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=1,611,264,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8393/500000] + train/ActionL1Loss=0.0623 + throughput/total_tokens=1,611,456,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8394/500000] + train/ActionL1Loss=0.0704 + throughput/total_tokens=1,611,648,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8395/500000] + train/ActionL1Loss=0.0543 + throughput/total_tokens=1,611,840,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8396/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=1,612,032,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8397/500000] + train/ActionL1Loss=0.0505 + throughput/total_tokens=1,612,224,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8398/500000] + train/ActionL1Loss=0.0442 + throughput/total_tokens=1,612,416,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8399/500000] + train/ActionL1Loss=0.0490 + throughput/total_tokens=1,612,608,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8400/500000] + optim/total_grad_norm=6.614 + train/ActionL1Loss=0.0581 + throughput/total_tokens=1,612,800,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8401/500000] + train/ActionL1Loss=0.0639 + throughput/total_tokens=1,612,992,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8402/500000] + train/ActionL1Loss=0.0586 + throughput/total_tokens=1,613,184,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8403/500000] + train/ActionL1Loss=0.0591 + throughput/total_tokens=1,613,376,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8404/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,613,568,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8405/500000] + train/ActionL1Loss=0.0718 + throughput/total_tokens=1,613,760,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8406/500000] + train/ActionL1Loss=0.0491 + throughput/total_tokens=1,613,952,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8407/500000] + train/ActionL1Loss=0.0598 + throughput/total_tokens=1,614,144,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8408/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=1,614,336,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8409/500000] + train/ActionL1Loss=0.0568 + throughput/total_tokens=1,614,528,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8410/500000] + train/ActionL1Loss=0.0407 + throughput/total_tokens=1,614,720,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8411/500000] + train/ActionL1Loss=0.0590 + throughput/total_tokens=1,614,912,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8412/500000] + train/ActionL1Loss=0.0549 + throughput/total_tokens=1,615,104,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8413/500000] + train/ActionL1Loss=0.0671 + throughput/total_tokens=1,615,296,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8414/500000] + train/ActionL1Loss=0.0764 + throughput/total_tokens=1,615,488,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8415/500000] + train/ActionL1Loss=0.0688 + throughput/total_tokens=1,615,680,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8416/500000] + train/ActionL1Loss=0.0583 + throughput/total_tokens=1,615,872,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8417/500000] + train/ActionL1Loss=0.0535 + throughput/total_tokens=1,616,064,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8418/500000] + train/ActionL1Loss=0.0630 + throughput/total_tokens=1,616,256,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8419/500000] + train/ActionL1Loss=0.0593 + throughput/total_tokens=1,616,448,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8420/500000] + optim/total_grad_norm=6.694 + train/ActionL1Loss=0.0646 + throughput/total_tokens=1,616,640,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8421/500000] + train/ActionL1Loss=0.0592 + throughput/total_tokens=1,616,832,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8422/500000] + train/ActionL1Loss=0.0418 + throughput/total_tokens=1,617,024,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8423/500000] + train/ActionL1Loss=0.0772 + throughput/total_tokens=1,617,216,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8424/500000] + train/ActionL1Loss=0.0524 + throughput/total_tokens=1,617,408,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8425/500000] + train/ActionL1Loss=0.0802 + throughput/total_tokens=1,617,600,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8426/500000] + train/ActionL1Loss=0.0500 + throughput/total_tokens=1,617,792,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8427/500000] + train/ActionL1Loss=0.0664 + throughput/total_tokens=1,617,984,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8428/500000] + train/ActionL1Loss=0.0618 + throughput/total_tokens=1,618,176,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8429/500000] + train/ActionL1Loss=0.0687 + throughput/total_tokens=1,618,368,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8430/500000] + train/ActionL1Loss=0.0681 + throughput/total_tokens=1,618,560,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8431/500000] + train/ActionL1Loss=0.0763 + throughput/total_tokens=1,618,752,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8432/500000] + train/ActionL1Loss=0.0669 + throughput/total_tokens=1,618,944,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8433/500000] + train/ActionL1Loss=0.0835 + throughput/total_tokens=1,619,136,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8434/500000] + train/ActionL1Loss=0.0575 + throughput/total_tokens=1,619,328,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8435/500000] + train/ActionL1Loss=0.0467 + throughput/total_tokens=1,619,520,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8436/500000] + train/ActionL1Loss=0.0542 + throughput/total_tokens=1,619,712,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8437/500000] + train/ActionL1Loss=0.0562 + throughput/total_tokens=1,619,904,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8438/500000] + train/ActionL1Loss=0.0570 + throughput/total_tokens=1,620,096,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8439/500000] + train/ActionL1Loss=0.0496 + throughput/total_tokens=1,620,288,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8440/500000] + optim/total_grad_norm=5.799 + train/ActionL1Loss=0.0580 + throughput/total_tokens=1,620,480,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8441/500000] + train/ActionL1Loss=0.0520 + throughput/total_tokens=1,620,672,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8442/500000] + train/ActionL1Loss=0.0443 + throughput/total_tokens=1,620,864,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8443/500000] + train/ActionL1Loss=0.0676 + throughput/total_tokens=1,621,056,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8444/500000] + train/ActionL1Loss=0.0550 + throughput/total_tokens=1,621,248,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8445/500000] + train/ActionL1Loss=0.0650 + throughput/total_tokens=1,621,440,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8446/500000] + train/ActionL1Loss=0.0553 + throughput/total_tokens=1,621,632,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8447/500000] + train/ActionL1Loss=0.0511 + throughput/total_tokens=1,621,824,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8448/500000] + train/ActionL1Loss=0.0574 + throughput/total_tokens=1,622,016,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8449/500000] + train/ActionL1Loss=0.0640 + throughput/total_tokens=1,622,208,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8450/500000] + train/ActionL1Loss=0.0530 + throughput/total_tokens=1,622,400,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8451/500000] + train/ActionL1Loss=0.0457 + throughput/total_tokens=1,622,592,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8452/500000] + train/ActionL1Loss=0.0529 + throughput/total_tokens=1,622,784,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8453/500000] + train/ActionL1Loss=0.0723 + throughput/total_tokens=1,622,976,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8454/500000] + train/ActionL1Loss=0.0649 + throughput/total_tokens=1,623,168,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8455/500000] + train/ActionL1Loss=0.0503 + throughput/total_tokens=1,623,360,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8456/500000] + train/ActionL1Loss=0.0498 + throughput/total_tokens=1,623,552,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8457/500000] + train/ActionL1Loss=0.0609 + throughput/total_tokens=1,623,744,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8458/500000] + train/ActionL1Loss=0.0690 + throughput/total_tokens=1,623,936,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8459/500000] + train/ActionL1Loss=0.0565 + throughput/total_tokens=1,624,128,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8460/500000] + optim/total_grad_norm=5.515 + train/ActionL1Loss=0.0519 + throughput/total_tokens=1,624,320,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8461/500000] + train/ActionL1Loss=0.0620 + throughput/total_tokens=1,624,512,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8462/500000] + train/ActionL1Loss=0.0533 + throughput/total_tokens=1,624,704,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8463/500000] + train/ActionL1Loss=0.0440 + throughput/total_tokens=1,624,896,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8464/500000] + train/ActionL1Loss=0.0556 + throughput/total_tokens=1,625,088,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8465/500000] + train/ActionL1Loss=0.0554 + throughput/total_tokens=1,625,280,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8466/500000] + train/ActionL1Loss=0.0550 + throughput/total_tokens=1,625,472,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8467/500000] + train/ActionL1Loss=0.0683 + throughput/total_tokens=1,625,664,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8468/500000] + train/ActionL1Loss=0.0407 + throughput/total_tokens=1,625,856,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8469/500000] + train/ActionL1Loss=0.0579 + throughput/total_tokens=1,626,048,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8470/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,626,240,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8471/500000] + train/ActionL1Loss=0.0548 + throughput/total_tokens=1,626,432,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8472/500000] + train/ActionL1Loss=0.0704 + throughput/total_tokens=1,626,624,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8473/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,626,816,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8474/500000] + train/ActionL1Loss=0.0770 + throughput/total_tokens=1,627,008,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8475/500000] + train/ActionL1Loss=0.0713 + throughput/total_tokens=1,627,200,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8476/500000] + train/ActionL1Loss=0.0485 + throughput/total_tokens=1,627,392,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8477/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,627,584,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8478/500000] + train/ActionL1Loss=0.0511 + throughput/total_tokens=1,627,776,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8479/500000] + train/ActionL1Loss=0.0580 + throughput/total_tokens=1,627,968,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8480/500000] + optim/total_grad_norm=7.240 + train/ActionL1Loss=0.0653 + throughput/total_tokens=1,628,160,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8481/500000] + train/ActionL1Loss=0.0711 + throughput/total_tokens=1,628,352,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8482/500000] + train/ActionL1Loss=0.0540 + throughput/total_tokens=1,628,544,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8483/500000] + train/ActionL1Loss=0.0829 + throughput/total_tokens=1,628,736,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8484/500000] + train/ActionL1Loss=0.0581 + throughput/total_tokens=1,628,928,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8485/500000] + train/ActionL1Loss=0.0558 + throughput/total_tokens=1,629,120,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8486/500000] + train/ActionL1Loss=0.0695 + throughput/total_tokens=1,629,312,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8487/500000] + train/ActionL1Loss=0.0424 + throughput/total_tokens=1,629,504,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8488/500000] + train/ActionL1Loss=0.0822 + throughput/total_tokens=1,629,696,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8489/500000] + train/ActionL1Loss=0.0727 + throughput/total_tokens=1,629,888,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8490/500000] + train/ActionL1Loss=0.0534 + throughput/total_tokens=1,630,080,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8491/500000] + train/ActionL1Loss=0.0601 + throughput/total_tokens=1,630,272,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8492/500000] + train/ActionL1Loss=0.0570 + throughput/total_tokens=1,630,464,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8493/500000] + train/ActionL1Loss=0.0705 + throughput/total_tokens=1,630,656,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8494/500000] + train/ActionL1Loss=0.0566 + throughput/total_tokens=1,630,848,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8495/500000] + train/ActionL1Loss=0.0426 + throughput/total_tokens=1,631,040,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8496/500000] + train/ActionL1Loss=0.0555 + throughput/total_tokens=1,631,232,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8497/500000] + train/ActionL1Loss=0.0500 + throughput/total_tokens=1,631,424,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8498/500000] + train/ActionL1Loss=0.0681 + throughput/total_tokens=1,631,616,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8499/500000] + train/ActionL1Loss=0.0829 + throughput/total_tokens=1,631,808,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8500/500000] + optim/total_grad_norm=5.669 + train/ActionL1Loss=0.0558 + throughput/total_tokens=1,632,000,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +10/02 [15:15:12] INFO | >> Saving config... checkpoint.py:608 +10/02 [15:16:07] INFO | >> Saving model state... checkpoint.py:796 +10/02 [15:17:19] INFO | >> Saving optim state... checkpoint.py:811 +10/02 [15:18:51] INFO | >> Saving trainer state... checkpoint.py:670 + INFO | >> Saving config... checkpoint.py:608 +[step=8501/500000] + train/ActionL1Loss=0.0632 + throughput/total_tokens=1,632,192,000 + throughput/device/tokens_per_second=1,209 + throughput/device/batches_per_second=0.0504 +[step=8502/500000] + train/ActionL1Loss=0.0503 + throughput/total_tokens=1,632,384,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8503/500000] + train/ActionL1Loss=0.0437 + throughput/total_tokens=1,632,576,000 + throughput/device/tokens_per_second=1,162 + throughput/device/batches_per_second=0.0484 +[step=8504/500000] + train/ActionL1Loss=0.0491 + throughput/total_tokens=1,632,768,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8505/500000] + train/ActionL1Loss=0.0663 + throughput/total_tokens=1,632,960,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8506/500000] + train/ActionL1Loss=0.0815 + throughput/total_tokens=1,633,152,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8507/500000] + train/ActionL1Loss=0.0818 + throughput/total_tokens=1,633,344,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8508/500000] + train/ActionL1Loss=0.0605 + throughput/total_tokens=1,633,536,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8509/500000] + train/ActionL1Loss=0.0554 + throughput/total_tokens=1,633,728,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8510/500000] + train/ActionL1Loss=0.0742 + throughput/total_tokens=1,633,920,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8511/500000] + train/ActionL1Loss=0.0684 + throughput/total_tokens=1,634,112,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8512/500000] + train/ActionL1Loss=0.0587 + throughput/total_tokens=1,634,304,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8513/500000] + train/ActionL1Loss=0.0457 + throughput/total_tokens=1,634,496,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8514/500000] + train/ActionL1Loss=0.0581 + throughput/total_tokens=1,634,688,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8515/500000] + train/ActionL1Loss=0.0567 + throughput/total_tokens=1,634,880,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8516/500000] + train/ActionL1Loss=0.0536 + throughput/total_tokens=1,635,072,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8517/500000] + train/ActionL1Loss=0.0517 + throughput/total_tokens=1,635,264,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8518/500000] + train/ActionL1Loss=0.0577 + throughput/total_tokens=1,635,456,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8519/500000] + train/ActionL1Loss=0.0440 + throughput/total_tokens=1,635,648,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8520/500000] + optim/total_grad_norm=7.000 + train/ActionL1Loss=0.0487 + throughput/total_tokens=1,635,840,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=8521/500000] + train/ActionL1Loss=0.0564 + throughput/total_tokens=1,636,032,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8522/500000] + train/ActionL1Loss=0.0524 + throughput/total_tokens=1,636,224,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8523/500000] + train/ActionL1Loss=0.0544 + throughput/total_tokens=1,636,416,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8524/500000] + train/ActionL1Loss=0.0809 + throughput/total_tokens=1,636,608,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8525/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=1,636,800,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8526/500000] + train/ActionL1Loss=0.0572 + throughput/total_tokens=1,636,992,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8527/500000] + train/ActionL1Loss=0.0672 + throughput/total_tokens=1,637,184,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8528/500000] + train/ActionL1Loss=0.0499 + throughput/total_tokens=1,637,376,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8529/500000] + train/ActionL1Loss=0.0585 + throughput/total_tokens=1,637,568,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8530/500000] + train/ActionL1Loss=0.0674 + throughput/total_tokens=1,637,760,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8531/500000] + train/ActionL1Loss=0.0568 + throughput/total_tokens=1,637,952,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8532/500000] + train/ActionL1Loss=0.0546 + throughput/total_tokens=1,638,144,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8533/500000] + train/ActionL1Loss=0.0767 + throughput/total_tokens=1,638,336,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8534/500000] + train/ActionL1Loss=0.0674 + throughput/total_tokens=1,638,528,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8535/500000] + train/ActionL1Loss=0.0601 + throughput/total_tokens=1,638,720,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8536/500000] + train/ActionL1Loss=0.0417 + throughput/total_tokens=1,638,912,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8537/500000] + train/ActionL1Loss=0.0690 + throughput/total_tokens=1,639,104,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8538/500000] + train/ActionL1Loss=0.0476 + throughput/total_tokens=1,639,296,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8539/500000] + train/ActionL1Loss=0.0635 + throughput/total_tokens=1,639,488,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8540/500000] + optim/total_grad_norm=5.603 + train/ActionL1Loss=0.0471 + throughput/total_tokens=1,639,680,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8541/500000] + train/ActionL1Loss=0.0506 + throughput/total_tokens=1,639,872,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8542/500000] + train/ActionL1Loss=0.0656 + throughput/total_tokens=1,640,064,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8543/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=1,640,256,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8544/500000] + train/ActionL1Loss=0.0630 + throughput/total_tokens=1,640,448,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8545/500000] + train/ActionL1Loss=0.0507 + throughput/total_tokens=1,640,640,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8546/500000] + train/ActionL1Loss=0.0627 + throughput/total_tokens=1,640,832,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8547/500000] + train/ActionL1Loss=0.0587 + throughput/total_tokens=1,641,024,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8548/500000] + train/ActionL1Loss=0.0550 + throughput/total_tokens=1,641,216,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8549/500000] + train/ActionL1Loss=0.0510 + throughput/total_tokens=1,641,408,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8550/500000] + train/ActionL1Loss=0.0587 + throughput/total_tokens=1,641,600,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8551/500000] + train/ActionL1Loss=0.0582 + throughput/total_tokens=1,641,792,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8552/500000] + train/ActionL1Loss=0.0542 + throughput/total_tokens=1,641,984,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8553/500000] + train/ActionL1Loss=0.0404 + throughput/total_tokens=1,642,176,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8554/500000] + train/ActionL1Loss=0.0682 + throughput/total_tokens=1,642,368,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8555/500000] + train/ActionL1Loss=0.0402 + throughput/total_tokens=1,642,560,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8556/500000] + train/ActionL1Loss=0.0521 + throughput/total_tokens=1,642,752,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8557/500000] + train/ActionL1Loss=0.0865 + throughput/total_tokens=1,642,944,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8558/500000] + train/ActionL1Loss=0.0516 + throughput/total_tokens=1,643,136,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8559/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=1,643,328,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8560/500000] + optim/total_grad_norm=6.421 + train/ActionL1Loss=0.0478 + throughput/total_tokens=1,643,520,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=8561/500000] + train/ActionL1Loss=0.0537 + throughput/total_tokens=1,643,712,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8562/500000] + train/ActionL1Loss=0.0451 + throughput/total_tokens=1,643,904,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8563/500000] + train/ActionL1Loss=0.0593 + throughput/total_tokens=1,644,096,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8564/500000] + train/ActionL1Loss=0.0529 + throughput/total_tokens=1,644,288,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8565/500000] + train/ActionL1Loss=0.0715 + throughput/total_tokens=1,644,480,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8566/500000] + train/ActionL1Loss=0.0485 + throughput/total_tokens=1,644,672,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8567/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=1,644,864,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8568/500000] + train/ActionL1Loss=0.0556 + throughput/total_tokens=1,645,056,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8569/500000] + train/ActionL1Loss=0.0534 + throughput/total_tokens=1,645,248,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8570/500000] + train/ActionL1Loss=0.0453 + throughput/total_tokens=1,645,440,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8571/500000] + train/ActionL1Loss=0.0669 + throughput/total_tokens=1,645,632,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8572/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=1,645,824,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8573/500000] + train/ActionL1Loss=0.0509 + throughput/total_tokens=1,646,016,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8574/500000] + train/ActionL1Loss=0.0478 + throughput/total_tokens=1,646,208,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8575/500000] + train/ActionL1Loss=0.0469 + throughput/total_tokens=1,646,400,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8576/500000] + train/ActionL1Loss=0.0494 + throughput/total_tokens=1,646,592,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8577/500000] + train/ActionL1Loss=0.0614 + throughput/total_tokens=1,646,784,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8578/500000] + train/ActionL1Loss=0.0661 + throughput/total_tokens=1,646,976,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8579/500000] + train/ActionL1Loss=0.0499 + throughput/total_tokens=1,647,168,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8580/500000] + optim/total_grad_norm=4.187 + train/ActionL1Loss=0.0700 + throughput/total_tokens=1,647,360,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8581/500000] + train/ActionL1Loss=0.0604 + throughput/total_tokens=1,647,552,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8582/500000] + train/ActionL1Loss=0.0574 + throughput/total_tokens=1,647,744,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8583/500000] + train/ActionL1Loss=0.0562 + throughput/total_tokens=1,647,936,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8584/500000] + train/ActionL1Loss=0.0644 + throughput/total_tokens=1,648,128,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8585/500000] + train/ActionL1Loss=0.0557 + throughput/total_tokens=1,648,320,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8586/500000] + train/ActionL1Loss=0.0614 + throughput/total_tokens=1,648,512,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8587/500000] + train/ActionL1Loss=0.0585 + throughput/total_tokens=1,648,704,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8588/500000] + train/ActionL1Loss=0.0576 + throughput/total_tokens=1,648,896,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8589/500000] + train/ActionL1Loss=0.0528 + throughput/total_tokens=1,649,088,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8590/500000] + train/ActionL1Loss=0.0426 + throughput/total_tokens=1,649,280,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8591/500000] + train/ActionL1Loss=0.0522 + throughput/total_tokens=1,649,472,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8592/500000] + train/ActionL1Loss=0.0529 + throughput/total_tokens=1,649,664,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8593/500000] + train/ActionL1Loss=0.0512 + throughput/total_tokens=1,649,856,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8594/500000] + train/ActionL1Loss=0.0668 + throughput/total_tokens=1,650,048,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8595/500000] + train/ActionL1Loss=0.0483 + throughput/total_tokens=1,650,240,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8596/500000] + train/ActionL1Loss=0.0577 + throughput/total_tokens=1,650,432,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8597/500000] + train/ActionL1Loss=0.0577 + throughput/total_tokens=1,650,624,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8598/500000] + train/ActionL1Loss=0.0537 + throughput/total_tokens=1,650,816,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8599/500000] + train/ActionL1Loss=0.0443 + throughput/total_tokens=1,651,008,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8600/500000] + optim/total_grad_norm=4.654 + train/ActionL1Loss=0.0572 + throughput/total_tokens=1,651,200,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8601/500000] + train/ActionL1Loss=0.0534 + throughput/total_tokens=1,651,392,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8602/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,651,584,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8603/500000] + train/ActionL1Loss=0.0489 + throughput/total_tokens=1,651,776,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8604/500000] + train/ActionL1Loss=0.0369 + throughput/total_tokens=1,651,968,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8605/500000] + train/ActionL1Loss=0.0541 + throughput/total_tokens=1,652,160,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8606/500000] + train/ActionL1Loss=0.0669 + throughput/total_tokens=1,652,352,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8607/500000] + train/ActionL1Loss=0.0511 + throughput/total_tokens=1,652,544,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8608/500000] + train/ActionL1Loss=0.0455 + throughput/total_tokens=1,652,736,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8609/500000] + train/ActionL1Loss=0.0521 + throughput/total_tokens=1,652,928,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8610/500000] + train/ActionL1Loss=0.0487 + throughput/total_tokens=1,653,120,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8611/500000] + train/ActionL1Loss=0.0476 + throughput/total_tokens=1,653,312,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8612/500000] + train/ActionL1Loss=0.0714 + throughput/total_tokens=1,653,504,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8613/500000] + train/ActionL1Loss=0.0787 + throughput/total_tokens=1,653,696,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8614/500000] + train/ActionL1Loss=0.0643 + throughput/total_tokens=1,653,888,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8615/500000] + train/ActionL1Loss=0.0476 + throughput/total_tokens=1,654,080,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8616/500000] + train/ActionL1Loss=0.0729 + throughput/total_tokens=1,654,272,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8617/500000] + train/ActionL1Loss=0.0697 + throughput/total_tokens=1,654,464,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8618/500000] + train/ActionL1Loss=0.0599 + throughput/total_tokens=1,654,656,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8619/500000] + train/ActionL1Loss=0.0600 + throughput/total_tokens=1,654,848,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8620/500000] + optim/total_grad_norm=5.179 + train/ActionL1Loss=0.0532 + throughput/total_tokens=1,655,040,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8621/500000] + train/ActionL1Loss=0.0517 + throughput/total_tokens=1,655,232,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8622/500000] + train/ActionL1Loss=0.0582 + throughput/total_tokens=1,655,424,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8623/500000] + train/ActionL1Loss=0.0614 + throughput/total_tokens=1,655,616,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8624/500000] + train/ActionL1Loss=0.0698 + throughput/total_tokens=1,655,808,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8625/500000] + train/ActionL1Loss=0.0629 + throughput/total_tokens=1,656,000,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8626/500000] + train/ActionL1Loss=0.0609 + throughput/total_tokens=1,656,192,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8627/500000] + train/ActionL1Loss=0.0834 + throughput/total_tokens=1,656,384,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8628/500000] + train/ActionL1Loss=0.0734 + throughput/total_tokens=1,656,576,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8629/500000] + train/ActionL1Loss=0.0534 + throughput/total_tokens=1,656,768,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8630/500000] + train/ActionL1Loss=0.0392 + throughput/total_tokens=1,656,960,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8631/500000] + train/ActionL1Loss=0.0466 + throughput/total_tokens=1,657,152,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8632/500000] + train/ActionL1Loss=0.0659 + throughput/total_tokens=1,657,344,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8633/500000] + train/ActionL1Loss=0.0603 + throughput/total_tokens=1,657,536,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8634/500000] + train/ActionL1Loss=0.0814 + throughput/total_tokens=1,657,728,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8635/500000] + train/ActionL1Loss=0.0549 + throughput/total_tokens=1,657,920,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8636/500000] + train/ActionL1Loss=0.0726 + throughput/total_tokens=1,658,112,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8637/500000] + train/ActionL1Loss=0.0643 + throughput/total_tokens=1,658,304,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8638/500000] + train/ActionL1Loss=0.0431 + throughput/total_tokens=1,658,496,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8639/500000] + train/ActionL1Loss=0.0576 + throughput/total_tokens=1,658,688,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8640/500000] + optim/total_grad_norm=6.998 + train/ActionL1Loss=0.0577 + throughput/total_tokens=1,658,880,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8641/500000] + train/ActionL1Loss=0.0773 + throughput/total_tokens=1,659,072,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8642/500000] + train/ActionL1Loss=0.0487 + throughput/total_tokens=1,659,264,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8643/500000] + train/ActionL1Loss=0.0540 + throughput/total_tokens=1,659,456,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8644/500000] + train/ActionL1Loss=0.0572 + throughput/total_tokens=1,659,648,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8645/500000] + train/ActionL1Loss=0.0573 + throughput/total_tokens=1,659,840,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8646/500000] + train/ActionL1Loss=0.0766 + throughput/total_tokens=1,660,032,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8647/500000] + train/ActionL1Loss=0.0475 + throughput/total_tokens=1,660,224,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8648/500000] + train/ActionL1Loss=0.0536 + throughput/total_tokens=1,660,416,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8649/500000] + train/ActionL1Loss=0.0590 + throughput/total_tokens=1,660,608,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 +[step=8650/500000] + train/ActionL1Loss=0.0525 + throughput/total_tokens=1,660,800,000 + throughput/device/tokens_per_second=1,169 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8651/500000] + train/ActionL1Loss=0.0608 + throughput/total_tokens=1,660,992,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8652/500000] + train/ActionL1Loss=0.0648 + throughput/total_tokens=1,661,184,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8653/500000] + train/ActionL1Loss=0.0410 + throughput/total_tokens=1,661,376,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8654/500000] + train/ActionL1Loss=0.0749 + throughput/total_tokens=1,661,568,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8655/500000] + train/ActionL1Loss=0.0478 + throughput/total_tokens=1,661,760,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8656/500000] + train/ActionL1Loss=0.0473 + throughput/total_tokens=1,661,952,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8657/500000] + train/ActionL1Loss=0.0501 + throughput/total_tokens=1,662,144,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8658/500000] + train/ActionL1Loss=0.0489 + throughput/total_tokens=1,662,336,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8659/500000] + train/ActionL1Loss=0.0464 + throughput/total_tokens=1,662,528,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8660/500000] + optim/total_grad_norm=5.296 + train/ActionL1Loss=0.0550 + throughput/total_tokens=1,662,720,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8661/500000] + train/ActionL1Loss=0.0492 + throughput/total_tokens=1,662,912,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8662/500000] + train/ActionL1Loss=0.0610 + throughput/total_tokens=1,663,104,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8663/500000] + train/ActionL1Loss=0.0428 + throughput/total_tokens=1,663,296,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8664/500000] + train/ActionL1Loss=0.0476 + throughput/total_tokens=1,663,488,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8665/500000] + train/ActionL1Loss=0.0513 + throughput/total_tokens=1,663,680,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8666/500000] + train/ActionL1Loss=0.0584 + throughput/total_tokens=1,663,872,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8667/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=1,664,064,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8668/500000] + train/ActionL1Loss=0.0578 + throughput/total_tokens=1,664,256,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8669/500000] + train/ActionL1Loss=0.0423 + throughput/total_tokens=1,664,448,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8670/500000] + train/ActionL1Loss=0.0495 + throughput/total_tokens=1,664,640,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8671/500000] + train/ActionL1Loss=0.0602 + throughput/total_tokens=1,664,832,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8672/500000] + train/ActionL1Loss=0.0610 + throughput/total_tokens=1,665,024,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8673/500000] + train/ActionL1Loss=0.0735 + throughput/total_tokens=1,665,216,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8674/500000] + train/ActionL1Loss=0.0671 + throughput/total_tokens=1,665,408,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8675/500000] + train/ActionL1Loss=0.0654 + throughput/total_tokens=1,665,600,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8676/500000] + train/ActionL1Loss=0.0486 + throughput/total_tokens=1,665,792,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8677/500000] + train/ActionL1Loss=0.0584 + throughput/total_tokens=1,665,984,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8678/500000] + train/ActionL1Loss=0.0729 + throughput/total_tokens=1,666,176,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8679/500000] + train/ActionL1Loss=0.0633 + throughput/total_tokens=1,666,368,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8680/500000] + optim/total_grad_norm=5.683 + train/ActionL1Loss=0.0624 + throughput/total_tokens=1,666,560,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8681/500000] + train/ActionL1Loss=0.0628 + throughput/total_tokens=1,666,752,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8682/500000] + train/ActionL1Loss=0.0666 + throughput/total_tokens=1,666,944,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8683/500000] + train/ActionL1Loss=0.0709 + throughput/total_tokens=1,667,136,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8684/500000] + train/ActionL1Loss=0.0693 + throughput/total_tokens=1,667,328,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8685/500000] + train/ActionL1Loss=0.0616 + throughput/total_tokens=1,667,520,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8686/500000] + train/ActionL1Loss=0.0588 + throughput/total_tokens=1,667,712,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8687/500000] + train/ActionL1Loss=0.0634 + throughput/total_tokens=1,667,904,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8688/500000] + train/ActionL1Loss=0.0681 + throughput/total_tokens=1,668,096,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8689/500000] + train/ActionL1Loss=0.0500 + throughput/total_tokens=1,668,288,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8690/500000] + train/ActionL1Loss=0.0589 + throughput/total_tokens=1,668,480,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8691/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,668,672,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8692/500000] + train/ActionL1Loss=0.0558 + throughput/total_tokens=1,668,864,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8693/500000] + train/ActionL1Loss=0.0653 + throughput/total_tokens=1,669,056,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8694/500000] + train/ActionL1Loss=0.0637 + throughput/total_tokens=1,669,248,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8695/500000] + train/ActionL1Loss=0.0487 + throughput/total_tokens=1,669,440,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8696/500000] + train/ActionL1Loss=0.0793 + throughput/total_tokens=1,669,632,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8697/500000] + train/ActionL1Loss=0.0606 + throughput/total_tokens=1,669,824,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8698/500000] + train/ActionL1Loss=0.0531 + throughput/total_tokens=1,670,016,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8699/500000] + train/ActionL1Loss=0.0621 + throughput/total_tokens=1,670,208,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8700/500000] + optim/total_grad_norm=7.272 + train/ActionL1Loss=0.0599 + throughput/total_tokens=1,670,400,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8701/500000] + train/ActionL1Loss=0.0622 + throughput/total_tokens=1,670,592,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8702/500000] + train/ActionL1Loss=0.0770 + throughput/total_tokens=1,670,784,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8703/500000] + train/ActionL1Loss=0.0608 + throughput/total_tokens=1,670,976,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8704/500000] + train/ActionL1Loss=0.0581 + throughput/total_tokens=1,671,168,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8705/500000] + train/ActionL1Loss=0.0619 + throughput/total_tokens=1,671,360,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8706/500000] + train/ActionL1Loss=0.0625 + throughput/total_tokens=1,671,552,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8707/500000] + train/ActionL1Loss=0.0576 + throughput/total_tokens=1,671,744,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8708/500000] + train/ActionL1Loss=0.0615 + throughput/total_tokens=1,671,936,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8709/500000] + train/ActionL1Loss=0.0687 + throughput/total_tokens=1,672,128,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8710/500000] + train/ActionL1Loss=0.0703 + throughput/total_tokens=1,672,320,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 + System/Peak GPU Memory (MB)=46,917 +[step=8711/500000] + train/ActionL1Loss=0.0571 + throughput/total_tokens=1,672,512,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8712/500000] + train/ActionL1Loss=0.0746 + throughput/total_tokens=1,672,704,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8713/500000] + train/ActionL1Loss=0.0645 + throughput/total_tokens=1,672,896,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8714/500000] + train/ActionL1Loss=0.0667 + throughput/total_tokens=1,673,088,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8715/500000] + train/ActionL1Loss=0.0517 + throughput/total_tokens=1,673,280,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8716/500000] + train/ActionL1Loss=0.0591 + throughput/total_tokens=1,673,472,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0486 +[step=8717/500000] + train/ActionL1Loss=0.0586 + throughput/total_tokens=1,673,664,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8718/500000] + train/ActionL1Loss=0.0679 + throughput/total_tokens=1,673,856,000 + throughput/device/tokens_per_second=1,165 + throughput/device/batches_per_second=0.0485 +[step=8719/500000] + train/ActionL1Loss=0.0732 + throughput/total_tokens=1,674,048,000 + throughput/device/tokens_per_second=1,164 + throughput/device/batches_per_second=0.0485 +[step=8720/500000] + optim/total_grad_norm=5.450 + train/ActionL1Loss=0.0558 + throughput/total_tokens=1,674,240,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8721/500000] + train/ActionL1Loss=0.0694 + throughput/total_tokens=1,674,432,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8722/500000] + train/ActionL1Loss=0.0614 + throughput/total_tokens=1,674,624,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8723/500000] + train/ActionL1Loss=0.0558 + throughput/total_tokens=1,674,816,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8724/500000] + train/ActionL1Loss=0.0611 + throughput/total_tokens=1,675,008,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8725/500000] + train/ActionL1Loss=0.0611 + throughput/total_tokens=1,675,200,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8726/500000] + train/ActionL1Loss=0.0555 + throughput/total_tokens=1,675,392,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 +[step=8727/500000] + train/ActionL1Loss=0.0603 + throughput/total_tokens=1,675,584,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8728/500000] + train/ActionL1Loss=0.0561 + throughput/total_tokens=1,675,776,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8729/500000] + train/ActionL1Loss=0.0519 + throughput/total_tokens=1,675,968,000 + throughput/device/tokens_per_second=1,168 + throughput/device/batches_per_second=0.0487 +[step=8730/500000] + train/ActionL1Loss=0.0613 + throughput/total_tokens=1,676,160,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0487 + System/Peak GPU Memory (MB)=46,917 +[step=8731/500000] + train/ActionL1Loss=0.0522 + throughput/total_tokens=1,676,352,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8732/500000] + train/ActionL1Loss=0.0512 + throughput/total_tokens=1,676,544,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8733/500000] + train/ActionL1Loss=0.0529 + throughput/total_tokens=1,676,736,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8734/500000] + train/ActionL1Loss=0.0532 + throughput/total_tokens=1,676,928,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8735/500000] + train/ActionL1Loss=0.0564 + throughput/total_tokens=1,677,120,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8736/500000] + train/ActionL1Loss=0.0585 + throughput/total_tokens=1,677,312,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8737/500000] + train/ActionL1Loss=0.0651 + throughput/total_tokens=1,677,504,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8738/500000] + train/ActionL1Loss=0.0477 + throughput/total_tokens=1,677,696,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8739/500000] + train/ActionL1Loss=0.0759 + throughput/total_tokens=1,677,888,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8740/500000] + optim/total_grad_norm=7.025 + train/ActionL1Loss=0.0444 + throughput/total_tokens=1,678,080,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 + System/Peak GPU Memory (MB)=46,917 +[step=8741/500000] + train/ActionL1Loss=0.0512 + throughput/total_tokens=1,678,272,000 + throughput/device/tokens_per_second=1,167 + throughput/device/batches_per_second=0.0486 +[step=8742/500000] + train/ActionL1Loss=0.0517 + throughput/total_tokens=1,678,464,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 +[step=8743/500000] + train/ActionL1Loss=0.0468 + throughput/total_tokens=1,678,656,000 + throughput/device/tokens_per_second=1,166 + throughput/device/batches_per_second=0.0486 diff --git a/all_l1/wandb/wandb/run-20250930_131250-ea1k0g3y/files/requirements.txt b/all_l1/wandb/wandb/run-20250930_131250-ea1k0g3y/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..5a5e971a5d9f3b6f6a75248e203682c5fbcd69e8 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_131250-ea1k0g3y/files/requirements.txt @@ -0,0 +1,286 @@ +ai2-molmo==0.0.0 +astunparse==1.6.3 +flatbuffers==25.2.10 +gast==0.6.0 +google-pasta==0.2.0 +h5py==3.14.0 +libclang==18.1.1 +Markdown==3.9 +namex==0.1.0 +opt_einsum==3.4.0 +optree==0.17.0 +tensorboard-data-server==0.7.2 +tensorflow-io-gcs-filesystem==0.37.1 +termcolor==3.1.0 +Werkzeug==3.1.3 +Brotli==1.1.0 +Farama-Notifications==0.0.4 +MarkupSafe==2.1.5 +PyYAML==6.0.2 +absl-py==2.3.1 +accelerate==1.10.1 +ai2-molmo==0.0.0 +aiofiles==24.1.0 +aiohappyeyeballs==2.6.1 +aiohttp==3.12.15 +aiosignal==1.4.0 +annotated-types==0.7.0 +antlr4-python3-runtime==4.9.3 +anyio==4.10.0 +array_record==0.8.1 +async-timeout==5.0.1 +attrs==25.3.0 +av==15.1.0 +backports.tarfile==1.2.0 +beaker-gantry==3.2.0 +beaker-py==2.5.0 +black==23.12.1 +blinker==1.9.0 +boltons==25.0.0 +boto3==1.40.33 +botocore==1.40.33 +build==1.3.0 +cached_path==1.7.3 +cached-property==2.0.1 +cachetools==5.5.2 +certifi==2025.8.3 +cffi==2.0.0 +charset-normalizer==3.4.3 +click==8.2.1 +click-help-colors==0.9.4 +click-option-group==0.5.7 +cloudpickle==3.1.1 +cmake==4.1.0 +contourpy==1.3.2 +cryptography==46.0.1 +cycler==0.12.1 +dataclass-extensions==0.2.3 +datasets==3.6.0 +decorator==5.2.1 +deepdiff==8.6.1 +diffusers==0.35.1 +dill==0.3.8 +distro==1.9.0 +dlimp==0.0.1 +dm-tree==0.1.9 +docutils==0.22.1 +draccus==0.10.0 +editdistance==0.8.1 +einops==0.8.1 +einops-exts==0.0.4 +et_xmlfile==2.0.0 +etils==1.13.0 +evdev==1.9.2 +exceptiongroup==1.3.0 +face==24.0.0 +fastapi==0.116.2 +ffmpy==0.6.1 +fiddle==0.3.0 +filelock==3.13.1 +Flask==3.1.2 +fonttools==4.60.0 +frozenlist==1.7.0 +fsspec==2023.9.2 +ftfy==6.3.1 +gcsfs==2023.9.2 +gitdb==4.0.12 +GitPython==3.1.45 +glom==24.11.0 +google-api-core==2.25.1 +google-auth==2.40.3 +google-auth-oauthlib==1.2.2 +google-cloud-core==2.4.3 +google-cloud-storage==2.19.0 +google-crc32c==1.7.1 +google-resumable-media==2.7.2 +googleapis-common-protos==1.70.0 +gradio==5.46.0 +gradio_client==1.13.0 +graphviz==0.21 +groovy==0.1.2 +grpcio==1.75.0 +gymnasium==0.29.1 +h11==0.16.0 +hf_transfer==0.1.9 +hf-xet==1.1.10 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==0.35.0 +id==1.5.0 +idna==3.10 +imageio==2.37.0 +imageio-ffmpeg==0.6.0 +importlib_metadata==8.7.0 +importlib_resources==6.5.2 +iniconfig==2.1.0 +inquirerpy==0.3.4 +isort==5.12.0 +itsdangerous==2.2.0 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.3.0 +jeepney==0.9.0 +Jinja2==3.1.4 +jiter==0.11.0 +jmespath==1.0.1 +joblib==1.5.2 +jsonlines==4.0.0 +keras==2.15.0 +keyring==25.6.0 +kiwisolver==1.4.9 +latex2sympy2_extended==1.10.2 +lerobot==0.3.4 +Levenshtein==0.27.1 +libcst==1.8.4 +lightning-utilities==0.15.2 +markdown-it-py==4.0.0 +math-verify==0.8.0 +matplotlib==3.10.6 +mdurl==0.1.2 +mergedeep==1.3.4 +ml-dtypes==0.2.0 +ml_dtypes==0.5.3 +more-itertools==10.8.0 +mpmath==1.3.0 +msgspec==0.19.0 +multidict==6.6.4 +multiprocess==0.70.16 +mypy==1.3.0 +mypy_extensions==1.1.0 +necessary==0.4.3 +networkx==3.3 +nh3==0.3.0 +nltk==3.9.1 +numpy==1.26.4 +oauthlib==3.3.1 +omegaconf==2.3.0 +openai==1.108.0 +opencv-python-headless==4.12.0.88 +OpenEXR==3.4.0 +openpyxl==3.1.5 +orderly-set==5.5.0 +orjson==3.11.3 +packaging==25.0 +pandas==2.3.2 +pathspec==0.12.1 +petname==2.6 +pfzy==0.3.4 +pillow==11.0.0 +pip==25.2 +platformdirs==4.4.0 +pluggy==1.6.0 +promise==2.3 +prompt_toolkit==3.0.52 +propcache==0.3.2 +proto-plus==1.26.1 +protobuf==4.21.12 +protobuf==6.32.1 +psutil==7.1.0 +pyarrow==21.0.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.2 +pycparser==2.23 +pydantic==2.11.9 +pydantic_core==2.33.2 +pydub==0.25.1 +Pygments==2.19.2 +pynput==1.8.1 +pyparsing==3.2.4 +pyproject_hooks==1.2.0 +pyserial==3.5 +pytest==8.4.2 +pytest-sphinx==0.6.3 +python-dateutil==2.9.0.post0 +python-Levenshtein==0.27.1 +python-multipart==0.0.20 +python-xlib==0.33 +pytorch-triton-rocm==3.4.0 +pytz==2025.2 +pyyaml-include==1.4.1 +RapidFuzz==3.14.1 +readme_renderer==44.0 +regex==2025.9.1 +requests==2.32.5 +requests-oauthlib==2.0.0 +requests-toolbelt==1.0.0 +requirements-parser==0.13.0 +rerun-sdk==0.22.1 +rfc3986==2.0.0 +rich==13.9.4 +rsa==4.9.1 +ruff==0.13.0 +s3transfer==0.14.0 +safehttpx==0.1.6 +safetensors==0.6.2 +scikit-learn==1.7.2 +scipy==1.15.3 +SecretStorage==3.4.0 +semantic-version==2.10.0 +sentencepiece==0.2.1 +sentry-sdk==2.38.0 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.17.0 +smart_open==7.3.1 +smashed==0.21.5 +smmap==5.0.2 +sniffio==1.3.1 +starlette==0.48.0 +sympy==1.13.3 +tensorboard==2.15.2 +tensorboard==2.19.0 +tensorflow==2.15.0 +tensorflow-addons==0.23.0 +tensorflow-datasets==4.9.3 +tensorflow-estimator==2.15.0 +tensorflow-graphics==2021.12.3 +tensorflow-metadata==1.17.2 +threadpoolctl==3.6.0 +timm==1.0.19 +tokenizers==0.22.0 +toml==0.10.2 +tomli==2.2.1 +tomlkit==0.13.3 +torch==2.8.0+rocm6.4 +torchcodec==0.5 +torchmetrics==1.8.2 +torchvision==0.23.0+rocm6.4 +tqdm==4.67.1 +transformers==4.56.1 +trimesh==4.8.2 +trouting==0.3.3 +twine==6.2.0 +typeguard==2.13.3 +typer==0.17.4 +typing_extensions==4.15.0 +typing-inspect==0.9.0 +typing-inspection==0.4.1 +tzdata==2025.2 +urllib3==2.5.0 +uvicorn==0.35.0 +wandb==0.21.4 +wcwidth==0.2.13 +websockets==15.0.1 +wheel==0.45.1 +wrapt==1.14.2 +xxhash==3.5.0 +yarl==1.20.1 +zipp==3.23.0 +lerobot==0.3.4 +minLoRA==0.1.0 +autocommand==2.2.2 +backports.tarfile==1.2.0 +importlib_metadata==8.0.0 +inflect==7.3.1 +jaraco.collections==5.1.0 +jaraco.context==5.3.0 +jaraco.functools==4.0.1 +jaraco.text==3.12.1 +more-itertools==10.3.0 +packaging==24.2 +platformdirs==4.2.2 +tomli==2.0.1 +typeguard==4.3.0 +typing_extensions==4.12.2 +wheel==0.45.1 +zipp==3.19.2 diff --git a/all_l1/wandb/wandb/run-20250930_131250-ea1k0g3y/files/wandb-metadata.json b/all_l1/wandb/wandb/run-20250930_131250-ea1k0g3y/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1bf5af562d1ceeed09c0950a4120295c46a7bf44 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_131250-ea1k0g3y/files/wandb-metadata.json @@ -0,0 +1,202 @@ +{ + "os": "Linux-5.15.0-140-generic-x86_64-with-glibc2.35", + "python": "CPython 3.10.18", + "startedAt": "2025-09-30T13:12:50.512180Z", + "args": [ + "qwen2_7b", + "save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1", + "--vision_backbone", + "openai", + "--action_head", + "l1_regression", + "--seq_len", + "1600", + "--ft_llm", + "--checkpoint", + "/vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924", + "--device_train_microbatch_size", + "16", + "--global_batch_size", + "126", + "--dataset", + "vla_dataset_realworld", + "--llm_learning_rate", + "5e-5", + "--wandb_entity", + "henryeap", + "--wandb_project", + "a1-realworld", + "--wandb_run_name", + "realworld", + "--save_overwrite" + ], + "program": "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py", + "codePath": "launch_scripts/train_vla.py", + "codePathLocal": "launch_scripts/train_vla.py", + "git": { + "remote": "https://github.com/Spatialtemporal-AI/A1.git", + "commit": "f2afcc15e05f491a8e50add64395fc1db0a1188d" + }, + "email": "ihenrykwok@outlook.com", + "root": "/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb", + "host": "auh7-1b-gpu-266", + "executable": "/vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10", + "cpu_count": 64, + "cpu_count_logical": 128, + "gpu": "Instinct MI210", + "gpu_count": 8, + "disk": { + "/": { + "total": "470343073792", + "used": "50365865984" + } + }, + "memory": { + "total": "2434606915584" + }, + "gpu_amd": [ + { + "id": "5", + "uniqueId": "0x55366988951d07ad", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "6", + "uniqueId": "0x38b61e086b79539", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "7", + "uniqueId": "0xec87d8b9ba4d4aa9", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "2", + "uniqueId": "0x433fd00772eb3fb7", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "3", + "uniqueId": "0xdaf8107e479aecf0", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "1", + "uniqueId": "0x1633d5497b099b7f", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "4", + "uniqueId": "0xe7ce40a1ea1cbac2", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + }, + { + "id": "0", + "uniqueId": "0xef820a5c24b0c740", + "vbiosVersion": "113-D67301V-073", + "performanceLevel": "auto", + "maxPower": "300.0", + "series": "Instinct MI210", + "model": "0x740f", + "vendor": "Advanced Micro Devices, Inc. [AMD/ATI]", + "sku": "D67301V", + "sclkRange": "500Mhz - 1700Mhz", + "mclkRange": "400Mhz - 1600Mhz" + } + ], + "slurm": { + "cluster_name": "ai-04r", + "conf": "/etc/slurm/slurm.conf", + "cpus_on_node": "128", + "gpus_on_node": "8", + "gtids": "0", + "job_account": "faculty-acc", + "job_cpus_per_node": "128", + "job_end_time": "1759497126", + "job_gid": "2000", + "job_gpus": "0,1,2,3,4,5,6,7", + "job_id": "2020", + "job_name": "mh_realworld", + "job_nodelist": "auh7-1b-gpu-266", + "job_num_nodes": "1", + "job_partition": "faculty", + "job_qos": "xdqos", + "job_start_time": "1759237926", + "job_uid": "2013", + "job_user": "xiaodan", + "jobid": "2020", + "localid": "0", + "nnodes": "1", + "nodeid": "0", + "nodelist": "auh7-1b-gpu-266", + "nprocs": "1", + "ntasks": "1", + "ntasks_per_node": "1", + "oom_kill_step": "0", + "prio_process": "0", + "procid": "0", + "submit_dir": "/vast/users/xiaodan/zhangjian/A1/launch_scripts", + "submit_host": "auh-1b-cpu-login-001", + "task_pid": "3355381", + "tasks_per_node": "1", + "topology_addr": "auh7-1b-gpu-266", + "topology_addr_pattern": "node" + }, + "writerId": "br57kuzpd4cnjl4v1nt32cpk941kjn1w" +} \ No newline at end of file diff --git a/all_l1/wandb/wandb/run-20250930_131250-ea1k0g3y/logs/debug-core.log b/all_l1/wandb/wandb/run-20250930_131250-ea1k0g3y/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..578a394a3ea8135822fdea1f1a024e6b4cbd66c4 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_131250-ea1k0g3y/logs/debug-core.log @@ -0,0 +1,6 @@ +{"time":"2025-09-30T13:12:50.569260504Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp7ktuvs87/port-3355558.txt","pid":3355558,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false} +{"time":"2025-09-30T13:12:50.569851234Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3355558} +{"time":"2025-09-30T13:12:50.569857124Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3355558-3355728-1383521146/socket","Net":"unix"}} +{"time":"2025-09-30T13:12:50.74547283Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"} +{"time":"2025-09-30T13:12:50.752005291Z","level":"INFO","msg":"handleInformInit: received","streamId":"ea1k0g3y","id":"1(@)"} +{"time":"2025-09-30T13:12:51.778015112Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"ea1k0g3y","id":"1(@)"} diff --git a/all_l1/wandb/wandb/run-20250930_131250-ea1k0g3y/logs/debug-internal.log b/all_l1/wandb/wandb/run-20250930_131250-ea1k0g3y/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f830e5e77134044528636ac1e7a7cedc908690b4 --- /dev/null +++ b/all_l1/wandb/wandb/run-20250930_131250-ea1k0g3y/logs/debug-internal.log @@ -0,0 +1,9 @@ +{"time":"2025-09-30T13:12:50.753807221Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"} +{"time":"2025-09-30T13:12:51.777951571Z","level":"INFO","msg":"stream: created new stream","id":"ea1k0g3y"} +{"time":"2025-09-30T13:12:51.778008702Z","level":"INFO","msg":"stream: started","id":"ea1k0g3y"} +{"time":"2025-09-30T13:12:51.778019413Z","level":"INFO","msg":"handler: started","stream_id":"ea1k0g3y"} +{"time":"2025-09-30T13:12:51.778022263Z","level":"INFO","msg":"sender: started","stream_id":"ea1k0g3y"} +{"time":"2025-09-30T13:12:51.778055373Z","level":"INFO","msg":"writer: started","stream_id":"ea1k0g3y"} +{"time":"2025-10-01T17:48:53.058127813Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/henryeap/a1-realworld/ea1k0g3y/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2025-10-02T06:50:26.681725611Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/henryeap/a1-realworld/ea1k0g3y/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2025-10-02T07:20:37.728905409Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"} diff --git a/all_l1/wandb/wandb/run-20250930_131250-ea1k0g3y/logs/debug.log b/all_l1/wandb/wandb/run-20250930_131250-ea1k0g3y/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cleandesk50_flow_matching/wandb/wandb/run-20251008_163834-quokv8gn/run-quokv8gn.wandb b/cleandesk50_flow_matching/wandb/wandb/run-20251008_163834-quokv8gn/run-quokv8gn.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ab7d3edeb78080f304eb9e5feae5958bb8bf7270 --- /dev/null +++ b/cleandesk50_flow_matching/wandb/wandb/run-20251008_163834-quokv8gn/run-quokv8gn.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a745d4f17f11679cdf5bf92aec764de48409f778dd66d1a9d82556acc272c36 +size 212789 diff --git a/cleandesk50_l1_regression/wandb/wandb/run-20251008_163831-fqdwkc8m/run-fqdwkc8m.wandb b/cleandesk50_l1_regression/wandb/wandb/run-20251008_163831-fqdwkc8m/run-fqdwkc8m.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5715f3ef97ddf3f48b4946ae665b0ca8a67383b9 --- /dev/null +++ b/cleandesk50_l1_regression/wandb/wandb/run-20251008_163831-fqdwkc8m/run-fqdwkc8m.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcd41307edd91290339717d666254857fafd5f8d6cb82e9ad0e0ea1e6c93bfb5 +size 206094 diff --git a/cleandesk_flow_matching/step12000-action-head/metadata.pt b/cleandesk_flow_matching/step12000-action-head/metadata.pt new file mode 100644 index 0000000000000000000000000000000000000000..752c8b547c4cac7f78746ac5066032167b1cf3c1 --- /dev/null +++ b/cleandesk_flow_matching/step12000-action-head/metadata.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:995307502120af3866f237cd0bc484fc848a652539d28e53cbea882abc16ba6b +size 1331 diff --git a/cleandesk_flow_matching/step12000-unsharded/lora.pt b/cleandesk_flow_matching/step12000-unsharded/lora.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3eb8fc48e155d8229340760896a20ba9fc413e2 --- /dev/null +++ b/cleandesk_flow_matching/step12000-unsharded/lora.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b09055f15a54dd092b4dd30833406731057005822da0c55c16231cf2e68f7f6 +size 1243 diff --git a/cleandesk_flow_matching/step12000-unsharded/train.pt b/cleandesk_flow_matching/step12000-unsharded/train.pt new file mode 100644 index 0000000000000000000000000000000000000000..067eeb392ccd2fc0c39b2b3aad15ff7da8cc27e7 --- /dev/null +++ b/cleandesk_flow_matching/step12000-unsharded/train.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:878ba9d72f994a81ed1a71b220badd89c2e842c5be4ff48e26d49c71f262847b +size 15061 diff --git a/glue_flow_matching/wandb/wandb/run-20251002_163728-tmwli25x/run-tmwli25x.wandb b/glue_flow_matching/wandb/wandb/run-20251002_163728-tmwli25x/run-tmwli25x.wandb new file mode 100644 index 0000000000000000000000000000000000000000..11aee9be87edb54d1f40db5064f016098eb1c351 --- /dev/null +++ b/glue_flow_matching/wandb/wandb/run-20251002_163728-tmwli25x/run-tmwli25x.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de648494435cefb93fe9231d01df14e2e77c0a85d968f930b4d3fba3df52ff32 +size 740786176