Add files using upload-large-folder tool

.gitattributes

@@ -39,3 +39,6 @@ wandb/wandb/run-20251002_155441-70dhy5dq/run-70dhy5dq.wandb filter=lfs diff=lfs
 wandb/wandb/run-20251002_150921-kqbx0cjv/run-kqbx0cjv.wandb filter=lfs diff=lfs merge=lfs -text
 cleandesk_l1_regression/wandb/wandb/run-20251008_163831-76mxu43t/run-76mxu43t.wandb filter=lfs diff=lfs merge=lfs -text
 wipe_l1_regression/wandb/wandb/run-20251005_163743-a1znetn8/run-a1znetn8.wandb filter=lfs diff=lfs merge=lfs -text
+cleandesk50_flow_matching/wandb/wandb/run-20251008_163834-quokv8gn/run-quokv8gn.wandb filter=lfs diff=lfs merge=lfs -text
+cleandesk50_l1_regression/wandb/wandb/run-20251008_163831-fqdwkc8m/run-fqdwkc8m.wandb filter=lfs diff=lfs merge=lfs -text
+glue_flow_matching/wandb/wandb/run-20251002_163728-tmwli25x/run-tmwli25x.wandb filter=lfs diff=lfs merge=lfs -text

all_flow_matching/glue_best/wandb/wandb/run-20250924_061930-dnrnwv30/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":0},"_runtime":0}

all_flow_matching/glue_best/wandb/wandb/run-20250924_061930-dnrnwv30/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2025-09-24T06:19:30.419454454Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmph06ly_es/port-2188820.txt","pid":2188820,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-24T06:19:30.420467727Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2188820}
+{"time":"2025-09-24T06:19:30.42153553Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2188820-2188995-771080915/socket","Net":"unix"}}
+{"time":"2025-09-24T06:19:30.607757412Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-24T06:19:30.624290644Z","level":"INFO","msg":"handleInformInit: received","streamId":"dnrnwv30","id":"1(@)"}
+{"time":"2025-09-24T06:19:31.778152452Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"dnrnwv30","id":"1(@)"}
+{"time":"2025-09-24T06:19:32.042400183Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-09-24T06:19:32.042449263Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-24T06:19:32.042441673Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-09-24T06:19:32.042538224Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2188820-2188995-771080915/socket","Net":"unix"}}
+{"time":"2025-09-24T06:19:32.042560085Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-09-24T06:19:32.340314533Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-09-24T06:19:32.340328313Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-09-24T06:19:32.340337613Z","level":"INFO","msg":"server is closed"}

all_flow_matching/glue_best/wandb/wandb/run-20250924_061930-dnrnwv30/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2025-09-24T06:19:30.626115817Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"}
+{"time":"2025-09-24T06:19:31.778111872Z","level":"INFO","msg":"stream: created new stream","id":"dnrnwv30"}
+{"time":"2025-09-24T06:19:31.778147652Z","level":"INFO","msg":"stream: started","id":"dnrnwv30"}
+{"time":"2025-09-24T06:19:31.778168212Z","level":"INFO","msg":"writer: started","stream_id":"dnrnwv30"}
+{"time":"2025-09-24T06:19:31.778194323Z","level":"INFO","msg":"sender: started","stream_id":"dnrnwv30"}
+{"time":"2025-09-24T06:19:31.778192543Z","level":"INFO","msg":"handler: started","stream_id":"dnrnwv30"}
+{"time":"2025-09-24T06:19:32.023079005Z","level":"ERROR","msg":"HTTP error","status":403,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2025-09-24T06:19:32.023158306Z","level":"ERROR","msg":"runupserter: failed to init run","error":"returned error 403: {\"data\":{\"upsertBucket\":null},\"errors\":[{\"message\":\"permission denied\",\"path\":[\"upsertBucket\"],\"extensions\":{\"code\":\"PERMISSION_ERROR\"}}]}"}
+{"time":"2025-09-24T06:19:32.042439853Z","level":"INFO","msg":"stream: closing","id":"dnrnwv30"}
+{"time":"2025-09-24T06:19:32.048537861Z","level":"ERROR","msg":"sender: uploadConfigFile: stream: no run"}
+{"time":"2025-09-24T06:19:32.332555964Z","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2025-09-24T06:19:32.332627655Z","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404: {\"data\":{\"createRunFiles\":null},\"errors\":[{\"message\":\"run a1-vla-xiaodan/dnrnwv30 not found during createRunFiles\",\"path\":[\"createRunFiles\"]}]}"}
+{"time":"2025-09-24T06:19:32.336908289Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-09-24T06:19:32.33694275Z","level":"INFO","msg":"handler: closed","stream_id":"dnrnwv30"}
+{"time":"2025-09-24T06:19:32.338990606Z","level":"INFO","msg":"sender: closed","stream_id":"dnrnwv30"}
+{"time":"2025-09-24T06:19:32.338998046Z","level":"INFO","msg":"stream: closed","id":"dnrnwv30"}

all_flow_matching/glue_best/wandb/wandb/run-20250924_062357-hmmpns57/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":0},"_runtime":0}

all_flow_matching/glue_best/wandb/wandb/run-20250924_062357-hmmpns57/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2025-09-24T06:23:57.33371267Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp7mdgm6nn/port-2190130.txt","pid":2190130,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-24T06:23:57.334085295Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2190130}
+{"time":"2025-09-24T06:23:57.334062305Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2190130-2190294-2704513851/socket","Net":"unix"}}
+{"time":"2025-09-24T06:23:57.511386371Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-24T06:23:57.517858166Z","level":"INFO","msg":"handleInformInit: received","streamId":"hmmpns57","id":"1(@)"}
+{"time":"2025-09-24T06:23:58.547393843Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"hmmpns57","id":"1(@)"}
+{"time":"2025-09-24T06:23:58.805591645Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-09-24T06:23:58.805637046Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-09-24T06:23:58.805655116Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-24T06:23:58.805681387Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-09-24T06:23:58.805837029Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2190130-2190294-2704513851/socket","Net":"unix"}}
+{"time":"2025-09-24T06:23:59.098817455Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-09-24T06:23:59.098836546Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-09-24T06:23:59.098846216Z","level":"INFO","msg":"server is closed"}

all_flow_matching/glue_best/wandb/wandb/run-20250924_062357-hmmpns57/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2025-09-24T06:23:57.51970738Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"}
+{"time":"2025-09-24T06:23:58.547344712Z","level":"INFO","msg":"stream: created new stream","id":"hmmpns57"}
+{"time":"2025-09-24T06:23:58.547388123Z","level":"INFO","msg":"stream: started","id":"hmmpns57"}
+{"time":"2025-09-24T06:23:58.547412053Z","level":"INFO","msg":"writer: started","stream_id":"hmmpns57"}
+{"time":"2025-09-24T06:23:58.547423083Z","level":"INFO","msg":"handler: started","stream_id":"hmmpns57"}
+{"time":"2025-09-24T06:23:58.547448154Z","level":"INFO","msg":"sender: started","stream_id":"hmmpns57"}
+{"time":"2025-09-24T06:23:58.797771793Z","level":"ERROR","msg":"HTTP error","status":403,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2025-09-24T06:23:58.797893015Z","level":"ERROR","msg":"runupserter: failed to init run","error":"returned error 403: {\"data\":{\"upsertBucket\":null},\"errors\":[{\"message\":\"permission denied\",\"path\":[\"upsertBucket\"],\"extensions\":{\"code\":\"PERMISSION_ERROR\"}}]}"}
+{"time":"2025-09-24T06:23:58.805654996Z","level":"INFO","msg":"stream: closing","id":"hmmpns57"}
+{"time":"2025-09-24T06:23:58.809349995Z","level":"ERROR","msg":"sender: uploadConfigFile: stream: no run"}
+{"time":"2025-09-24T06:23:59.095800216Z","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2025-09-24T06:23:59.095886847Z","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404: {\"data\":{\"createRunFiles\":null},\"errors\":[{\"message\":\"run a1-vla-xiaodan/hmmpns57 not found during createRunFiles\",\"path\":[\"createRunFiles\"]}]}"}
+{"time":"2025-09-24T06:23:59.096209701Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-09-24T06:23:59.096244302Z","level":"INFO","msg":"handler: closed","stream_id":"hmmpns57"}
+{"time":"2025-09-24T06:23:59.097755731Z","level":"INFO","msg":"sender: closed","stream_id":"hmmpns57"}
+{"time":"2025-09-24T06:23:59.097762801Z","level":"INFO","msg":"stream: closed","id":"hmmpns57"}

all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/files/requirements.txt

@@ -0,0 +1,283 @@
+ai2-molmo==0.0.0
+astunparse==1.6.3
+flatbuffers==25.2.10
+gast==0.6.0
+google-pasta==0.2.0
+h5py==3.14.0
+libclang==18.1.1
+Markdown==3.9
+namex==0.1.0
+opt_einsum==3.4.0
+optree==0.17.0
+tensorboard-data-server==0.7.2
+tensorflow-io-gcs-filesystem==0.37.1
+tensorflow-rocm==2.16.2
+termcolor==3.1.0
+Werkzeug==3.1.3
+Brotli==1.1.0
+Farama-Notifications==0.0.4
+MarkupSafe==2.1.5
+PyYAML==6.0.2
+absl-py==2.3.1
+accelerate==1.10.1
+ai2-molmo==0.0.0
+aiofiles==24.1.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.12.15
+aiosignal==1.4.0
+annotated-types==0.7.0
+antlr4-python3-runtime==4.9.3
+anyio==4.10.0
+array_record==0.8.1
+async-timeout==5.0.1
+attrs==25.3.0
+av==15.1.0
+backports.tarfile==1.2.0
+beaker-gantry==3.2.0
+beaker-py==2.5.0
+black==23.12.1
+boltons==25.0.0
+boto3==1.40.33
+botocore==1.40.33
+build==1.3.0
+cached_path==1.7.3
+cached-property==2.0.1
+cachetools==5.5.2
+certifi==2025.8.3
+cffi==2.0.0
+charset-normalizer==3.4.3
+click==8.2.1
+click-help-colors==0.9.4
+click-option-group==0.5.7
+cloudpickle==3.1.1
+cmake==4.1.0
+contourpy==1.3.2
+cryptography==46.0.1
+cycler==0.12.1
+dataclass-extensions==0.2.3
+datasets==3.6.0
+decorator==5.2.1
+deepdiff==8.6.1
+diffusers==0.35.1
+dill==0.3.8
+distro==1.9.0
+dlimp==0.0.1
+dm-tree==0.1.9
+docutils==0.22.1
+draccus==0.10.0
+editdistance==0.8.1
+einops==0.8.1
+einops-exts==0.0.4
+etils==1.13.0
+evdev==1.9.2
+exceptiongroup==1.3.0
+face==24.0.0
+fastapi==0.116.2
+ffmpy==0.6.1
+fiddle==0.3.0
+filelock==3.13.1
+fonttools==4.60.0
+frozenlist==1.7.0
+fsspec==2023.9.2
+ftfy==6.3.1
+gcsfs==2023.9.2
+gitdb==4.0.12
+GitPython==3.1.45
+glom==24.11.0
+google-api-core==2.25.1
+google-auth==2.40.3
+google-auth-oauthlib==1.2.2
+google-cloud-core==2.4.3
+google-cloud-storage==2.19.0
+google-crc32c==1.7.1
+google-resumable-media==2.7.2
+googleapis-common-protos==1.70.0
+gradio==5.46.0
+gradio_client==1.13.0
+graphviz==0.21
+groovy==0.1.2
+grpcio==1.75.0
+gymnasium==0.29.1
+h11==0.16.0
+hf_transfer==0.1.9
+hf-xet==1.1.10
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==0.35.0
+id==1.5.0
+idna==3.10
+imageio==2.37.0
+imageio-ffmpeg==0.6.0
+importlib_metadata==8.7.0
+importlib_resources==6.5.2
+iniconfig==2.1.0
+inquirerpy==0.3.4
+isort==5.12.0
+jaraco.classes==3.4.0
+jaraco.context==6.0.1
+jaraco.functools==4.3.0
+jeepney==0.9.0
+Jinja2==3.1.4
+jiter==0.11.0
+jmespath==1.0.1
+joblib==1.5.2
+jsonlines==4.0.0
+keras==2.15.0
+keyring==25.6.0
+kiwisolver==1.4.9
+latex2sympy2_extended==1.10.2
+lerobot==0.3.4
+Levenshtein==0.27.1
+libcst==1.8.4
+lightning-utilities==0.15.2
+markdown-it-py==4.0.0
+math-verify==0.8.0
+matplotlib==3.10.6
+mdurl==0.1.2
+mergedeep==1.3.4
+ml-dtypes==0.2.0
+ml_dtypes==0.5.3
+more-itertools==10.8.0
+mpmath==1.3.0
+msgspec==0.19.0
+multidict==6.6.4
+multiprocess==0.70.16
+mypy==1.3.0
+mypy_extensions==1.1.0
+necessary==0.4.3
+networkx==3.3
+nh3==0.3.0
+nltk==3.9.1
+numpy==1.26.4
+numpy==2.2.6
+oauthlib==3.3.1
+omegaconf==2.3.0
+openai==1.108.0
+opencv-python-headless==4.12.0.88
+OpenEXR==3.4.0
+orderly-set==5.5.0
+orjson==3.11.3
+packaging==25.0
+pandas==2.3.2
+pathspec==0.12.1
+petname==2.6
+pfzy==0.3.4
+pillow==11.0.0
+pip==25.2
+platformdirs==4.4.0
+pluggy==1.6.0
+promise==2.3
+prompt_toolkit==3.0.52
+propcache==0.3.2
+proto-plus==1.26.1
+protobuf==4.21.12
+protobuf==6.32.1
+psutil==7.1.0
+pyarrow==21.0.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.2
+pycparser==2.23
+pydantic==2.11.9
+pydantic_core==2.33.2
+pydub==0.25.1
+Pygments==2.19.2
+pynput==1.8.1
+pyparsing==3.2.4
+pyproject_hooks==1.2.0
+pyserial==3.5
+pytest==8.4.2
+pytest-sphinx==0.6.3
+python-dateutil==2.9.0.post0
+python-Levenshtein==0.27.1
+python-multipart==0.0.20
+python-xlib==0.33
+pytorch-triton-rocm==3.4.0
+pytz==2025.2
+pyyaml-include==1.4.1
+RapidFuzz==3.14.1
+readme_renderer==44.0
+regex==2025.9.1
+requests==2.32.5
+requests-oauthlib==2.0.0
+requests-toolbelt==1.0.0
+requirements-parser==0.13.0
+rerun-sdk==0.22.1
+rfc3986==2.0.0
+rich==13.9.4
+rsa==4.9.1
+ruff==0.13.0
+s3transfer==0.14.0
+safehttpx==0.1.6
+safetensors==0.6.2
+scikit-learn==1.7.2
+scipy==1.15.3
+SecretStorage==3.4.0
+semantic-version==2.10.0
+sentencepiece==0.2.1
+sentry-sdk==2.38.0
+setuptools==78.1.1
+shellingham==1.5.4
+six==1.17.0
+smart_open==7.3.1
+smashed==0.21.5
+smmap==5.0.2
+sniffio==1.3.1
+starlette==0.48.0
+sympy==1.13.3
+tensorboard==2.15.2
+tensorboard==2.19.0
+tensorflow==2.15.0
+tensorflow-addons==0.23.0
+tensorflow-datasets==4.9.3
+tensorflow-estimator==2.15.0
+tensorflow-graphics==2021.12.3
+tensorflow-metadata==1.17.2
+threadpoolctl==3.6.0
+timm==1.0.19
+tokenizers==0.22.0
+toml==0.10.2
+tomli==2.2.1
+tomlkit==0.13.3
+torch==2.8.0+rocm6.4
+torchcodec==0.5
+torchmetrics==1.8.2
+torchvision==0.23.0+rocm6.4
+tqdm==4.67.1
+transformers==4.56.1
+trimesh==4.8.2
+trouting==0.3.3
+twine==6.2.0
+typeguard==2.13.3
+typer==0.17.4
+typing_extensions==4.15.0
+typing-inspect==0.9.0
+typing-inspection==0.4.1
+tzdata==2025.2
+urllib3==2.5.0
+uvicorn==0.35.0
+wandb==0.21.4
+wcwidth==0.2.13
+websockets==15.0.1
+wheel==0.45.1
+wrapt==1.14.2
+xxhash==3.5.0
+yarl==1.20.1
+zipp==3.23.0
+lerobot==0.3.4
+minLoRA==0.1.0
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.2
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.45.1
+zipp==3.19.2

all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/files/wandb-metadata.json

@@ -0,0 +1,204 @@
+{
+  "os":  "Linux-5.15.0-140-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.18",
+  "startedAt":  "2025-09-24T06:31:28.005264Z",
+  "args":  [
+    "qwen2_7b",
+    "save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt",
+    "--vision_backbone",
+    "openai",
+    "--action_head",
+    "flow_matching",
+    "--seq_len",
+    "768",
+    "--lora_rank",
+    "32",
+    "--lora_llm",
+    "--checkpoint",
+    "/vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924",
+    "--device_train_microbatch_size",
+    "22",
+    "--global_batch_size",
+    "176",
+    "--dataset",
+    "vla_dataset_realworld",
+    "--llm_learning_rate",
+    "5e-5",
+    "--wandb_entity",
+    "henryeap",
+    "--wandb_project",
+    "a1-realworld",
+    "--wandb_run_name",
+    "glue",
+    "--save_overwrite"
+  ],
+  "program":  "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py",
+  "codePath":  "launch_scripts/train_vla.py",
+  "codePathLocal":  "launch_scripts/train_vla.py",
+  "git":  {
+    "remote":  "https://github.com/Spatialtemporal-AI/A1.git",
+    "commit":  "c13f2763af61e0d729a8b5ab4bdefc512205bcc5"
+  },
+  "email":  "ihenrykwok@outlook.com",
+  "root":  "/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/wandb",
+  "host":  "auh7-1b-gpu-188",
+  "executable":  "/vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10",
+  "cpu_count":  64,
+  "cpu_count_logical":  128,
+  "gpu":  "Instinct MI210",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "470343073792",
+      "used":  "51147874304"
+    }
+  },
+  "memory":  {
+    "total":  "2434606952448"
+  },
+  "gpu_amd":  [
+    {
+      "id":  "5",
+      "uniqueId":  "0x137c9ede1bb1518e",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "7",
+      "uniqueId":  "0x21a2e88d06c419dc",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "2",
+      "uniqueId":  "0x399226d2b2bfa544",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "0",
+      "uniqueId":  "0x3558c3014c813fdb",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "3",
+      "uniqueId":  "0xf61ec17df11883bd",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "1",
+      "uniqueId":  "0x9b5c1c302c8129f8",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "6",
+      "uniqueId":  "0xfa8b85a4625b04f",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "4",
+      "uniqueId":  "0xa515afd8ced1d39d",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "ai-04r",
+    "conf":  "/etc/slurm/slurm.conf",
+    "cpus_on_node":  "128",
+    "gpus_on_node":  "8",
+    "gtids":  "0",
+    "job_account":  "faculty-acc",
+    "job_cpus_per_node":  "128",
+    "job_end_time":  "1758954648",
+    "job_gid":  "2000",
+    "job_gpus":  "0,1,2,3,4,5,6,7",
+    "job_id":  "1605",
+    "job_name":  "realworld_mh",
+    "job_nodelist":  "auh7-1b-gpu-188",
+    "job_num_nodes":  "1",
+    "job_partition":  "faculty",
+    "job_qos":  "xdqos",
+    "job_start_time":  "1758695448",
+    "job_uid":  "2013",
+    "job_user":  "xiaodan",
+    "jobid":  "1605",
+    "localid":  "0",
+    "nnodes":  "1",
+    "nodeid":  "0",
+    "nodelist":  "auh7-1b-gpu-188",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "oom_kill_step":  "0",
+    "prio_process":  "0",
+    "procid":  "0",
+    "submit_dir":  "/vast/users/xiaodan/zhangjian/A1/launch_scripts",
+    "submit_host":  "auh-1b-cpu-login-001",
+    "task_pid":  "2191329",
+    "tasks_per_node":  "1",
+    "topology_addr":  "auh7-1b-gpu-188",
+    "topology_addr_pattern":  "node"
+  },
+  "writerId":  "o421nvn5u6ub6ruog26gg83x0g2lmgbt"
+}

all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_runtime":2,"_wandb":{"runtime":2}}

all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2025-09-24T06:31:28.064738272Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpt5x3_6pq/port-2191415.txt","pid":2191415,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-24T06:31:28.066318992Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2191415}
+{"time":"2025-09-24T06:31:28.066291612Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2191415-2191580-2229050380/socket","Net":"unix"}}
+{"time":"2025-09-24T06:31:28.241348786Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-24T06:31:28.248471068Z","level":"INFO","msg":"handleInformInit: received","streamId":"wtatxotn","id":"1(@)"}
+{"time":"2025-09-24T06:31:29.271764603Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"wtatxotn","id":"1(@)"}
+{"time":"2025-09-24T06:31:32.368379213Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-09-24T06:31:32.368730117Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-24T06:31:32.368724477Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-09-24T06:31:32.368772508Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-09-24T06:31:32.368826649Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2191415-2191580-2229050380/socket","Net":"unix"}}
+{"time":"2025-09-24T06:31:33.781258776Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-09-24T06:31:33.781274636Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-09-24T06:31:33.781290036Z","level":"INFO","msg":"server is closed"}

all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2025-09-24T06:31:28.251016151Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"}
+{"time":"2025-09-24T06:31:29.271706933Z","level":"INFO","msg":"stream: created new stream","id":"wtatxotn"}
+{"time":"2025-09-24T06:31:29.271758583Z","level":"INFO","msg":"stream: started","id":"wtatxotn"}
+{"time":"2025-09-24T06:31:29.271781634Z","level":"INFO","msg":"handler: started","stream_id":"wtatxotn"}
+{"time":"2025-09-24T06:31:29.271778354Z","level":"INFO","msg":"writer: started","stream_id":"wtatxotn"}
+{"time":"2025-09-24T06:31:29.271793104Z","level":"INFO","msg":"sender: started","stream_id":"wtatxotn"}
+{"time":"2025-09-24T06:31:32.368726307Z","level":"INFO","msg":"stream: closing","id":"wtatxotn"}
+{"time":"2025-09-24T06:31:33.421915218Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-09-24T06:31:33.778168056Z","level":"INFO","msg":"handler: closed","stream_id":"wtatxotn"}
+{"time":"2025-09-24T06:31:33.780128001Z","level":"INFO","msg":"sender: closed","stream_id":"wtatxotn"}
+{"time":"2025-09-24T06:31:33.780158132Z","level":"INFO","msg":"stream: closed","id":"wtatxotn"}

all_flow_matching/glue_best/wandb/wandb/run-20250924_063128-wtatxotn/logs/debug.log

@@ -0,0 +1 @@
+2025-09-24 06:31:32,368 INFO    wandb-AsyncioManager-main:2191415 [service_client.py:_forward_responses():84] Reached EOF.

all_flow_matching/glue_best/wandb/wandb/run-20250924_063642-6tj2c8pr/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2025-09-24T06:36:42.858452895Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmphjacd0ae/port-2192743.txt","pid":2192743,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-24T06:36:42.859464168Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":2192743}
+{"time":"2025-09-24T06:36:42.859402097Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-2192743-2192908-1607882430/socket","Net":"unix"}}
+{"time":"2025-09-24T06:36:43.043042785Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-24T06:36:43.049816992Z","level":"INFO","msg":"handleInformInit: received","streamId":"6tj2c8pr","id":"1(@)"}
+{"time":"2025-09-24T06:36:44.180372147Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"6tj2c8pr","id":"1(@)"}
+{"time":"2025-09-24T06:36:47.096248789Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-09-24T06:36:47.096289969Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-09-24T06:36:47.09631223Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-24T06:36:47.09637208Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-09-24T06:36:47.096556563Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-2192743-2192908-1607882430/socket","Net":"unix"}}
+{"time":"2025-09-24T06:36:48.708711296Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-09-24T06:36:48.709060661Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-09-24T06:36:48.709072391Z","level":"INFO","msg":"server is closed"}

all_flow_matching/glue_best/wandb/wandb/run-20250924_063642-6tj2c8pr/logs/debug.log

@@ -0,0 +1 @@
+2025-09-24 06:36:47,096 INFO    wandb-AsyncioManager-main:2192743 [service_client.py:_forward_responses():84] Reached EOF.

all_l1/step8000-action-head/metadata.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:883bb1160e5c5ece44d882f29247664bba7a98ff980d816e4b7684797ae2d601
+size 1331

all_l1/step8500-action-head/metadata.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80287bb60307f9ac09aacfce4b1df8f509d0f9c0571939e3a38e6f558e2a874c
+size 1331

all_l1/step8500-unsharded/config.yaml

@@ -0,0 +1,322 @@
+run_name: realworld_20250930_131219
+seed: 6198
+epoch: null
+dry_run: false
+model:
+  d_model: 3584
+  n_heads: 28
+  n_kv_heads: 4
+  qkv_bias: true
+  clip_qkv: null
+  n_layers: 28
+  mlp_ratio: 4
+  mlp_hidden_size: 37888
+  activation_type: swiglu
+  block_type: sequential
+  block_group_size: 1
+  rope: true
+  rope_full_precision: true
+  rope_theta: 1000000.0
+  vision_backbone:
+    image_model_type: openai
+    image_default_input_size:
+    - 336
+    - 336
+    image_patch_size: 14
+    image_pos_patch_size: 14
+    image_emb_dim: 1024
+    image_num_heads: 16
+    image_num_key_value_heads: 16
+    image_num_layers: 23
+    image_head_dim: 64
+    image_mlp_dim: 4096
+    image_mlp_activations: quick_gelu
+    image_dropout_rate: 0.0
+    image_num_pos: 577
+    image_norm_eps: 1.0e-05
+    attention_dropout: 0.0
+    residual_dropout: 0.0
+    initializer_range: 0.02
+    fsdp_wrap: false
+    resize_mode: default
+  vit_load_path: /weka/oe-training-default/mm-olmo/pretrained_image_encoders/vit-l-14-336.pt
+  llm_load_path: /weka/oe-training-default/mm-olmo/pretrained_llms/qwen2-7b.pt
+  low_cpu_fsdp: true
+  attention_type: sdpa
+  float32_attention: true
+  attention_dropout: 0.0
+  attention_layer_norm: false
+  residual_dropout: 0.1
+  response_residual_dropout: 0.0
+  embedding_dropout: 0.0
+  layer_norm_type: rms
+  layer_norm_with_affine: true
+  layer_norm_eps: 1.0e-06
+  attention_layer_norm_with_affine: true
+  max_sequence_length: 4096
+  max_position_embeddings: null
+  include_bias: false
+  bias_for_layer_norm: null
+  scale_logits: false
+  vocab_size: 152064
+  embedding_size: 152064
+  ff_out_size: 0
+  additional_vocab_size: 128
+  new_embedding_init_range: 0.02
+  weight_tying: false
+  init_device: null
+  init_fn: normal
+  init_std: 0.02
+  init_cutoff_factor: null
+  norm_after: false
+  precision: amp_bf16
+  max_crops: 12
+  crop_mode: overlap-and-resize-c2
+  use_col_tokens: true
+  prompt_type: uber_model
+  system_prompt_kind: demo_or_style
+  message_formatting: role
+  always_start_with_space: true
+  multi_annotation_weighting: root_subsegments
+  default_inference_len: 65
+  overlap_margins:
+  - 4
+  - 4
+  pad_value: 0.0
+  image_padding_embed: pad_and_partial_pad
+  fix_image_padding: true
+  vit_layers:
+  - -2
+  - -9
+  image_pooling_h: 2
+  image_pooling_w: 2
+  image_pooling_2d: attention_meanq
+  image_projector: mlp
+  image_feature_dropout: 0.0
+  initializer_range: 0.02
+  normalize_input_embeds: false
+  use_position_ids: true
+  head_dim: null
+  action_tokenizer:
+    identifier: physical-intelligence/fast
+    tokenizer_dir: null
+  action_dim: 7
+  horizon: 8
+  tokenizer:
+    identifier: Qwen/Qwen2-7B
+    tokenizer_dir: null
+  pad_tokenizer: true
+  moe_num_experts: 8
+  moe_top_k: 2
+  moe_mlp_impl: sparse
+  moe_log_expert_assignment: false
+  moe_shared_expert: false
+  moe_lbl_in_fp32: false
+  moe_interleave: false
+  moe_loss_weight: 0.1
+  moe_zloss_weight: null
+  moe_dropless: true
+  moe_capacity_factor: 1.25
+  action_head: l1_regression
+  num_diffusion_steps: 1000
+  num_diffusion_inference_steps: 30
+  use_proprio: true
+  action_head_dit_hidden_size: 1152
+  action_head_dit_depth: 28
+  action_head_dit_num_heads: 16
+  llm_causal_attention: false
+  action_use_left_eef: true
+  action_use_mobile_base: false
+allow_resume: false
+ft_llm: true
+ft_vit: false
+ft_connector: false
+ft_embedding: lm_head
+lora: false
+use_lora: true
+lora_rank: 8
+lora_llm: false
+lora_vit: false
+lora_connector: false
+early_exit: false
+train_exit_random_layer: false
+optimizer:
+  name: adamw
+  learning_rate: 0.0001
+  weight_decay: 0.01
+  betas:
+  - 0.9
+  - 0.95
+  eps: 1.0e-05
+  connector_learning_rate: 0.0002
+  vit_learning_rate: 6.0e-06
+  llm_learning_rate: 5.0e-05
+  connector_weight_decay: 0.0
+  vit_weight_decay: 0.0
+  llm_weight_decay: 0.0
+  connector_betas:
+  - 0.9
+  - 0.95
+  vit_betas:
+  - 0.9
+  - 0.95
+  llm_betas:
+  - 0.9
+  - 0.95
+  connector_eps: 1.0e-06
+  vit_eps: 1.0e-06
+  llm_eps: 1.0e-06
+  metrics_log_interval: 20
+scheduler:
+  name: multimodal
+  units: steps
+  t_warmup: 100
+  t_max: null
+  alpha_f: 0.1
+  connector_t_warmup: 200
+  vit_t_warmup: 2000
+  llm_t_warmup: 2000
+  grad_clip_warmup_steps: null
+  grad_clip_warmup_factor: null
+  warmup_min_lr: 0.0
+data:
+  dataset: vla_dataset_realworld
+  mixture: null
+  root_size_mixture: null
+  split: train
+  seed: 95818
+  shuffle_messages: false
+  pad: to_max
+  sequence_length: 1600
+  shuffle: true
+  for_inference: false
+  multi_modal: torch
+  num_workers: 0
+  drop_last: true
+  pin_memory: true
+  prefetch_factor: null
+  persistent_workers: false
+  timeout: 0
+  rlds_dataset_name: libero_4_task_suites_no_noops
+  rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe
+  use_wrist_image: true
+  use_proprio: true
+  rlds_shuffle_buffer_size: 100000
+  rlds_traj_threads: 8
+  rlds_read_threads: 8
+  lerobot_episode_index_start: null
+  lerobot_episode_index_end: null
+restore_dataloader: true
+fast_forward_batches: null
+evaluators:
+- label: val
+  data:
+    dataset: vla_dataset_realworld
+    mixture: null
+    root_size_mixture: null
+    split: validation
+    seed: null
+    shuffle_messages: false
+    pad: to_max
+    sequence_length: 1600
+    shuffle: false
+    for_inference: false
+    multi_modal: torch
+    num_workers: 0
+    drop_last: true
+    pin_memory: true
+    prefetch_factor: null
+    persistent_workers: true
+    timeout: 0
+    rlds_dataset_name: libero_4_task_suites_no_noops
+    rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds
+    use_wrist_image: true
+    use_proprio: true
+    rlds_shuffle_buffer_size: 256000
+    rlds_traj_threads: 8
+    rlds_read_threads: 8
+    lerobot_episode_index_start: 353
+    lerobot_episode_index_end: 765
+  device_eval_batch_size: null
+  subset_num_batches: 64
+  max_examples: null
+  max_new_tokens: 448
+  mm_evaluator: null
+  save_dir: null
+  save_to_checkpoint_dir: false
+  eval_name: null
+  skip_if_metrics_cached: true
+eval_interval: 0
+inf_eval_interval: -1
+inf_evaluators: []
+save_folder: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1
+remote_save_folder: null
+canceled_check_interval: 50
+save_interval: 500
+save_interval_unsharded: 500
+save_interval_ephemeral: null
+save_interval_action_head: 500
+save_num_checkpoints_to_keep: 1
+save_num_unsharded_checkpoints_to_keep: 1
+save_num_action_head_checkpoints_to_keep: 2
+save_overwrite: true
+force_save_unsharded: false
+no_pre_train_checkpoint: true
+initial_model_checkpoint: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924
+load_model_config: null
+checkpoint_dir: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924
+load_path: null
+load_path_sharded_checkpointer: null
+reset_optimizer_state: false
+reset_trainer_state: false
+save_dataloader_state: false
+reset_dataloader_state: false
+keep_lr_on_load: true
+sharded_checkpointer: torch_legacy
+max_duration: 500000
+global_train_batch_size: 126
+device_train_batch_size: 15
+device_train_microbatch_size: 16
+device_eval_batch_size: 4
+eval_subset_num_batches: -1
+eval_on_load: false
+device_inf_eval_batch_size: 16
+inf_eval_subset_num_batches: -1
+device_train_grad_accum: 0
+max_grad_norm: 1.0
+multi_component_grad_norm: true
+batch_divisor: global_batch
+max_grad_norm_ratio: null
+precision: amp_bf16
+wandb:
+  project: a1-realworld
+  entity: henryeap
+  group: null
+  name: realworld_20250930_131219
+  tags:
+  - watching
+  log_artifacts: false
+  rank_zero_only: true
+  log_interval: 1
+speed_monitor:
+  window_size: 20
+  gpu_flops_available: null
+console_log_interval: 1
+gen1_gc_interval: 1
+compile: null
+fsdp:
+  use_orig_params: true
+  sharding_strategy: FULL_SHARD
+  wrapping_strategy: by_block_and_size
+  precision: float
+  hybrid_sharding_num_model_replicas: null
+softmax_auxiliary_loss: true
+softmax_auxiliary_loss_scale: 0.0001
+time_limit: null
+extra_steps_after_cancel: 10
+python_profiling: false
+torch_profiling: false
+stop_at: 500000
+stop_after: null
+activation_checkpointing: whole_layer
+fused_loss: null

all_l1/step8500-unsharded/lora.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b09055f15a54dd092b4dd30833406731057005822da0c55c16231cf2e68f7f6
+size 1243

all_l1/step8500-unsharded/train.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a978718ae923b1476f2fe06dd89422aa8d20b5aca459e47e40db6b02d009001
+size 15061

all_l1/wandb/wandb/debug-internal.log

@@ -0,0 +1,9 @@
+{"time":"2025-09-30T13:12:50.753807221Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"}
+{"time":"2025-09-30T13:12:51.777951571Z","level":"INFO","msg":"stream: created new stream","id":"ea1k0g3y"}
+{"time":"2025-09-30T13:12:51.778008702Z","level":"INFO","msg":"stream: started","id":"ea1k0g3y"}
+{"time":"2025-09-30T13:12:51.778019413Z","level":"INFO","msg":"handler: started","stream_id":"ea1k0g3y"}
+{"time":"2025-09-30T13:12:51.778022263Z","level":"INFO","msg":"sender: started","stream_id":"ea1k0g3y"}
+{"time":"2025-09-30T13:12:51.778055373Z","level":"INFO","msg":"writer: started","stream_id":"ea1k0g3y"}
+{"time":"2025-10-01T17:48:53.058127813Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/henryeap/a1-realworld/ea1k0g3y/file_stream","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
+{"time":"2025-10-02T06:50:26.681725611Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/henryeap/a1-realworld/ea1k0g3y/file_stream","body":"\n<html><head>\n<meta http-equiv=\"content-type\" content=\"text/html;charset=utf-8\">\n<title>502 Server Error</title>\n</head>\n<body text=#000000 bgcolor=#ffffff>\n<h1>Error: Server Error</h1>\n<h2>The server encountered a temporary error and could not complete your request.<p>Please try again in 30 seconds.</h2>\n<h2></h2>\n</body></html>\n"}
+{"time":"2025-10-02T07:20:37.728905409Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}

all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/config.yaml

@@ -0,0 +1,621 @@
+_wandb:
+    value:
+        cli_version: 0.21.4
+        e:
+            zpxgtt7ztfwstl3gysdjfozjodgqnvkm:
+                args:
+                    - qwen2_7b
+                    - save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1
+                    - --vision_backbone
+                    - openai
+                    - --action_head
+                    - l1_regression
+                    - --seq_len
+                    - "1600"
+                    - --ft_llm
+                    - --checkpoint
+                    - /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924
+                    - --device_train_microbatch_size
+                    - "16"
+                    - --global_batch_size
+                    - "126"
+                    - --dataset
+                    - vla_dataset_realworld
+                    - --llm_learning_rate
+                    - "5e-5"
+                    - --wandb_entity
+                    - henryeap
+                    - --wandb_project
+                    - a1-realworld
+                    - --wandb_run_name
+                    - realworld
+                    - --save_overwrite
+                codePath: launch_scripts/train_vla.py
+                codePathLocal: launch_scripts/train_vla.py
+                cpu_count: 64
+                cpu_count_logical: 128
+                disk:
+                    /:
+                        total: "470343073792"
+                        used: "50842279936"
+                email: ihenrykwok@outlook.com
+                executable: /vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10
+                git:
+                    commit: 50cf9fd3143e218eb94104381c16c0482ac52f0d
+                    remote: https://github.com/Spatialtemporal-AI/A1.git
+                gpu: Instinct MI210
+                gpu_amd:
+                    - id: "0"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0xc6ed7c5159e83b1"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "5"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0xa95e252cd7a6e54e"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "1"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0x333c966d5f3b3375"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "7"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0x5656afec2788d569"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "6"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0xad1bf75f50313455"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "3"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0xbabaa83bdb6fe877"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "4"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0xdee6d87744a36ae"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "2"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0x2a954aa975e59d15"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                gpu_count: 8
+                host: auh7-1b-gpu-292
+                memory:
+                    total: "2434606923776"
+                os: Linux-5.15.0-140-generic-x86_64-with-glibc2.35
+                program: /vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py
+                python: CPython 3.10.18
+                root: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb
+                slurm:
+                    cluster_name: ai-04r
+                    conf: /etc/slurm/slurm.conf
+                    cpus_on_node: "128"
+                    gpus_on_node: "8"
+                    gtids: "0"
+                    job_account: faculty-acc
+                    job_cpus_per_node: "128"
+                    job_end_time: "1759424668"
+                    job_gid: "2000"
+                    job_gpus: 0,1,2,3,4,5,6,7
+                    job_id: "1934"
+                    job_name: mh_realworld
+                    job_nodelist: auh7-1b-gpu-292
+                    job_num_nodes: "1"
+                    job_partition: faculty
+                    job_qos: xdqos
+                    job_start_time: "1759165468"
+                    job_uid: "2013"
+                    job_user: xiaodan
+                    jobid: "1934"
+                    localid: "0"
+                    nnodes: "1"
+                    nodeid: "0"
+                    nodelist: auh7-1b-gpu-292
+                    nprocs: "1"
+                    ntasks: "1"
+                    ntasks_per_node: "1"
+                    oom_kill_step: "0"
+                    prio_process: "0"
+                    procid: "0"
+                    submit_dir: /vast/users/xiaodan/zhangjian/A1/launch_scripts
+                    submit_host: auh-1b-cpu-login-001
+                    task_pid: "1357871"
+                    tasks_per_node: "1"
+                    topology_addr: auh7-1b-gpu-292
+                    topology_addr_pattern: node
+                startedAt: "2025-09-29T17:05:12.548535Z"
+                writerId: zpxgtt7ztfwstl3gysdjfozjodgqnvkm
+        m: []
+        python_version: 3.10.18
+        t:
+            "1":
+                - 1
+                - 2
+                - 3
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 63
+                - 71
+                - 83
+                - 95
+                - 105
+            "2":
+                - 1
+                - 2
+                - 3
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 63
+                - 71
+                - 83
+                - 95
+                - 105
+            "3":
+                - 2
+                - 13
+                - 15
+                - 16
+                - 61
+            "4": 3.10.18
+            "5": 0.21.4
+            "6": 4.56.1
+            "10":
+                - 19
+            "12": 0.21.4
+            "13": linux-x86_64
+activation_checkpointing:
+    value: whole_layer
+allow_resume:
+    value: false
+batch_divisor:
+    value: global_batch
+canceled_check_interval:
+    value: 50
+checkpoint_dir:
+    value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924
+compile:
+    value: null
+console_log_interval:
+    value: 1
+data:
+    value:
+        dataset: vla_dataset_realworld
+        drop_last: true
+        for_inference: false
+        lerobot_episode_index_end: null
+        lerobot_episode_index_start: null
+        mixture: null
+        multi_modal: torch
+        num_workers: 0
+        pad: to_max
+        persistent_workers: false
+        pin_memory: true
+        prefetch_factor: null
+        rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds
+        rlds_dataset_name: libero_4_task_suites_no_noops
+        rlds_read_threads: 8
+        rlds_shuffle_buffer_size: 100000
+        rlds_traj_threads: 8
+        root_size_mixture: null
+        seed: 95818
+        sequence_length: 1600
+        shuffle: true
+        shuffle_messages: false
+        split: train
+        timeout: 0
+        use_proprio: true
+        use_wrist_image: true
+device_eval_batch_size:
+    value: 4
+device_inf_eval_batch_size:
+    value: 16
+device_train_batch_size:
+    value: 15
+device_train_grad_accum:
+    value: 0
+device_train_microbatch_size:
+    value: 16
+dry_run:
+    value: false
+early_exit:
+    value: false
+epoch:
+    value: null
+eval_interval:
+    value: 0
+eval_on_load:
+    value: false
+eval_subset_num_batches:
+    value: -1
+evaluators:
+    value:
+        - data:
+            dataset: vla_dataset_realworld
+            drop_last: true
+            for_inference: false
+            lerobot_episode_index_end: 765
+            lerobot_episode_index_start: 353
+            mixture: null
+            multi_modal: torch
+            num_workers: 0
+            pad: to_max
+            persistent_workers: true
+            pin_memory: true
+            prefetch_factor: null
+            rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds
+            rlds_dataset_name: libero_4_task_suites_no_noops
+            rlds_read_threads: 8
+            rlds_shuffle_buffer_size: 256000
+            rlds_traj_threads: 8
+            root_size_mixture: null
+            seed: null
+            sequence_length: 1600
+            shuffle: false
+            shuffle_messages: false
+            split: validation
+            timeout: 0
+            use_proprio: true
+            use_wrist_image: true
+          device_eval_batch_size: null
+          eval_name: null
+          label: val
+          max_examples: null
+          max_new_tokens: 448
+          mm_evaluator: null
+          save_dir: null
+          save_to_checkpoint_dir: false
+          skip_if_metrics_cached: true
+          subset_num_batches: 64
+extra_steps_after_cancel:
+    value: 10
+fast_forward_batches:
+    value: null
+force_save_unsharded:
+    value: false
+fsdp:
+    value:
+        hybrid_sharding_num_model_replicas: null
+        precision: float
+        sharding_strategy: FULL_SHARD
+        use_orig_params: true
+        wrapping_strategy: by_block_and_size
+ft_connector:
+    value: false
+ft_embedding:
+    value: lm_head
+ft_llm:
+    value: true
+ft_vit:
+    value: false
+fused_loss:
+    value: null
+gen1_gc_interval:
+    value: 1
+global_train_batch_size:
+    value: 126
+inf_eval_interval:
+    value: -1
+inf_eval_subset_num_batches:
+    value: -1
+inf_evaluators:
+    value: []
+initial_model_checkpoint:
+    value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924
+keep_lr_on_load:
+    value: true
+load_model_config:
+    value: null
+load_path:
+    value: null
+load_path_sharded_checkpointer:
+    value: null
+lora:
+    value: false
+lora_connector:
+    value: false
+lora_llm:
+    value: false
+lora_rank:
+    value: 8
+lora_vit:
+    value: false
+max_duration:
+    value: 500000
+max_grad_norm:
+    value: 1
+max_grad_norm_ratio:
+    value: null
+model:
+    value:
+        action_dim: 7
+        action_head: l1_regression
+        action_head_dit_depth: 28
+        action_head_dit_hidden_size: 1152
+        action_head_dit_num_heads: 16
+        action_tokenizer:
+            identifier: physical-intelligence/fast
+            tokenizer_dir: null
+        action_use_left_eef: false
+        action_use_mobile_base: false
+        activation_type: swiglu
+        additional_vocab_size: 128
+        always_start_with_space: true
+        attention_dropout: 0
+        attention_layer_norm: false
+        attention_layer_norm_with_affine: true
+        attention_type: sdpa
+        bias_for_layer_norm: null
+        block_group_size: 1
+        block_type: sequential
+        clip_qkv: null
+        crop_mode: overlap-and-resize-c2
+        d_model: 3584
+        default_inference_len: 65
+        embedding_dropout: 0
+        embedding_size: 152064
+        ff_out_size: 0
+        fix_image_padding: true
+        float32_attention: true
+        head_dim: null
+        horizon: 8
+        image_feature_dropout: 0
+        image_padding_embed: pad_and_partial_pad
+        image_pooling_2d: attention_meanq
+        image_pooling_h: 2
+        image_pooling_w: 2
+        image_projector: mlp
+        include_bias: false
+        init_cutoff_factor: null
+        init_device: null
+        init_fn: normal
+        init_std: 0.02
+        initializer_range: 0.02
+        layer_norm_eps: 1e-06
+        layer_norm_type: rms
+        layer_norm_with_affine: true
+        llm_causal_attention: false
+        llm_load_path: /weka/oe-training-default/mm-olmo/pretrained_llms/qwen2-7b.pt
+        low_cpu_fsdp: true
+        max_crops: 12
+        max_position_embeddings: null
+        max_sequence_length: 4096
+        message_formatting: role
+        mlp_hidden_size: 37888
+        mlp_ratio: 4
+        moe_capacity_factor: 1.25
+        moe_dropless: true
+        moe_interleave: false
+        moe_lbl_in_fp32: false
+        moe_log_expert_assignment: false
+        moe_loss_weight: 0.1
+        moe_mlp_impl: sparse
+        moe_num_experts: 8
+        moe_shared_expert: false
+        moe_top_k: 2
+        moe_zloss_weight: null
+        multi_annotation_weighting: root_subsegments
+        n_heads: 28
+        n_kv_heads: 4
+        n_layers: 28
+        new_embedding_init_range: 0.02
+        norm_after: false
+        normalize_input_embeds: false
+        num_diffusion_inference_steps: 30
+        num_diffusion_steps: 1000
+        overlap_margins:
+            - 4
+            - 4
+        pad_tokenizer: true
+        pad_value: 0
+        precision: amp_bf16
+        prompt_type: uber_model
+        qkv_bias: true
+        residual_dropout: 0.1
+        response_residual_dropout: 0
+        rope: true
+        rope_full_precision: true
+        rope_theta: 1e+06
+        scale_logits: false
+        system_prompt_kind: demo_or_style
+        tokenizer:
+            identifier: Qwen/Qwen2-7B
+            tokenizer_dir: null
+        use_col_tokens: true
+        use_position_ids: true
+        use_proprio: true
+        vision_backbone:
+            attention_dropout: 0
+            fsdp_wrap: false
+            image_default_input_size:
+                - 336
+                - 336
+            image_dropout_rate: 0
+            image_emb_dim: 1024
+            image_head_dim: 64
+            image_mlp_activations: quick_gelu
+            image_mlp_dim: 4096
+            image_model_type: openai
+            image_norm_eps: 1e-05
+            image_num_heads: 16
+            image_num_key_value_heads: 16
+            image_num_layers: 23
+            image_num_pos: 577
+            image_patch_size: 14
+            image_pos_patch_size: 14
+            initializer_range: 0.02
+            residual_dropout: 0
+            resize_mode: default
+        vit_layers:
+            - -2
+            - -9
+        vit_load_path: /weka/oe-training-default/mm-olmo/pretrained_image_encoders/vit-l-14-336.pt
+        vocab_size: 152064
+        weight_tying: false
+multi_component_grad_norm:
+    value: true
+no_pre_train_checkpoint:
+    value: true
+optimizer:
+    value:
+        betas:
+            - 0.9
+            - 0.95
+        connector_betas:
+            - 0.9
+            - 0.95
+        connector_eps: 1e-06
+        connector_learning_rate: 0.0002
+        connector_weight_decay: 0
+        eps: 1e-05
+        learning_rate: 0.0001
+        llm_betas:
+            - 0.9
+            - 0.95
+        llm_eps: 1e-06
+        llm_learning_rate: 5e-05
+        llm_weight_decay: 0
+        metrics_log_interval: 20
+        name: adamw
+        vit_betas:
+            - 0.9
+            - 0.95
+        vit_eps: 1e-06
+        vit_learning_rate: 6e-06
+        vit_weight_decay: 0
+        weight_decay: 0.01
+precision:
+    value: amp_bf16
+python_profiling:
+    value: false
+remote_save_folder:
+    value: null
+reset_dataloader_state:
+    value: false
+reset_optimizer_state:
+    value: false
+reset_trainer_state:
+    value: false
+restore_dataloader:
+    value: true
+run_name:
+    value: realworld_20250929_170441
+save_dataloader_state:
+    value: false
+save_folder:
+    value: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1
+save_interval:
+    value: 500
+save_interval_action_head:
+    value: 500
+save_interval_ephemeral:
+    value: null
+save_interval_unsharded:
+    value: 500
+save_num_action_head_checkpoints_to_keep:
+    value: 2
+save_num_checkpoints_to_keep:
+    value: 1
+save_num_unsharded_checkpoints_to_keep:
+    value: 1
+save_overwrite:
+    value: true
+scheduler:
+    value:
+        alpha_f: 0.1
+        connector_t_warmup: 200
+        grad_clip_warmup_factor: null
+        grad_clip_warmup_steps: null
+        llm_t_warmup: 2000
+        name: multimodal
+        t_max: null
+        t_warmup: 100
+        units: steps
+        vit_t_warmup: 2000
+        warmup_min_lr: 0
+seed:
+    value: 6198
+sharded_checkpointer:
+    value: torch_legacy
+softmax_auxiliary_loss:
+    value: true
+softmax_auxiliary_loss_scale:
+    value: 0.0001
+speed_monitor:
+    value:
+        gpu_flops_available: null
+        window_size: 20
+stop_after:
+    value: null
+stop_at:
+    value: 500000
+time_limit:
+    value: null
+torch_profiling:
+    value: false
+train_exit_random_layer:
+    value: false
+use_lora:
+    value: true

all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/output.log

@@ -0,0 +1,122 @@
+wandb: Detected [openai] in use.
+wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
+wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/
+09/29 [17:05:14] WARNING  | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No       warnings.py:109
+                          device id is provided via `init_process_group` or `barrier `. Using the current device set by the user.
+                            warnings.warn(  # warn only once
+
+****** vla_cfg: {'datasets': {'rlds': {'name': None, 'path': None, 'weight': 1.0, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk', 0.1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50', 0.1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser', 0.1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue', 0.1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen', 0.1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote', 0.1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue', 0.1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe', 0.1, 'bounds']], 'open-source-real-world': {'rlds': {'name': 'a1_real_world', 'path': '/vast/users/xiaodan/zhangjian/datasets/OXE', 'weight': 0.1, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [], 'agibot': {'path': None, 'weight': 0.1, 'action_proprio_normalization_type': None}}}, 'model': {'action_head': {'action_dim': 16, 'proprio_dim': 16, 'num_actions_chunk': 8, 'action_tokens_mapping': {'left_end_effector': 8, 'right_end_effector': 8}, 'use_left_eef': True, 'use_mobile_base': False}}}
+****** Skip RLDS main; path not found: None
+****** start build LeRobot main...
+build_tokenizer, cache_dir None tokenizer_dir None
+09/29 [17:05:28] INFO     | >> Padding tokenizer with 418 tokens                                                                                                    tokenizer.py:130
+09/29 [17:05:29] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk
+****** length of the dataset: 72641
+09/29 [17:05:37] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50
+****** length of the dataset: 27906
+09/29 [17:05:44] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser
+****** length of the dataset: 13441
+09/29 [17:05:48] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue
+****** length of the dataset: 10316
+09/29 [17:05:49] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen
+****** length of the dataset: 17131
+09/29 [17:05:50] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote
+****** length of the dataset: 15765
+09/29 [17:05:51] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue
+****** length of the dataset: 90
+09/29 [17:05:52] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe
+****** length of the dataset: 18397
+****** Skip RLDS open-source-real-world; mixture 'a1_real_world' not found under: /vast/users/xiaodan/zhangjian/datasets/OXE
+****** Expect one of: []
+****** path: None
+****** Skip AgiBotWorld-Alpha open-source-real-world; path not found: None
+****** After build vla train dataset...
+****** iterable_sources: [<olmo.data.dataset.IterableDatasetWrapper object at 0x7fd400f07970>, <olmo.data.dataset.IterableDatasetWrapper object at 0x7fd40649db70>, <olmo.data.dataset.IterableDatasetWrapper object at 0x7fd4063572b0>, <olmo.data.dataset.IterableDatasetWrapper object at 0x7fd405f41ae0>, <olmo.data.dataset.IterableDatasetWrapper object at 0x7fd40647a710>, <olmo.data.dataset.IterableDatasetWrapper object at 0x7fd40637cd30>, <olmo.data.dataset.IterableDatasetWrapper object at 0x7fd406013e50>, <olmo.data.dataset.IterableDatasetWrapper object at 0x7fd405f42ef0>]
+****** Before build mixed iterable dataset...
+****** Build vla train dataloader successfully!
+************************* Build train_dataloader successful!
+************************* Before build_inf_evaluators
+09/29 [17:05:53] WARNING  | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No       warnings.py:109
+                          device id is provided via `init_process_group` or `barrier `. Using the current device set by the user.
+                            warnings.warn(  # warn only once
+
+************************* Build evaluators successful!
+************************* Early exit flags: early_exit=False
+************************* Initialize model successful!
+***** state_dict_path: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924/model.pt
+***** Load checkpoint successful!
+missing keys: ['action_head.model.layer_norm1.weight', 'action_head.model.layer_norm1.bias', 'action_head.model.fc1.weight', 'action_head.model.fc1.bias', 'action_head.model.mlp_resnet_blocks.0.ffn.0.weight', 'action_head.model.mlp_resnet_blocks.0.ffn.0.bias', 'action_head.model.mlp_resnet_blocks.0.ffn.1.weight', 'action_head.model.mlp_resnet_blocks.0.ffn.1.bias', 'action_head.model.mlp_resnet_blocks.1.ffn.0.weight', 'action_head.model.mlp_resnet_blocks.1.ffn.0.bias', 'action_head.model.mlp_resnet_blocks.1.ffn.1.weight', 'action_head.model.mlp_resnet_blocks.1.ffn.1.bias', 'action_head.model.layer_norm2.weight', 'action_head.model.layer_norm2.bias', 'action_head.model.fc2.weight', 'action_head.model.fc2.bias', 'proprio_projector.fc1.weight', 'proprio_projector.fc1.bias', 'proprio_projector.fc2.weight', 'proprio_projector.fc2.bias']
+unexpected keys: []
+************************* Initialize model successful!
+************************* LoRA flags: use_lora=True, lora_llm=False, lora_vit=False, lora_connector=False
+************************* Before add lora to model
+************************* Before FSDP model wrapping
+************************* FSDP model wrapping successful!
+************************* Before building optimizer and scheduler
+************* Before get lora params
+************* After get lora params successfully
+09/29 [17:07:04] INFO     | >> Constructing optimizer with 2 param groups                                                                                              optim.py:1283
+**************************************************
+After building optimizer and scheduler and model, before training, peak GPU memory (MB): 35614
+************************* VLATrainer initialized successfully!
+************************* Before trainer.fit()
+Pre-train system metrics
+    System/Peak GPU Memory (MB)=35,614
+09/29 [17:07:05] WARNING  | >> /vast/users/xiaodan/zhangjian/A1/olmo/data/collator.py:200: UserWarning: To copy construct from a tensor, it is recommended to use    warnings.py:109
+                          sourceTensor.detach().clone() or sourceTensor.detach().clone().requires_grad_(True), rather than torch.tensor(sourceTensor).
+                            timestep_list = [torch.tensor(ex["timestep"], dtype=torch.int64) for ex in batch]
+
+!!!Training failed: Given normalized_shape=[57344], expected input with shape [*, 57344], but got input of size[15, 8, 28672]
+Traceback (most recent call last):
+  File "/vast/users/xiaodan/zhangjian/A1/scripts/train_for_action.py", line 593, in main
+    trainer.fit()
+  File "/vast/users/xiaodan/zhangjian/A1/olmo/train.py", line 2331, in fit
+    metrics = self.train_step(batch, reduce_global_loss=should_log_this_step)
+  File "/vast/users/xiaodan/zhangjian/A1/olmo/train.py", line 2038, in train_step
+    ce_batch_loss, z_batch_loss, batch_accuracy, lb_batch_loss, moe_z_batch_loss, expert_assignments,action_loss = self.train_batch(batch)
+  File "/vast/users/xiaodan/zhangjian/A1/olmo/train.py", line 1956, in train_batch
+    accuracy, ce_loss, z_loss, logits, action_loss = self.model_forward(
+  File "/vast/users/xiaodan/zhangjian/A1/olmo/train.py", line 1813, in model_forward
+    outputs = self.fsdp_model.forward(
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py", line 854, in forward
+    output = self._fsdp_wrapped_module(*args, **kwargs)
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/vast/users/xiaodan/zhangjian/A1/olmo/vla/affordvla.py", line 1042, in forward
+    predicted_actions = self.action_head.predict_action(action_hidden_states)
+  File "/vast/users/xiaodan/zhangjian/A1/olmo/vla/action_heads.py", line 249, in predict_action
+    action = self.model(rearranged_actions_hidden_states)
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/vast/users/xiaodan/zhangjian/A1/olmo/vla/action_heads.py", line 210, in forward
+    x = self.layer_norm1(x)  # shape: (batch_size, input_dim)
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/modules/normalization.py", line 217, in forward
+    return F.layer_norm(
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/nn/functional.py", line 2905, in layer_norm
+    return torch.layer_norm(
+RuntimeError: Given normalized_shape=[57344], expected input with shape [*, 57344], but got input of size[15, 8, 28672]
+wandb: WARNING The `quiet` argument to `wandb.run.finish()` is deprecated, use `wandb.Settings(quiet=...)` to set this instead.

all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/requirements.txt

@@ -0,0 +1,284 @@
+ai2-molmo==0.0.0
+astunparse==1.6.3
+flatbuffers==25.2.10
+gast==0.6.0
+google-pasta==0.2.0
+h5py==3.14.0
+libclang==18.1.1
+Markdown==3.9
+namex==0.1.0
+opt_einsum==3.4.0
+optree==0.17.0
+tensorboard-data-server==0.7.2
+tensorflow-io-gcs-filesystem==0.37.1
+termcolor==3.1.0
+Werkzeug==3.1.3
+Brotli==1.1.0
+Farama-Notifications==0.0.4
+MarkupSafe==2.1.5
+PyYAML==6.0.2
+absl-py==2.3.1
+accelerate==1.10.1
+ai2-molmo==0.0.0
+aiofiles==24.1.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.12.15
+aiosignal==1.4.0
+annotated-types==0.7.0
+antlr4-python3-runtime==4.9.3
+anyio==4.10.0
+array_record==0.8.1
+async-timeout==5.0.1
+attrs==25.3.0
+av==15.1.0
+backports.tarfile==1.2.0
+beaker-gantry==3.2.0
+beaker-py==2.5.0
+black==23.12.1
+blinker==1.9.0
+boltons==25.0.0
+boto3==1.40.33
+botocore==1.40.33
+build==1.3.0
+cached_path==1.7.3
+cached-property==2.0.1
+cachetools==5.5.2
+certifi==2025.8.3
+cffi==2.0.0
+charset-normalizer==3.4.3
+click==8.2.1
+click-help-colors==0.9.4
+click-option-group==0.5.7
+cloudpickle==3.1.1
+cmake==4.1.0
+contourpy==1.3.2
+cryptography==46.0.1
+cycler==0.12.1
+dataclass-extensions==0.2.3
+datasets==3.6.0
+decorator==5.2.1
+deepdiff==8.6.1
+diffusers==0.35.1
+dill==0.3.8
+distro==1.9.0
+dlimp==0.0.1
+dm-tree==0.1.9
+docutils==0.22.1
+draccus==0.10.0
+editdistance==0.8.1
+einops==0.8.1
+einops-exts==0.0.4
+etils==1.13.0
+evdev==1.9.2
+exceptiongroup==1.3.0
+face==24.0.0
+fastapi==0.116.2
+ffmpy==0.6.1
+fiddle==0.3.0
+filelock==3.13.1
+Flask==3.1.2
+fonttools==4.60.0
+frozenlist==1.7.0
+fsspec==2023.9.2
+ftfy==6.3.1
+gcsfs==2023.9.2
+gitdb==4.0.12
+GitPython==3.1.45
+glom==24.11.0
+google-api-core==2.25.1
+google-auth==2.40.3
+google-auth-oauthlib==1.2.2
+google-cloud-core==2.4.3
+google-cloud-storage==2.19.0
+google-crc32c==1.7.1
+google-resumable-media==2.7.2
+googleapis-common-protos==1.70.0
+gradio==5.46.0
+gradio_client==1.13.0
+graphviz==0.21
+groovy==0.1.2
+grpcio==1.75.0
+gymnasium==0.29.1
+h11==0.16.0
+hf_transfer==0.1.9
+hf-xet==1.1.10
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==0.35.0
+id==1.5.0
+idna==3.10
+imageio==2.37.0
+imageio-ffmpeg==0.6.0
+importlib_metadata==8.7.0
+importlib_resources==6.5.2
+iniconfig==2.1.0
+inquirerpy==0.3.4
+isort==5.12.0
+itsdangerous==2.2.0
+jaraco.classes==3.4.0
+jaraco.context==6.0.1
+jaraco.functools==4.3.0
+jeepney==0.9.0
+Jinja2==3.1.4
+jiter==0.11.0
+jmespath==1.0.1
+joblib==1.5.2
+jsonlines==4.0.0
+keras==2.15.0
+keyring==25.6.0
+kiwisolver==1.4.9
+latex2sympy2_extended==1.10.2
+lerobot==0.3.4
+Levenshtein==0.27.1
+libcst==1.8.4
+lightning-utilities==0.15.2
+markdown-it-py==4.0.0
+math-verify==0.8.0
+matplotlib==3.10.6
+mdurl==0.1.2
+mergedeep==1.3.4
+ml-dtypes==0.2.0
+ml_dtypes==0.5.3
+more-itertools==10.8.0
+mpmath==1.3.0
+msgspec==0.19.0
+multidict==6.6.4
+multiprocess==0.70.16
+mypy==1.3.0
+mypy_extensions==1.1.0
+necessary==0.4.3
+networkx==3.3
+nh3==0.3.0
+nltk==3.9.1
+numpy==1.26.4
+oauthlib==3.3.1
+omegaconf==2.3.0
+openai==1.108.0
+opencv-python-headless==4.12.0.88
+OpenEXR==3.4.0
+orderly-set==5.5.0
+orjson==3.11.3
+packaging==25.0
+pandas==2.3.2
+pathspec==0.12.1
+petname==2.6
+pfzy==0.3.4
+pillow==11.0.0
+pip==25.2
+platformdirs==4.4.0
+pluggy==1.6.0
+promise==2.3
+prompt_toolkit==3.0.52
+propcache==0.3.2
+proto-plus==1.26.1
+protobuf==4.21.12
+protobuf==6.32.1
+psutil==7.1.0
+pyarrow==21.0.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.2
+pycparser==2.23
+pydantic==2.11.9
+pydantic_core==2.33.2
+pydub==0.25.1
+Pygments==2.19.2
+pynput==1.8.1
+pyparsing==3.2.4
+pyproject_hooks==1.2.0
+pyserial==3.5
+pytest==8.4.2
+pytest-sphinx==0.6.3
+python-dateutil==2.9.0.post0
+python-Levenshtein==0.27.1
+python-multipart==0.0.20
+python-xlib==0.33
+pytorch-triton-rocm==3.4.0
+pytz==2025.2
+pyyaml-include==1.4.1
+RapidFuzz==3.14.1
+readme_renderer==44.0
+regex==2025.9.1
+requests==2.32.5
+requests-oauthlib==2.0.0
+requests-toolbelt==1.0.0
+requirements-parser==0.13.0
+rerun-sdk==0.22.1
+rfc3986==2.0.0
+rich==13.9.4
+rsa==4.9.1
+ruff==0.13.0
+s3transfer==0.14.0
+safehttpx==0.1.6
+safetensors==0.6.2
+scikit-learn==1.7.2
+scipy==1.15.3
+SecretStorage==3.4.0
+semantic-version==2.10.0
+sentencepiece==0.2.1
+sentry-sdk==2.38.0
+setuptools==78.1.1
+shellingham==1.5.4
+six==1.17.0
+smart_open==7.3.1
+smashed==0.21.5
+smmap==5.0.2
+sniffio==1.3.1
+starlette==0.48.0
+sympy==1.13.3
+tensorboard==2.15.2
+tensorboard==2.19.0
+tensorflow==2.15.0
+tensorflow-addons==0.23.0
+tensorflow-datasets==4.9.3
+tensorflow-estimator==2.15.0
+tensorflow-graphics==2021.12.3
+tensorflow-metadata==1.17.2
+threadpoolctl==3.6.0
+timm==1.0.19
+tokenizers==0.22.0
+toml==0.10.2
+tomli==2.2.1
+tomlkit==0.13.3
+torch==2.8.0+rocm6.4
+torchcodec==0.5
+torchmetrics==1.8.2
+torchvision==0.23.0+rocm6.4
+tqdm==4.67.1
+transformers==4.56.1
+trimesh==4.8.2
+trouting==0.3.3
+twine==6.2.0
+typeguard==2.13.3
+typer==0.17.4
+typing_extensions==4.15.0
+typing-inspect==0.9.0
+typing-inspection==0.4.1
+tzdata==2025.2
+urllib3==2.5.0
+uvicorn==0.35.0
+wandb==0.21.4
+wcwidth==0.2.13
+websockets==15.0.1
+wheel==0.45.1
+wrapt==1.14.2
+xxhash==3.5.0
+yarl==1.20.1
+zipp==3.23.0
+lerobot==0.3.4
+minLoRA==0.1.0
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.2
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.45.1
+zipp==3.19.2

all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/wandb-metadata.json

@@ -0,0 +1,202 @@
+{
+  "os":  "Linux-5.15.0-140-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.18",
+  "startedAt":  "2025-09-29T17:05:12.548535Z",
+  "args":  [
+    "qwen2_7b",
+    "save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1",
+    "--vision_backbone",
+    "openai",
+    "--action_head",
+    "l1_regression",
+    "--seq_len",
+    "1600",
+    "--ft_llm",
+    "--checkpoint",
+    "/vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924",
+    "--device_train_microbatch_size",
+    "16",
+    "--global_batch_size",
+    "126",
+    "--dataset",
+    "vla_dataset_realworld",
+    "--llm_learning_rate",
+    "5e-5",
+    "--wandb_entity",
+    "henryeap",
+    "--wandb_project",
+    "a1-realworld",
+    "--wandb_run_name",
+    "realworld",
+    "--save_overwrite"
+  ],
+  "program":  "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py",
+  "codePath":  "launch_scripts/train_vla.py",
+  "codePathLocal":  "launch_scripts/train_vla.py",
+  "git":  {
+    "remote":  "https://github.com/Spatialtemporal-AI/A1.git",
+    "commit":  "50cf9fd3143e218eb94104381c16c0482ac52f0d"
+  },
+  "email":  "ihenrykwok@outlook.com",
+  "root":  "/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb",
+  "host":  "auh7-1b-gpu-292",
+  "executable":  "/vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10",
+  "cpu_count":  64,
+  "cpu_count_logical":  128,
+  "gpu":  "Instinct MI210",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "470343073792",
+      "used":  "50842279936"
+    }
+  },
+  "memory":  {
+    "total":  "2434606923776"
+  },
+  "gpu_amd":  [
+    {
+      "id":  "0",
+      "uniqueId":  "0xc6ed7c5159e83b1",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "5",
+      "uniqueId":  "0xa95e252cd7a6e54e",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "1",
+      "uniqueId":  "0x333c966d5f3b3375",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "7",
+      "uniqueId":  "0x5656afec2788d569",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "6",
+      "uniqueId":  "0xad1bf75f50313455",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "3",
+      "uniqueId":  "0xbabaa83bdb6fe877",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "4",
+      "uniqueId":  "0xdee6d87744a36ae",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "2",
+      "uniqueId":  "0x2a954aa975e59d15",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "ai-04r",
+    "conf":  "/etc/slurm/slurm.conf",
+    "cpus_on_node":  "128",
+    "gpus_on_node":  "8",
+    "gtids":  "0",
+    "job_account":  "faculty-acc",
+    "job_cpus_per_node":  "128",
+    "job_end_time":  "1759424668",
+    "job_gid":  "2000",
+    "job_gpus":  "0,1,2,3,4,5,6,7",
+    "job_id":  "1934",
+    "job_name":  "mh_realworld",
+    "job_nodelist":  "auh7-1b-gpu-292",
+    "job_num_nodes":  "1",
+    "job_partition":  "faculty",
+    "job_qos":  "xdqos",
+    "job_start_time":  "1759165468",
+    "job_uid":  "2013",
+    "job_user":  "xiaodan",
+    "jobid":  "1934",
+    "localid":  "0",
+    "nnodes":  "1",
+    "nodeid":  "0",
+    "nodelist":  "auh7-1b-gpu-292",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "oom_kill_step":  "0",
+    "prio_process":  "0",
+    "procid":  "0",
+    "submit_dir":  "/vast/users/xiaodan/zhangjian/A1/launch_scripts",
+    "submit_host":  "auh-1b-cpu-login-001",
+    "task_pid":  "1357871",
+    "tasks_per_node":  "1",
+    "topology_addr":  "auh7-1b-gpu-292",
+    "topology_addr_pattern":  "node"
+  },
+  "writerId":  "zpxgtt7ztfwstl3gysdjfozjodgqnvkm"
+}

all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_step":0,"System/Peak GPU Memory (MB)":35614.78125,"_timestamp":1.759165624842901e+09,"_wandb":{"runtime":118},"_runtime":118.9149238}

all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/logs/debug-core.log

@@ -0,0 +1,16 @@
+{"time":"2025-09-29T17:05:12.599767574Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpblcfcrd2/port-1358048.txt","pid":1358048,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-29T17:05:12.601129497Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1358048}
+{"time":"2025-09-29T17:05:12.601134567Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1358048-1358213-1657513860/socket","Net":"unix"}}
+{"time":"2025-09-29T17:05:12.784034321Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-29T17:05:12.788802342Z","level":"INFO","msg":"handleInformInit: received","streamId":"2lq20p1f","id":"1(@)"}
+{"time":"2025-09-29T17:05:13.92427811Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"2lq20p1f","id":"1(@)"}
+{"time":"2025-09-29T17:07:15.469972392Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"2lq20p1f","id":"1(@)"}
+{"time":"2025-09-29T17:07:15.473846438Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"2lq20p1f","id":"1(@)"}
+{"time":"2025-09-29T17:07:15.473852038Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-09-29T17:07:15.473858278Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-09-29T17:07:15.473865358Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-29T17:07:15.473865688Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-09-29T17:07:15.473901259Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-09-29T17:07:15.473904939Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-09-29T17:07:15.473942719Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-1358048-1358213-1657513860/socket","Net":"unix"}}
+{"time":"2025-09-29T17:07:15.47396587Z","level":"INFO","msg":"server is closed"}

all_l1/wandb/wandb/run-20250929_170512-2lq20p1f/logs/debug-internal.log

@@ -0,0 +1,12 @@
+{"time":"2025-09-29T17:05:12.790661794Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"}
+{"time":"2025-09-29T17:05:13.924227629Z","level":"INFO","msg":"stream: created new stream","id":"2lq20p1f"}
+{"time":"2025-09-29T17:05:13.92427238Z","level":"INFO","msg":"stream: started","id":"2lq20p1f"}
+{"time":"2025-09-29T17:05:13.92428108Z","level":"INFO","msg":"writer: started","stream_id":"2lq20p1f"}
+{"time":"2025-09-29T17:05:13.924301101Z","level":"INFO","msg":"handler: started","stream_id":"2lq20p1f"}
+{"time":"2025-09-29T17:05:13.924330541Z","level":"INFO","msg":"sender: started","stream_id":"2lq20p1f"}
+{"time":"2025-09-29T17:07:13.196500373Z","level":"INFO","msg":"handler: operation stats","stats":{"operations":[{"desc":"updating run metadata","runtime_seconds":0.001296762}],"total_operations":1}}
+{"time":"2025-09-29T17:07:15.018224116Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-09-29T17:07:15.470258007Z","level":"INFO","msg":"stream: closing","id":"2lq20p1f"}
+{"time":"2025-09-29T17:07:15.470272437Z","level":"INFO","msg":"handler: closed","stream_id":"2lq20p1f"}
+{"time":"2025-09-29T17:07:15.472123178Z","level":"INFO","msg":"sender: closed","stream_id":"2lq20p1f"}
+{"time":"2025-09-29T17:07:15.472129189Z","level":"INFO","msg":"stream: closed","id":"2lq20p1f"}

all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/config.yaml

@@ -0,0 +1,617 @@
+_wandb:
+    value:
+        cli_version: 0.21.4
+        e:
+            xw28krbghcx1p9g77lw10b5hvgwgqmco:
+                args:
+                    - qwen2_7b
+                    - save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1
+                    - --vision_backbone
+                    - openai
+                    - --action_head
+                    - l1_regression
+                    - --seq_len
+                    - "1600"
+                    - --ft_llm
+                    - --checkpoint
+                    - /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924
+                    - --device_train_microbatch_size
+                    - "16"
+                    - --global_batch_size
+                    - "126"
+                    - --dataset
+                    - vla_dataset_realworld
+                    - --llm_learning_rate
+                    - "5e-5"
+                    - --wandb_entity
+                    - henryeap
+                    - --wandb_project
+                    - a1-realworld
+                    - --wandb_run_name
+                    - realworld
+                    - --save_overwrite
+                codePath: launch_scripts/train_vla.py
+                codePathLocal: launch_scripts/train_vla.py
+                cpu_count: 64
+                cpu_count_logical: 128
+                disk:
+                    /:
+                        total: "470343073792"
+                        used: "50128396288"
+                email: ihenrykwok@outlook.com
+                executable: /vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10
+                git:
+                    commit: f2afcc15e05f491a8e50add64395fc1db0a1188d
+                    remote: https://github.com/Spatialtemporal-AI/A1.git
+                gpu: Instinct MI210
+                gpu_amd:
+                    - id: "5"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0x413935505e32b8da"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "3"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0x95be8fdc770fcfd7"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "1"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0x27087f06439a527d"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "0"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0x82728d7f9bd937e4"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "7"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0xa0442ab3bdd405c1"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "2"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0xaabcddaa244a3d6e"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "6"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0x12140cd9e24f12e9"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "4"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0x24ee801b7c402006"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                gpu_count: 8
+                host: auh7-1b-gpu-306
+                memory:
+                    total: "2434611519488"
+                os: Linux-5.15.0-140-generic-x86_64-with-glibc2.35
+                program: /vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py
+                python: CPython 3.10.18
+                root: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb
+                slurm:
+                    cluster_name: ai-04r
+                    conf: /etc/slurm/slurm.conf
+                    cpus_on_node: "128"
+                    gpus_on_node: "8"
+                    gtids: "0"
+                    job_account: faculty-acc
+                    job_cpus_per_node: "128"
+                    job_end_time: "1759481466"
+                    job_gid: "2000"
+                    job_gpus: 0,1,2,3,4,5,6,7
+                    job_id: "1970"
+                    job_name: mh_realworld
+                    job_nodelist: auh7-1b-gpu-306
+                    job_num_nodes: "1"
+                    job_partition: faculty
+                    job_qos: xdqos
+                    job_start_time: "1759222266"
+                    job_uid: "2013"
+                    job_user: xiaodan
+                    jobid: "1970"
+                    localid: "0"
+                    nnodes: "1"
+                    nodeid: "0"
+                    nodelist: auh7-1b-gpu-306
+                    nprocs: "1"
+                    ntasks: "1"
+                    ntasks_per_node: "1"
+                    oom_kill_step: "0"
+                    prio_process: "0"
+                    procid: "0"
+                    submit_dir: /vast/users/xiaodan/zhangjian/A1/launch_scripts
+                    submit_host: auh-1b-cpu-login-001
+                    task_pid: "589646"
+                    tasks_per_node: "1"
+                    topology_addr: auh7-1b-gpu-306
+                    topology_addr_pattern: node
+                startedAt: "2025-09-30T08:52:06.337927Z"
+                writerId: xw28krbghcx1p9g77lw10b5hvgwgqmco
+        m: []
+        python_version: 3.10.18
+        t:
+            "1":
+                - 1
+                - 2
+                - 3
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 63
+                - 71
+                - 83
+                - 95
+                - 105
+            "2":
+                - 1
+                - 2
+                - 3
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 63
+                - 71
+                - 83
+                - 95
+                - 105
+            "3":
+                - 13
+                - 15
+                - 16
+            "4": 3.10.18
+            "5": 0.21.4
+            "6": 4.56.1
+            "12": 0.21.4
+            "13": linux-x86_64
+activation_checkpointing:
+    value: whole_layer
+allow_resume:
+    value: false
+batch_divisor:
+    value: global_batch
+canceled_check_interval:
+    value: 50
+checkpoint_dir:
+    value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924
+compile:
+    value: null
+console_log_interval:
+    value: 1
+data:
+    value:
+        dataset: vla_dataset_realworld
+        drop_last: true
+        for_inference: false
+        lerobot_episode_index_end: null
+        lerobot_episode_index_start: null
+        mixture: null
+        multi_modal: torch
+        num_workers: 0
+        pad: to_max
+        persistent_workers: false
+        pin_memory: true
+        prefetch_factor: null
+        rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds
+        rlds_dataset_name: libero_4_task_suites_no_noops
+        rlds_read_threads: 8
+        rlds_shuffle_buffer_size: 100000
+        rlds_traj_threads: 8
+        root_size_mixture: null
+        seed: 95818
+        sequence_length: 1600
+        shuffle: true
+        shuffle_messages: false
+        split: train
+        timeout: 0
+        use_proprio: true
+        use_wrist_image: true
+device_eval_batch_size:
+    value: 4
+device_inf_eval_batch_size:
+    value: 16
+device_train_batch_size:
+    value: 15
+device_train_grad_accum:
+    value: 0
+device_train_microbatch_size:
+    value: 16
+dry_run:
+    value: false
+early_exit:
+    value: false
+epoch:
+    value: null
+eval_interval:
+    value: 0
+eval_on_load:
+    value: false
+eval_subset_num_batches:
+    value: -1
+evaluators:
+    value:
+        - data:
+            dataset: vla_dataset_realworld
+            drop_last: true
+            for_inference: false
+            lerobot_episode_index_end: 765
+            lerobot_episode_index_start: 353
+            mixture: null
+            multi_modal: torch
+            num_workers: 0
+            pad: to_max
+            persistent_workers: true
+            pin_memory: true
+            prefetch_factor: null
+            rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds
+            rlds_dataset_name: libero_4_task_suites_no_noops
+            rlds_read_threads: 8
+            rlds_shuffle_buffer_size: 256000
+            rlds_traj_threads: 8
+            root_size_mixture: null
+            seed: null
+            sequence_length: 1600
+            shuffle: false
+            shuffle_messages: false
+            split: validation
+            timeout: 0
+            use_proprio: true
+            use_wrist_image: true
+          device_eval_batch_size: null
+          eval_name: null
+          label: val
+          max_examples: null
+          max_new_tokens: 448
+          mm_evaluator: null
+          save_dir: null
+          save_to_checkpoint_dir: false
+          skip_if_metrics_cached: true
+          subset_num_batches: 64
+extra_steps_after_cancel:
+    value: 10
+fast_forward_batches:
+    value: null
+force_save_unsharded:
+    value: false
+fsdp:
+    value:
+        hybrid_sharding_num_model_replicas: null
+        precision: float
+        sharding_strategy: FULL_SHARD
+        use_orig_params: true
+        wrapping_strategy: by_block_and_size
+ft_connector:
+    value: false
+ft_embedding:
+    value: lm_head
+ft_llm:
+    value: true
+ft_vit:
+    value: false
+fused_loss:
+    value: null
+gen1_gc_interval:
+    value: 1
+global_train_batch_size:
+    value: 126
+inf_eval_interval:
+    value: -1
+inf_eval_subset_num_batches:
+    value: -1
+inf_evaluators:
+    value: []
+initial_model_checkpoint:
+    value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924
+keep_lr_on_load:
+    value: true
+load_model_config:
+    value: null
+load_path:
+    value: null
+load_path_sharded_checkpointer:
+    value: null
+lora:
+    value: false
+lora_connector:
+    value: false
+lora_llm:
+    value: false
+lora_rank:
+    value: 8
+lora_vit:
+    value: false
+max_duration:
+    value: 500000
+max_grad_norm:
+    value: 1
+max_grad_norm_ratio:
+    value: null
+model:
+    value:
+        action_dim: 7
+        action_head: l1_regression
+        action_head_dit_depth: 28
+        action_head_dit_hidden_size: 1152
+        action_head_dit_num_heads: 16
+        action_tokenizer:
+            identifier: physical-intelligence/fast
+            tokenizer_dir: null
+        action_use_left_eef: false
+        action_use_mobile_base: false
+        activation_type: swiglu
+        additional_vocab_size: 128
+        always_start_with_space: true
+        attention_dropout: 0
+        attention_layer_norm: false
+        attention_layer_norm_with_affine: true
+        attention_type: sdpa
+        bias_for_layer_norm: null
+        block_group_size: 1
+        block_type: sequential
+        clip_qkv: null
+        crop_mode: overlap-and-resize-c2
+        d_model: 3584
+        default_inference_len: 65
+        embedding_dropout: 0
+        embedding_size: 152064
+        ff_out_size: 0
+        fix_image_padding: true
+        float32_attention: true
+        head_dim: null
+        horizon: 8
+        image_feature_dropout: 0
+        image_padding_embed: pad_and_partial_pad
+        image_pooling_2d: attention_meanq
+        image_pooling_h: 2
+        image_pooling_w: 2
+        image_projector: mlp
+        include_bias: false
+        init_cutoff_factor: null
+        init_device: null
+        init_fn: normal
+        init_std: 0.02
+        initializer_range: 0.02
+        layer_norm_eps: 1e-06
+        layer_norm_type: rms
+        layer_norm_with_affine: true
+        llm_causal_attention: false
+        llm_load_path: /weka/oe-training-default/mm-olmo/pretrained_llms/qwen2-7b.pt
+        low_cpu_fsdp: true
+        max_crops: 12
+        max_position_embeddings: null
+        max_sequence_length: 4096
+        message_formatting: role
+        mlp_hidden_size: 37888
+        mlp_ratio: 4
+        moe_capacity_factor: 1.25
+        moe_dropless: true
+        moe_interleave: false
+        moe_lbl_in_fp32: false
+        moe_log_expert_assignment: false
+        moe_loss_weight: 0.1
+        moe_mlp_impl: sparse
+        moe_num_experts: 8
+        moe_shared_expert: false
+        moe_top_k: 2
+        moe_zloss_weight: null
+        multi_annotation_weighting: root_subsegments
+        n_heads: 28
+        n_kv_heads: 4
+        n_layers: 28
+        new_embedding_init_range: 0.02
+        norm_after: false
+        normalize_input_embeds: false
+        num_diffusion_inference_steps: 30
+        num_diffusion_steps: 1000
+        overlap_margins:
+            - 4
+            - 4
+        pad_tokenizer: true
+        pad_value: 0
+        precision: amp_bf16
+        prompt_type: uber_model
+        qkv_bias: true
+        residual_dropout: 0.1
+        response_residual_dropout: 0
+        rope: true
+        rope_full_precision: true
+        rope_theta: 1e+06
+        scale_logits: false
+        system_prompt_kind: demo_or_style
+        tokenizer:
+            identifier: Qwen/Qwen2-7B
+            tokenizer_dir: null
+        use_col_tokens: true
+        use_position_ids: true
+        use_proprio: true
+        vision_backbone:
+            attention_dropout: 0
+            fsdp_wrap: false
+            image_default_input_size:
+                - 336
+                - 336
+            image_dropout_rate: 0
+            image_emb_dim: 1024
+            image_head_dim: 64
+            image_mlp_activations: quick_gelu
+            image_mlp_dim: 4096
+            image_model_type: openai
+            image_norm_eps: 1e-05
+            image_num_heads: 16
+            image_num_key_value_heads: 16
+            image_num_layers: 23
+            image_num_pos: 577
+            image_patch_size: 14
+            image_pos_patch_size: 14
+            initializer_range: 0.02
+            residual_dropout: 0
+            resize_mode: default
+        vit_layers:
+            - -2
+            - -9
+        vit_load_path: /weka/oe-training-default/mm-olmo/pretrained_image_encoders/vit-l-14-336.pt
+        vocab_size: 152064
+        weight_tying: false
+multi_component_grad_norm:
+    value: true
+no_pre_train_checkpoint:
+    value: true
+optimizer:
+    value:
+        betas:
+            - 0.9
+            - 0.95
+        connector_betas:
+            - 0.9
+            - 0.95
+        connector_eps: 1e-06
+        connector_learning_rate: 0.0002
+        connector_weight_decay: 0
+        eps: 1e-05
+        learning_rate: 0.0001
+        llm_betas:
+            - 0.9
+            - 0.95
+        llm_eps: 1e-06
+        llm_learning_rate: 5e-05
+        llm_weight_decay: 0
+        metrics_log_interval: 20
+        name: adamw
+        vit_betas:
+            - 0.9
+            - 0.95
+        vit_eps: 1e-06
+        vit_learning_rate: 6e-06
+        vit_weight_decay: 0
+        weight_decay: 0.01
+precision:
+    value: amp_bf16
+python_profiling:
+    value: false
+remote_save_folder:
+    value: null
+reset_dataloader_state:
+    value: false
+reset_optimizer_state:
+    value: false
+reset_trainer_state:
+    value: false
+restore_dataloader:
+    value: true
+run_name:
+    value: realworld_20250930_085126
+save_dataloader_state:
+    value: false
+save_folder:
+    value: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1
+save_interval:
+    value: 500
+save_interval_action_head:
+    value: 500
+save_interval_ephemeral:
+    value: null
+save_interval_unsharded:
+    value: 500
+save_num_action_head_checkpoints_to_keep:
+    value: 2
+save_num_checkpoints_to_keep:
+    value: 1
+save_num_unsharded_checkpoints_to_keep:
+    value: 1
+save_overwrite:
+    value: true
+scheduler:
+    value:
+        alpha_f: 0.1
+        connector_t_warmup: 200
+        grad_clip_warmup_factor: null
+        grad_clip_warmup_steps: null
+        llm_t_warmup: 2000
+        name: multimodal
+        t_max: null
+        t_warmup: 100
+        units: steps
+        vit_t_warmup: 2000
+        warmup_min_lr: 0
+seed:
+    value: 6198
+sharded_checkpointer:
+    value: torch_legacy
+softmax_auxiliary_loss:
+    value: true
+softmax_auxiliary_loss_scale:
+    value: 0.0001
+speed_monitor:
+    value:
+        gpu_flops_available: null
+        window_size: 20
+stop_after:
+    value: null
+stop_at:
+    value: 500000
+time_limit:
+    value: null
+torch_profiling:
+    value: false
+train_exit_random_layer:
+    value: false
+use_lora:
+    value: true

all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/output.log

@@ -0,0 +1,86 @@
+wandb: Detected [openai] in use.
+wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
+wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/
+09/30 [08:52:08] WARNING  | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No       warnings.py:109
+                          device id is provided via `init_process_group` or `barrier `. Using the current device set by the user.
+                            warnings.warn(  # warn only once
+
+****** vla_cfg: {'datasets': {'rlds': {'name': None, 'path': None, 'weight': 1.0, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe', 1, 'bounds']], 'open-source-real-world': {'rlds': {'name': 'a1_real_world', 'path': '/vast/users/xiaodan/zhangjian/datasets/OXE', 'weight': 8, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [], 'agibot': {'path': '/vast/users/xiaodan/zhangjian/datasets/AgiBotWorld-Alpha', 'weight': 8, 'action_proprio_normalization_type': None}}}, 'model': {'action_head': {'action_dim': 16, 'proprio_dim': 16, 'num_actions_chunk': 8, 'action_tokens_mapping': {'left_end_effector': 8, 'right_end_effector': 8}, 'use_left_eef': True, 'use_mobile_base': False}}}
+****** Skip RLDS main; path not found: None
+****** start build LeRobot main...
+build_tokenizer, cache_dir None tokenizer_dir None
+09/30 [08:52:19] INFO     | >> Padding tokenizer with 418 tokens                                                                                                    tokenizer.py:130
+09/30 [08:52:20] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk
+****** length of the dataset: 72641
+09/30 [08:52:26] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50
+****** length of the dataset: 27906
+09/30 [08:52:27] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser
+****** length of the dataset: 13441
+09/30 [08:52:28] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue
+****** length of the dataset: 10316
+09/30 [08:52:29] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen
+****** length of the dataset: 17131
+09/30 [08:52:31] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote
+****** length of the dataset: 15765
+09/30 [08:52:32] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue
+****** length of the dataset: 90
+                 INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe
+****** length of the dataset: 18397
+****** Skip RLDS open-source-real-world; mixture 'a1_real_world' not found under: /vast/users/xiaodan/zhangjian/datasets/OXE
+****** Expect one of: []
+****** path: /vast/users/xiaodan/zhangjian/datasets/AgiBotWorld-Alpha
+****** before AgiBotWorldAlpha dataset...
+09/30 [08:52:33] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:485
+Traceback (most recent call last):
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/compat/_optional.py", line 135, in import_optional_dependency
+    module = importlib.import_module(name)
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1004, in _find_and_load_unlocked
+ModuleNotFoundError: No module named 'openpyxl'
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py", line 397, in <module>
+    train(cfg)
+  File "/vast/users/xiaodan/zhangjian/A1/scripts/train_for_action.py", line 160, in main
+    train_loader = build_train_dataloader(cfg, device)
+  File "/vast/users/xiaodan/zhangjian/A1/olmo/data/__init__.py", line 197, in build_train_dataloader
+    return build_vla_train_dataloader(train_config, device)
+  File "/vast/users/xiaodan/zhangjian/A1/olmo/data/__init__.py", line 384, in build_vla_train_dataloader
+    ds = build_agibot_train_dataset(train_config, normalization_type, device)
+  File "/vast/users/xiaodan/zhangjian/A1/olmo/data/__init__.py", line 487, in build_agibot_train_dataset
+    dataset = AgiBotWorldAlphaDataset(
+  File "<string>", line 13, in __init__
+  File "/vast/users/xiaodan/zhangjian/A1/olmo/data/vla/agibot_dataset.py", line 145, in __post_init__
+    self._frame_ranges_map: Optional[Dict[Tuple[str, int], Tuple[int, int]]] = self._load_frame_ranges_excel()
+  File "/vast/users/xiaodan/zhangjian/A1/olmo/data/vla/agibot_dataset.py", line 411, in _load_frame_ranges_excel
+    df = pd.read_excel(excel_path)
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/io/excel/_base.py", line 495, in read_excel
+    io = ExcelFile(
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/io/excel/_base.py", line 1567, in __init__
+    self._reader = self._engines[engine](
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/io/excel/_openpyxl.py", line 552, in __init__
+    import_optional_dependency("openpyxl")
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/compat/_optional.py", line 138, in import_optional_dependency
+    raise ImportError(msg)
+ImportError: Missing optional dependency 'openpyxl'.  Use pip or conda to install openpyxl.

all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/requirements.txt

@@ -0,0 +1,284 @@
+ai2-molmo==0.0.0
+astunparse==1.6.3
+flatbuffers==25.2.10
+gast==0.6.0
+google-pasta==0.2.0
+h5py==3.14.0
+libclang==18.1.1
+Markdown==3.9
+namex==0.1.0
+opt_einsum==3.4.0
+optree==0.17.0
+tensorboard-data-server==0.7.2
+tensorflow-io-gcs-filesystem==0.37.1
+termcolor==3.1.0
+Werkzeug==3.1.3
+Brotli==1.1.0
+Farama-Notifications==0.0.4
+MarkupSafe==2.1.5
+PyYAML==6.0.2
+absl-py==2.3.1
+accelerate==1.10.1
+ai2-molmo==0.0.0
+aiofiles==24.1.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.12.15
+aiosignal==1.4.0
+annotated-types==0.7.0
+antlr4-python3-runtime==4.9.3
+anyio==4.10.0
+array_record==0.8.1
+async-timeout==5.0.1
+attrs==25.3.0
+av==15.1.0
+backports.tarfile==1.2.0
+beaker-gantry==3.2.0
+beaker-py==2.5.0
+black==23.12.1
+blinker==1.9.0
+boltons==25.0.0
+boto3==1.40.33
+botocore==1.40.33
+build==1.3.0
+cached_path==1.7.3
+cached-property==2.0.1
+cachetools==5.5.2
+certifi==2025.8.3
+cffi==2.0.0
+charset-normalizer==3.4.3
+click==8.2.1
+click-help-colors==0.9.4
+click-option-group==0.5.7
+cloudpickle==3.1.1
+cmake==4.1.0
+contourpy==1.3.2
+cryptography==46.0.1
+cycler==0.12.1
+dataclass-extensions==0.2.3
+datasets==3.6.0
+decorator==5.2.1
+deepdiff==8.6.1
+diffusers==0.35.1
+dill==0.3.8
+distro==1.9.0
+dlimp==0.0.1
+dm-tree==0.1.9
+docutils==0.22.1
+draccus==0.10.0
+editdistance==0.8.1
+einops==0.8.1
+einops-exts==0.0.4
+etils==1.13.0
+evdev==1.9.2
+exceptiongroup==1.3.0
+face==24.0.0
+fastapi==0.116.2
+ffmpy==0.6.1
+fiddle==0.3.0
+filelock==3.13.1
+Flask==3.1.2
+fonttools==4.60.0
+frozenlist==1.7.0
+fsspec==2023.9.2
+ftfy==6.3.1
+gcsfs==2023.9.2
+gitdb==4.0.12
+GitPython==3.1.45
+glom==24.11.0
+google-api-core==2.25.1
+google-auth==2.40.3
+google-auth-oauthlib==1.2.2
+google-cloud-core==2.4.3
+google-cloud-storage==2.19.0
+google-crc32c==1.7.1
+google-resumable-media==2.7.2
+googleapis-common-protos==1.70.0
+gradio==5.46.0
+gradio_client==1.13.0
+graphviz==0.21
+groovy==0.1.2
+grpcio==1.75.0
+gymnasium==0.29.1
+h11==0.16.0
+hf_transfer==0.1.9
+hf-xet==1.1.10
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==0.35.0
+id==1.5.0
+idna==3.10
+imageio==2.37.0
+imageio-ffmpeg==0.6.0
+importlib_metadata==8.7.0
+importlib_resources==6.5.2
+iniconfig==2.1.0
+inquirerpy==0.3.4
+isort==5.12.0
+itsdangerous==2.2.0
+jaraco.classes==3.4.0
+jaraco.context==6.0.1
+jaraco.functools==4.3.0
+jeepney==0.9.0
+Jinja2==3.1.4
+jiter==0.11.0
+jmespath==1.0.1
+joblib==1.5.2
+jsonlines==4.0.0
+keras==2.15.0
+keyring==25.6.0
+kiwisolver==1.4.9
+latex2sympy2_extended==1.10.2
+lerobot==0.3.4
+Levenshtein==0.27.1
+libcst==1.8.4
+lightning-utilities==0.15.2
+markdown-it-py==4.0.0
+math-verify==0.8.0
+matplotlib==3.10.6
+mdurl==0.1.2
+mergedeep==1.3.4
+ml-dtypes==0.2.0
+ml_dtypes==0.5.3
+more-itertools==10.8.0
+mpmath==1.3.0
+msgspec==0.19.0
+multidict==6.6.4
+multiprocess==0.70.16
+mypy==1.3.0
+mypy_extensions==1.1.0
+necessary==0.4.3
+networkx==3.3
+nh3==0.3.0
+nltk==3.9.1
+numpy==1.26.4
+oauthlib==3.3.1
+omegaconf==2.3.0
+openai==1.108.0
+opencv-python-headless==4.12.0.88
+OpenEXR==3.4.0
+orderly-set==5.5.0
+orjson==3.11.3
+packaging==25.0
+pandas==2.3.2
+pathspec==0.12.1
+petname==2.6
+pfzy==0.3.4
+pillow==11.0.0
+pip==25.2
+platformdirs==4.4.0
+pluggy==1.6.0
+promise==2.3
+prompt_toolkit==3.0.52
+propcache==0.3.2
+proto-plus==1.26.1
+protobuf==4.21.12
+protobuf==6.32.1
+psutil==7.1.0
+pyarrow==21.0.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.2
+pycparser==2.23
+pydantic==2.11.9
+pydantic_core==2.33.2
+pydub==0.25.1
+Pygments==2.19.2
+pynput==1.8.1
+pyparsing==3.2.4
+pyproject_hooks==1.2.0
+pyserial==3.5
+pytest==8.4.2
+pytest-sphinx==0.6.3
+python-dateutil==2.9.0.post0
+python-Levenshtein==0.27.1
+python-multipart==0.0.20
+python-xlib==0.33
+pytorch-triton-rocm==3.4.0
+pytz==2025.2
+pyyaml-include==1.4.1
+RapidFuzz==3.14.1
+readme_renderer==44.0
+regex==2025.9.1
+requests==2.32.5
+requests-oauthlib==2.0.0
+requests-toolbelt==1.0.0
+requirements-parser==0.13.0
+rerun-sdk==0.22.1
+rfc3986==2.0.0
+rich==13.9.4
+rsa==4.9.1
+ruff==0.13.0
+s3transfer==0.14.0
+safehttpx==0.1.6
+safetensors==0.6.2
+scikit-learn==1.7.2
+scipy==1.15.3
+SecretStorage==3.4.0
+semantic-version==2.10.0
+sentencepiece==0.2.1
+sentry-sdk==2.38.0
+setuptools==78.1.1
+shellingham==1.5.4
+six==1.17.0
+smart_open==7.3.1
+smashed==0.21.5
+smmap==5.0.2
+sniffio==1.3.1
+starlette==0.48.0
+sympy==1.13.3
+tensorboard==2.15.2
+tensorboard==2.19.0
+tensorflow==2.15.0
+tensorflow-addons==0.23.0
+tensorflow-datasets==4.9.3
+tensorflow-estimator==2.15.0
+tensorflow-graphics==2021.12.3
+tensorflow-metadata==1.17.2
+threadpoolctl==3.6.0
+timm==1.0.19
+tokenizers==0.22.0
+toml==0.10.2
+tomli==2.2.1
+tomlkit==0.13.3
+torch==2.8.0+rocm6.4
+torchcodec==0.5
+torchmetrics==1.8.2
+torchvision==0.23.0+rocm6.4
+tqdm==4.67.1
+transformers==4.56.1
+trimesh==4.8.2
+trouting==0.3.3
+twine==6.2.0
+typeguard==2.13.3
+typer==0.17.4
+typing_extensions==4.15.0
+typing-inspect==0.9.0
+typing-inspection==0.4.1
+tzdata==2025.2
+urllib3==2.5.0
+uvicorn==0.35.0
+wandb==0.21.4
+wcwidth==0.2.13
+websockets==15.0.1
+wheel==0.45.1
+wrapt==1.14.2
+xxhash==3.5.0
+yarl==1.20.1
+zipp==3.23.0
+lerobot==0.3.4
+minLoRA==0.1.0
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.2
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.45.1
+zipp==3.19.2

all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/wandb-metadata.json

@@ -0,0 +1,202 @@
+{
+  "os":  "Linux-5.15.0-140-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.18",
+  "startedAt":  "2025-09-30T08:52:06.337927Z",
+  "args":  [
+    "qwen2_7b",
+    "save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1",
+    "--vision_backbone",
+    "openai",
+    "--action_head",
+    "l1_regression",
+    "--seq_len",
+    "1600",
+    "--ft_llm",
+    "--checkpoint",
+    "/vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924",
+    "--device_train_microbatch_size",
+    "16",
+    "--global_batch_size",
+    "126",
+    "--dataset",
+    "vla_dataset_realworld",
+    "--llm_learning_rate",
+    "5e-5",
+    "--wandb_entity",
+    "henryeap",
+    "--wandb_project",
+    "a1-realworld",
+    "--wandb_run_name",
+    "realworld",
+    "--save_overwrite"
+  ],
+  "program":  "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py",
+  "codePath":  "launch_scripts/train_vla.py",
+  "codePathLocal":  "launch_scripts/train_vla.py",
+  "git":  {
+    "remote":  "https://github.com/Spatialtemporal-AI/A1.git",
+    "commit":  "f2afcc15e05f491a8e50add64395fc1db0a1188d"
+  },
+  "email":  "ihenrykwok@outlook.com",
+  "root":  "/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb",
+  "host":  "auh7-1b-gpu-306",
+  "executable":  "/vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10",
+  "cpu_count":  64,
+  "cpu_count_logical":  128,
+  "gpu":  "Instinct MI210",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "470343073792",
+      "used":  "50128396288"
+    }
+  },
+  "memory":  {
+    "total":  "2434611519488"
+  },
+  "gpu_amd":  [
+    {
+      "id":  "5",
+      "uniqueId":  "0x413935505e32b8da",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "3",
+      "uniqueId":  "0x95be8fdc770fcfd7",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "1",
+      "uniqueId":  "0x27087f06439a527d",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "0",
+      "uniqueId":  "0x82728d7f9bd937e4",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "7",
+      "uniqueId":  "0xa0442ab3bdd405c1",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "2",
+      "uniqueId":  "0xaabcddaa244a3d6e",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "6",
+      "uniqueId":  "0x12140cd9e24f12e9",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "4",
+      "uniqueId":  "0x24ee801b7c402006",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "ai-04r",
+    "conf":  "/etc/slurm/slurm.conf",
+    "cpus_on_node":  "128",
+    "gpus_on_node":  "8",
+    "gtids":  "0",
+    "job_account":  "faculty-acc",
+    "job_cpus_per_node":  "128",
+    "job_end_time":  "1759481466",
+    "job_gid":  "2000",
+    "job_gpus":  "0,1,2,3,4,5,6,7",
+    "job_id":  "1970",
+    "job_name":  "mh_realworld",
+    "job_nodelist":  "auh7-1b-gpu-306",
+    "job_num_nodes":  "1",
+    "job_partition":  "faculty",
+    "job_qos":  "xdqos",
+    "job_start_time":  "1759222266",
+    "job_uid":  "2013",
+    "job_user":  "xiaodan",
+    "jobid":  "1970",
+    "localid":  "0",
+    "nnodes":  "1",
+    "nodeid":  "0",
+    "nodelist":  "auh7-1b-gpu-306",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "oom_kill_step":  "0",
+    "prio_process":  "0",
+    "procid":  "0",
+    "submit_dir":  "/vast/users/xiaodan/zhangjian/A1/launch_scripts",
+    "submit_host":  "auh-1b-cpu-login-001",
+    "task_pid":  "589646",
+    "tasks_per_node":  "1",
+    "topology_addr":  "auh7-1b-gpu-306",
+    "topology_addr_pattern":  "node"
+  },
+  "writerId":  "xw28krbghcx1p9g77lw10b5hvgwgqmco"
+}

all_l1/wandb/wandb/run-20250930_085206-50kj35c0/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":25},"_runtime":25}

all_l1/wandb/wandb/run-20250930_085206-50kj35c0/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2025-09-30T08:52:06.638339714Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpyjj93vky/port-589823.txt","pid":589823,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-30T08:52:06.641262155Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":589823}
+{"time":"2025-09-30T08:52:06.642250983Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-589823-590216-2576664124/socket","Net":"unix"}}
+{"time":"2025-09-30T08:52:06.792135331Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-30T08:52:06.805602998Z","level":"INFO","msg":"handleInformInit: received","streamId":"50kj35c0","id":"1(@)"}
+{"time":"2025-09-30T08:52:08.062408989Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"50kj35c0","id":"1(@)"}
+{"time":"2025-09-30T08:52:34.430005899Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-09-30T08:52:34.430229963Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-30T08:52:34.430227373Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-09-30T08:52:34.430267094Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-09-30T08:52:34.430322415Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-589823-590216-2576664124/socket","Net":"unix"}}
+{"time":"2025-09-30T08:52:36.423142352Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-09-30T08:52:36.423156882Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-09-30T08:52:36.423167352Z","level":"INFO","msg":"server is closed"}

all_l1/wandb/wandb/run-20250930_085206-50kj35c0/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2025-09-30T08:52:06.807490411Z","level":"INFO","msg":"stream: starting","core version":"0.21.4"}
+{"time":"2025-09-30T08:52:08.062344498Z","level":"INFO","msg":"stream: created new stream","id":"50kj35c0"}
+{"time":"2025-09-30T08:52:08.062403319Z","level":"INFO","msg":"stream: started","id":"50kj35c0"}
+{"time":"2025-09-30T08:52:08.062425769Z","level":"INFO","msg":"handler: started","stream_id":"50kj35c0"}
+{"time":"2025-09-30T08:52:08.06244382Z","level":"INFO","msg":"sender: started","stream_id":"50kj35c0"}
+{"time":"2025-09-30T08:52:08.06243294Z","level":"INFO","msg":"writer: started","stream_id":"50kj35c0"}
+{"time":"2025-09-30T08:52:34.430242003Z","level":"INFO","msg":"stream: closing","id":"50kj35c0"}
+{"time":"2025-09-30T08:52:36.099290171Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-09-30T08:52:36.421230078Z","level":"INFO","msg":"handler: closed","stream_id":"50kj35c0"}
+{"time":"2025-09-30T08:52:36.422272446Z","level":"INFO","msg":"sender: closed","stream_id":"50kj35c0"}
+{"time":"2025-09-30T08:52:36.422279596Z","level":"INFO","msg":"stream: closed","id":"50kj35c0"}

all_l1/wandb/wandb/run-20250930_085206-50kj35c0/logs/debug.log

@@ -0,0 +1 @@
+2025-09-30 08:52:34,430 INFO    wandb-AsyncioManager-main:589823 [service_client.py:_forward_responses():84] Reached EOF.

all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/config.yaml

@@ -0,0 +1,617 @@
+_wandb:
+    value:
+        cli_version: 0.21.4
+        e:
+            e39r496xjes4qj7ky2l3e9tlyr84a0v3:
+                args:
+                    - qwen2_7b
+                    - save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1
+                    - --vision_backbone
+                    - openai
+                    - --action_head
+                    - l1_regression
+                    - --seq_len
+                    - "1600"
+                    - --ft_llm
+                    - --checkpoint
+                    - /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924
+                    - --device_train_microbatch_size
+                    - "16"
+                    - --global_batch_size
+                    - "126"
+                    - --dataset
+                    - vla_dataset_realworld
+                    - --llm_learning_rate
+                    - "5e-5"
+                    - --wandb_entity
+                    - henryeap
+                    - --wandb_project
+                    - a1-realworld
+                    - --wandb_run_name
+                    - realworld
+                    - --save_overwrite
+                codePath: launch_scripts/train_vla.py
+                codePathLocal: launch_scripts/train_vla.py
+                cpu_count: 64
+                cpu_count_logical: 128
+                disk:
+                    /:
+                        total: "470343073792"
+                        used: "50128465920"
+                email: ihenrykwok@outlook.com
+                executable: /vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10
+                git:
+                    commit: f2afcc15e05f491a8e50add64395fc1db0a1188d
+                    remote: https://github.com/Spatialtemporal-AI/A1.git
+                gpu: Instinct MI210
+                gpu_amd:
+                    - id: "6"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0x12140cd9e24f12e9"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "3"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0x95be8fdc770fcfd7"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "7"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0xa0442ab3bdd405c1"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "4"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0x24ee801b7c402006"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "2"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0xaabcddaa244a3d6e"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "0"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0x82728d7f9bd937e4"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "5"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0x413935505e32b8da"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                    - id: "1"
+                      maxPower: "300.0"
+                      mclkRange: 400Mhz - 1600Mhz
+                      model: "0x740f"
+                      performanceLevel: auto
+                      sclkRange: 500Mhz - 1700Mhz
+                      series: Instinct MI210
+                      sku: D67301V
+                      uniqueId: "0x27087f06439a527d"
+                      vbiosVersion: 113-D67301V-073
+                      vendor: Advanced Micro Devices, Inc. [AMD/ATI]
+                gpu_count: 8
+                host: auh7-1b-gpu-306
+                memory:
+                    total: "2434611519488"
+                os: Linux-5.15.0-140-generic-x86_64-with-glibc2.35
+                program: /vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py
+                python: CPython 3.10.18
+                root: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb
+                slurm:
+                    cluster_name: ai-04r
+                    conf: /etc/slurm/slurm.conf
+                    cpus_on_node: "128"
+                    gpus_on_node: "8"
+                    gtids: "0"
+                    job_account: faculty-acc
+                    job_cpus_per_node: "128"
+                    job_end_time: "1759482071"
+                    job_gid: "2000"
+                    job_gpus: 0,1,2,3,4,5,6,7
+                    job_id: "1973"
+                    job_name: mh_realworld
+                    job_nodelist: auh7-1b-gpu-306
+                    job_num_nodes: "1"
+                    job_partition: faculty
+                    job_qos: xdqos
+                    job_start_time: "1759222871"
+                    job_uid: "2013"
+                    job_user: xiaodan
+                    jobid: "1973"
+                    localid: "0"
+                    nnodes: "1"
+                    nodeid: "0"
+                    nodelist: auh7-1b-gpu-306
+                    nprocs: "1"
+                    ntasks: "1"
+                    ntasks_per_node: "1"
+                    oom_kill_step: "0"
+                    prio_process: "0"
+                    procid: "0"
+                    submit_dir: /vast/users/xiaodan/zhangjian/A1/launch_scripts
+                    submit_host: auh-1b-cpu-login-001
+                    task_pid: "594412"
+                    tasks_per_node: "1"
+                    topology_addr: auh7-1b-gpu-306
+                    topology_addr_pattern: node
+                startedAt: "2025-09-30T09:01:55.101472Z"
+                writerId: e39r496xjes4qj7ky2l3e9tlyr84a0v3
+        m: []
+        python_version: 3.10.18
+        t:
+            "1":
+                - 1
+                - 2
+                - 3
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 63
+                - 71
+                - 83
+                - 95
+                - 105
+            "2":
+                - 1
+                - 2
+                - 3
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 63
+                - 71
+                - 83
+                - 95
+                - 105
+            "3":
+                - 13
+                - 15
+                - 16
+            "4": 3.10.18
+            "5": 0.21.4
+            "6": 4.56.1
+            "12": 0.21.4
+            "13": linux-x86_64
+activation_checkpointing:
+    value: whole_layer
+allow_resume:
+    value: false
+batch_divisor:
+    value: global_batch
+canceled_check_interval:
+    value: 50
+checkpoint_dir:
+    value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924
+compile:
+    value: null
+console_log_interval:
+    value: 1
+data:
+    value:
+        dataset: vla_dataset_realworld
+        drop_last: true
+        for_inference: false
+        lerobot_episode_index_end: null
+        lerobot_episode_index_start: null
+        mixture: null
+        multi_modal: torch
+        num_workers: 0
+        pad: to_max
+        persistent_workers: false
+        pin_memory: true
+        prefetch_factor: null
+        rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds
+        rlds_dataset_name: libero_4_task_suites_no_noops
+        rlds_read_threads: 8
+        rlds_shuffle_buffer_size: 100000
+        rlds_traj_threads: 8
+        root_size_mixture: null
+        seed: 95818
+        sequence_length: 1600
+        shuffle: true
+        shuffle_messages: false
+        split: train
+        timeout: 0
+        use_proprio: true
+        use_wrist_image: true
+device_eval_batch_size:
+    value: 4
+device_inf_eval_batch_size:
+    value: 16
+device_train_batch_size:
+    value: 15
+device_train_grad_accum:
+    value: 0
+device_train_microbatch_size:
+    value: 16
+dry_run:
+    value: false
+early_exit:
+    value: false
+epoch:
+    value: null
+eval_interval:
+    value: 0
+eval_on_load:
+    value: false
+eval_subset_num_batches:
+    value: -1
+evaluators:
+    value:
+        - data:
+            dataset: vla_dataset_realworld
+            drop_last: true
+            for_inference: false
+            lerobot_episode_index_end: 765
+            lerobot_episode_index_start: 353
+            mixture: null
+            multi_modal: torch
+            num_workers: 0
+            pad: to_max
+            persistent_workers: true
+            pin_memory: true
+            prefetch_factor: null
+            rlds_data_root_dir: /mnt/data/zhangjian/dataset/Simulation/datasets--openvla--modified_libero_rlds
+            rlds_dataset_name: libero_4_task_suites_no_noops
+            rlds_read_threads: 8
+            rlds_shuffle_buffer_size: 256000
+            rlds_traj_threads: 8
+            root_size_mixture: null
+            seed: null
+            sequence_length: 1600
+            shuffle: false
+            shuffle_messages: false
+            split: validation
+            timeout: 0
+            use_proprio: true
+            use_wrist_image: true
+          device_eval_batch_size: null
+          eval_name: null
+          label: val
+          max_examples: null
+          max_new_tokens: 448
+          mm_evaluator: null
+          save_dir: null
+          save_to_checkpoint_dir: false
+          skip_if_metrics_cached: true
+          subset_num_batches: 64
+extra_steps_after_cancel:
+    value: 10
+fast_forward_batches:
+    value: null
+force_save_unsharded:
+    value: false
+fsdp:
+    value:
+        hybrid_sharding_num_model_replicas: null
+        precision: float
+        sharding_strategy: FULL_SHARD
+        use_orig_params: true
+        wrapping_strategy: by_block_and_size
+ft_connector:
+    value: false
+ft_embedding:
+    value: lm_head
+ft_llm:
+    value: true
+ft_vit:
+    value: false
+fused_loss:
+    value: null
+gen1_gc_interval:
+    value: 1
+global_train_batch_size:
+    value: 126
+inf_eval_interval:
+    value: -1
+inf_eval_subset_num_batches:
+    value: -1
+inf_evaluators:
+    value: []
+initial_model_checkpoint:
+    value: /vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924
+keep_lr_on_load:
+    value: true
+load_model_config:
+    value: null
+load_path:
+    value: null
+load_path_sharded_checkpointer:
+    value: null
+lora:
+    value: false
+lora_connector:
+    value: false
+lora_llm:
+    value: false
+lora_rank:
+    value: 8
+lora_vit:
+    value: false
+max_duration:
+    value: 500000
+max_grad_norm:
+    value: 1
+max_grad_norm_ratio:
+    value: null
+model:
+    value:
+        action_dim: 7
+        action_head: l1_regression
+        action_head_dit_depth: 28
+        action_head_dit_hidden_size: 1152
+        action_head_dit_num_heads: 16
+        action_tokenizer:
+            identifier: physical-intelligence/fast
+            tokenizer_dir: null
+        action_use_left_eef: false
+        action_use_mobile_base: false
+        activation_type: swiglu
+        additional_vocab_size: 128
+        always_start_with_space: true
+        attention_dropout: 0
+        attention_layer_norm: false
+        attention_layer_norm_with_affine: true
+        attention_type: sdpa
+        bias_for_layer_norm: null
+        block_group_size: 1
+        block_type: sequential
+        clip_qkv: null
+        crop_mode: overlap-and-resize-c2
+        d_model: 3584
+        default_inference_len: 65
+        embedding_dropout: 0
+        embedding_size: 152064
+        ff_out_size: 0
+        fix_image_padding: true
+        float32_attention: true
+        head_dim: null
+        horizon: 8
+        image_feature_dropout: 0
+        image_padding_embed: pad_and_partial_pad
+        image_pooling_2d: attention_meanq
+        image_pooling_h: 2
+        image_pooling_w: 2
+        image_projector: mlp
+        include_bias: false
+        init_cutoff_factor: null
+        init_device: null
+        init_fn: normal
+        init_std: 0.02
+        initializer_range: 0.02
+        layer_norm_eps: 1e-06
+        layer_norm_type: rms
+        layer_norm_with_affine: true
+        llm_causal_attention: false
+        llm_load_path: /weka/oe-training-default/mm-olmo/pretrained_llms/qwen2-7b.pt
+        low_cpu_fsdp: true
+        max_crops: 12
+        max_position_embeddings: null
+        max_sequence_length: 4096
+        message_formatting: role
+        mlp_hidden_size: 37888
+        mlp_ratio: 4
+        moe_capacity_factor: 1.25
+        moe_dropless: true
+        moe_interleave: false
+        moe_lbl_in_fp32: false
+        moe_log_expert_assignment: false
+        moe_loss_weight: 0.1
+        moe_mlp_impl: sparse
+        moe_num_experts: 8
+        moe_shared_expert: false
+        moe_top_k: 2
+        moe_zloss_weight: null
+        multi_annotation_weighting: root_subsegments
+        n_heads: 28
+        n_kv_heads: 4
+        n_layers: 28
+        new_embedding_init_range: 0.02
+        norm_after: false
+        normalize_input_embeds: false
+        num_diffusion_inference_steps: 30
+        num_diffusion_steps: 1000
+        overlap_margins:
+            - 4
+            - 4
+        pad_tokenizer: true
+        pad_value: 0
+        precision: amp_bf16
+        prompt_type: uber_model
+        qkv_bias: true
+        residual_dropout: 0.1
+        response_residual_dropout: 0
+        rope: true
+        rope_full_precision: true
+        rope_theta: 1e+06
+        scale_logits: false
+        system_prompt_kind: demo_or_style
+        tokenizer:
+            identifier: Qwen/Qwen2-7B
+            tokenizer_dir: null
+        use_col_tokens: true
+        use_position_ids: true
+        use_proprio: true
+        vision_backbone:
+            attention_dropout: 0
+            fsdp_wrap: false
+            image_default_input_size:
+                - 336
+                - 336
+            image_dropout_rate: 0
+            image_emb_dim: 1024
+            image_head_dim: 64
+            image_mlp_activations: quick_gelu
+            image_mlp_dim: 4096
+            image_model_type: openai
+            image_norm_eps: 1e-05
+            image_num_heads: 16
+            image_num_key_value_heads: 16
+            image_num_layers: 23
+            image_num_pos: 577
+            image_patch_size: 14
+            image_pos_patch_size: 14
+            initializer_range: 0.02
+            residual_dropout: 0
+            resize_mode: default
+        vit_layers:
+            - -2
+            - -9
+        vit_load_path: /weka/oe-training-default/mm-olmo/pretrained_image_encoders/vit-l-14-336.pt
+        vocab_size: 152064
+        weight_tying: false
+multi_component_grad_norm:
+    value: true
+no_pre_train_checkpoint:
+    value: true
+optimizer:
+    value:
+        betas:
+            - 0.9
+            - 0.95
+        connector_betas:
+            - 0.9
+            - 0.95
+        connector_eps: 1e-06
+        connector_learning_rate: 0.0002
+        connector_weight_decay: 0
+        eps: 1e-05
+        learning_rate: 0.0001
+        llm_betas:
+            - 0.9
+            - 0.95
+        llm_eps: 1e-06
+        llm_learning_rate: 5e-05
+        llm_weight_decay: 0
+        metrics_log_interval: 20
+        name: adamw
+        vit_betas:
+            - 0.9
+            - 0.95
+        vit_eps: 1e-06
+        vit_learning_rate: 6e-06
+        vit_weight_decay: 0
+        weight_decay: 0.01
+precision:
+    value: amp_bf16
+python_profiling:
+    value: false
+remote_save_folder:
+    value: null
+reset_dataloader_state:
+    value: false
+reset_optimizer_state:
+    value: false
+reset_trainer_state:
+    value: false
+restore_dataloader:
+    value: true
+run_name:
+    value: realworld_20250930_090124
+save_dataloader_state:
+    value: false
+save_folder:
+    value: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1
+save_interval:
+    value: 500
+save_interval_action_head:
+    value: 500
+save_interval_ephemeral:
+    value: null
+save_interval_unsharded:
+    value: 500
+save_num_action_head_checkpoints_to_keep:
+    value: 2
+save_num_checkpoints_to_keep:
+    value: 1
+save_num_unsharded_checkpoints_to_keep:
+    value: 1
+save_overwrite:
+    value: true
+scheduler:
+    value:
+        alpha_f: 0.1
+        connector_t_warmup: 200
+        grad_clip_warmup_factor: null
+        grad_clip_warmup_steps: null
+        llm_t_warmup: 2000
+        name: multimodal
+        t_max: null
+        t_warmup: 100
+        units: steps
+        vit_t_warmup: 2000
+        warmup_min_lr: 0
+seed:
+    value: 6198
+sharded_checkpointer:
+    value: torch_legacy
+softmax_auxiliary_loss:
+    value: true
+softmax_auxiliary_loss_scale:
+    value: 0.0001
+speed_monitor:
+    value:
+        gpu_flops_available: null
+        window_size: 20
+stop_after:
+    value: null
+stop_at:
+    value: 500000
+time_limit:
+    value: null
+torch_profiling:
+    value: false
+train_exit_random_layer:
+    value: false
+use_lora:
+    value: true

all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/output.log

@@ -0,0 +1,85 @@
+wandb: Detected [openai] in use.
+wandb: Use W&B Weave for improved LLM call tracing. Install Weave with `pip install weave` then add `import weave` to the top of your script.
+wandb: For more information, check out the docs at: https://weave-docs.wandb.ai/
+09/30 [09:01:56] WARNING  | >> /vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4807: UserWarning: No       warnings.py:109
+                          device id is provided via `init_process_group` or `barrier `. Using the current device set by the user.
+                            warnings.warn(  # warn only once
+
+****** vla_cfg: {'datasets': {'rlds': {'name': None, 'path': None, 'weight': 1.0, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue', 1, 'bounds'], ['/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe', 1, 'bounds']], 'open-source-real-world': {'rlds': {'name': 'a1_real_world', 'path': None, 'weight': 8, 'action_proprio_normalization_type': 'bounds_q99', 'image_augmentation': False}, 'lerobot': [], 'agibot': {'path': '/vast/users/xiaodan/zhangjian/datasets/AgiBotWorld-Alpha', 'weight': 8, 'action_proprio_normalization_type': None}}}, 'model': {'action_head': {'action_dim': 16, 'proprio_dim': 16, 'num_actions_chunk': 8, 'action_tokens_mapping': {'left_end_effector': 8, 'right_end_effector': 8}, 'use_left_eef': True, 'use_mobile_base': False}}}
+****** Skip RLDS main; path not found: None
+****** start build LeRobot main...
+build_tokenizer, cache_dir None tokenizer_dir None
+09/30 [09:01:58] INFO     | >> Padding tokenizer with 418 tokens                                                                                                    tokenizer.py:130
+                 INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk
+****** length of the dataset: 72641
+09/30 [09:02:07] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_CleanDesk50
+****** length of the dataset: 27906
+09/30 [09:02:09] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Eraser
+****** length of the dataset: 13441
+09/30 [09:02:10] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Glue
+****** length of the dataset: 10316
+09/30 [09:02:11] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Pen
+****** length of the dataset: 17131
+09/30 [09:02:12] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_StickyNote
+****** length of the dataset: 15765
+09/30 [09:02:13] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Test_Glue
+****** length of the dataset: 90
+09/30 [09:02:14] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:435
+****** before LeRobot dataset...
+****** data_config.rlds_data_root_dir: /vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/dataset/realworld/Lerobot_Wipe
+****** length of the dataset: 18397
+****** Skip RLDS open-source-real-world; path not found: None
+****** path: /vast/users/xiaodan/zhangjian/datasets/AgiBotWorld-Alpha
+****** before AgiBotWorldAlpha dataset...
+09/30 [09:02:15] INFO     | >> Loading train dataset: vla_dataset_realworld/train                                                                                    __init__.py:485
+Traceback (most recent call last):
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/compat/_optional.py", line 135, in import_optional_dependency
+    module = importlib.import_module(name)
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/importlib/__init__.py", line 126, in import_module
+    return _bootstrap._gcd_import(name[level:], package, level)
+  File "<frozen importlib._bootstrap>", line 1050, in _gcd_import
+  File "<frozen importlib._bootstrap>", line 1027, in _find_and_load
+  File "<frozen importlib._bootstrap>", line 1004, in _find_and_load_unlocked
+ModuleNotFoundError: No module named 'openpyxl'
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py", line 397, in <module>
+    train(cfg)
+  File "/vast/users/xiaodan/zhangjian/A1/scripts/train_for_action.py", line 160, in main
+    train_loader = build_train_dataloader(cfg, device)
+  File "/vast/users/xiaodan/zhangjian/A1/olmo/data/__init__.py", line 197, in build_train_dataloader
+    return build_vla_train_dataloader(train_config, device)
+  File "/vast/users/xiaodan/zhangjian/A1/olmo/data/__init__.py", line 384, in build_vla_train_dataloader
+    ds = build_agibot_train_dataset(train_config, normalization_type, device)
+  File "/vast/users/xiaodan/zhangjian/A1/olmo/data/__init__.py", line 487, in build_agibot_train_dataset
+    dataset = AgiBotWorldAlphaDataset(
+  File "<string>", line 13, in __init__
+  File "/vast/users/xiaodan/zhangjian/A1/olmo/data/vla/agibot_dataset.py", line 145, in __post_init__
+    self._frame_ranges_map: Optional[Dict[Tuple[str, int], Tuple[int, int]]] = self._load_frame_ranges_excel()
+  File "/vast/users/xiaodan/zhangjian/A1/olmo/data/vla/agibot_dataset.py", line 411, in _load_frame_ranges_excel
+    df = pd.read_excel(excel_path)
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/io/excel/_base.py", line 495, in read_excel
+    io = ExcelFile(
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/io/excel/_base.py", line 1567, in __init__
+    self._reader = self._engines[engine](
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/io/excel/_openpyxl.py", line 552, in __init__
+    import_optional_dependency("openpyxl")
+  File "/vast/users/xiaodan/miniconda3/envs/a1/lib/python3.10/site-packages/pandas/compat/_optional.py", line 138, in import_optional_dependency
+    raise ImportError(msg)
+ImportError: Missing optional dependency 'openpyxl'.  Use pip or conda to install openpyxl.

all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/requirements.txt

@@ -0,0 +1,284 @@
+ai2-molmo==0.0.0
+astunparse==1.6.3
+flatbuffers==25.2.10
+gast==0.6.0
+google-pasta==0.2.0
+h5py==3.14.0
+libclang==18.1.1
+Markdown==3.9
+namex==0.1.0
+opt_einsum==3.4.0
+optree==0.17.0
+tensorboard-data-server==0.7.2
+tensorflow-io-gcs-filesystem==0.37.1
+termcolor==3.1.0
+Werkzeug==3.1.3
+Brotli==1.1.0
+Farama-Notifications==0.0.4
+MarkupSafe==2.1.5
+PyYAML==6.0.2
+absl-py==2.3.1
+accelerate==1.10.1
+ai2-molmo==0.0.0
+aiofiles==24.1.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.12.15
+aiosignal==1.4.0
+annotated-types==0.7.0
+antlr4-python3-runtime==4.9.3
+anyio==4.10.0
+array_record==0.8.1
+async-timeout==5.0.1
+attrs==25.3.0
+av==15.1.0
+backports.tarfile==1.2.0
+beaker-gantry==3.2.0
+beaker-py==2.5.0
+black==23.12.1
+blinker==1.9.0
+boltons==25.0.0
+boto3==1.40.33
+botocore==1.40.33
+build==1.3.0
+cached_path==1.7.3
+cached-property==2.0.1
+cachetools==5.5.2
+certifi==2025.8.3
+cffi==2.0.0
+charset-normalizer==3.4.3
+click==8.2.1
+click-help-colors==0.9.4
+click-option-group==0.5.7
+cloudpickle==3.1.1
+cmake==4.1.0
+contourpy==1.3.2
+cryptography==46.0.1
+cycler==0.12.1
+dataclass-extensions==0.2.3
+datasets==3.6.0
+decorator==5.2.1
+deepdiff==8.6.1
+diffusers==0.35.1
+dill==0.3.8
+distro==1.9.0
+dlimp==0.0.1
+dm-tree==0.1.9
+docutils==0.22.1
+draccus==0.10.0
+editdistance==0.8.1
+einops==0.8.1
+einops-exts==0.0.4
+etils==1.13.0
+evdev==1.9.2
+exceptiongroup==1.3.0
+face==24.0.0
+fastapi==0.116.2
+ffmpy==0.6.1
+fiddle==0.3.0
+filelock==3.13.1
+Flask==3.1.2
+fonttools==4.60.0
+frozenlist==1.7.0
+fsspec==2023.9.2
+ftfy==6.3.1
+gcsfs==2023.9.2
+gitdb==4.0.12
+GitPython==3.1.45
+glom==24.11.0
+google-api-core==2.25.1
+google-auth==2.40.3
+google-auth-oauthlib==1.2.2
+google-cloud-core==2.4.3
+google-cloud-storage==2.19.0
+google-crc32c==1.7.1
+google-resumable-media==2.7.2
+googleapis-common-protos==1.70.0
+gradio==5.46.0
+gradio_client==1.13.0
+graphviz==0.21
+groovy==0.1.2
+grpcio==1.75.0
+gymnasium==0.29.1
+h11==0.16.0
+hf_transfer==0.1.9
+hf-xet==1.1.10
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==0.35.0
+id==1.5.0
+idna==3.10
+imageio==2.37.0
+imageio-ffmpeg==0.6.0
+importlib_metadata==8.7.0
+importlib_resources==6.5.2
+iniconfig==2.1.0
+inquirerpy==0.3.4
+isort==5.12.0
+itsdangerous==2.2.0
+jaraco.classes==3.4.0
+jaraco.context==6.0.1
+jaraco.functools==4.3.0
+jeepney==0.9.0
+Jinja2==3.1.4
+jiter==0.11.0
+jmespath==1.0.1
+joblib==1.5.2
+jsonlines==4.0.0
+keras==2.15.0
+keyring==25.6.0
+kiwisolver==1.4.9
+latex2sympy2_extended==1.10.2
+lerobot==0.3.4
+Levenshtein==0.27.1
+libcst==1.8.4
+lightning-utilities==0.15.2
+markdown-it-py==4.0.0
+math-verify==0.8.0
+matplotlib==3.10.6
+mdurl==0.1.2
+mergedeep==1.3.4
+ml-dtypes==0.2.0
+ml_dtypes==0.5.3
+more-itertools==10.8.0
+mpmath==1.3.0
+msgspec==0.19.0
+multidict==6.6.4
+multiprocess==0.70.16
+mypy==1.3.0
+mypy_extensions==1.1.0
+necessary==0.4.3
+networkx==3.3
+nh3==0.3.0
+nltk==3.9.1
+numpy==1.26.4
+oauthlib==3.3.1
+omegaconf==2.3.0
+openai==1.108.0
+opencv-python-headless==4.12.0.88
+OpenEXR==3.4.0
+orderly-set==5.5.0
+orjson==3.11.3
+packaging==25.0
+pandas==2.3.2
+pathspec==0.12.1
+petname==2.6
+pfzy==0.3.4
+pillow==11.0.0
+pip==25.2
+platformdirs==4.4.0
+pluggy==1.6.0
+promise==2.3
+prompt_toolkit==3.0.52
+propcache==0.3.2
+proto-plus==1.26.1
+protobuf==4.21.12
+protobuf==6.32.1
+psutil==7.1.0
+pyarrow==21.0.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.2
+pycparser==2.23
+pydantic==2.11.9
+pydantic_core==2.33.2
+pydub==0.25.1
+Pygments==2.19.2
+pynput==1.8.1
+pyparsing==3.2.4
+pyproject_hooks==1.2.0
+pyserial==3.5
+pytest==8.4.2
+pytest-sphinx==0.6.3
+python-dateutil==2.9.0.post0
+python-Levenshtein==0.27.1
+python-multipart==0.0.20
+python-xlib==0.33
+pytorch-triton-rocm==3.4.0
+pytz==2025.2
+pyyaml-include==1.4.1
+RapidFuzz==3.14.1
+readme_renderer==44.0
+regex==2025.9.1
+requests==2.32.5
+requests-oauthlib==2.0.0
+requests-toolbelt==1.0.0
+requirements-parser==0.13.0
+rerun-sdk==0.22.1
+rfc3986==2.0.0
+rich==13.9.4
+rsa==4.9.1
+ruff==0.13.0
+s3transfer==0.14.0
+safehttpx==0.1.6
+safetensors==0.6.2
+scikit-learn==1.7.2
+scipy==1.15.3
+SecretStorage==3.4.0
+semantic-version==2.10.0
+sentencepiece==0.2.1
+sentry-sdk==2.38.0
+setuptools==78.1.1
+shellingham==1.5.4
+six==1.17.0
+smart_open==7.3.1
+smashed==0.21.5
+smmap==5.0.2
+sniffio==1.3.1
+starlette==0.48.0
+sympy==1.13.3
+tensorboard==2.15.2
+tensorboard==2.19.0
+tensorflow==2.15.0
+tensorflow-addons==0.23.0
+tensorflow-datasets==4.9.3
+tensorflow-estimator==2.15.0
+tensorflow-graphics==2021.12.3
+tensorflow-metadata==1.17.2
+threadpoolctl==3.6.0
+timm==1.0.19
+tokenizers==0.22.0
+toml==0.10.2
+tomli==2.2.1
+tomlkit==0.13.3
+torch==2.8.0+rocm6.4
+torchcodec==0.5
+torchmetrics==1.8.2
+torchvision==0.23.0+rocm6.4
+tqdm==4.67.1
+transformers==4.56.1
+trimesh==4.8.2
+trouting==0.3.3
+twine==6.2.0
+typeguard==2.13.3
+typer==0.17.4
+typing_extensions==4.15.0
+typing-inspect==0.9.0
+typing-inspection==0.4.1
+tzdata==2025.2
+urllib3==2.5.0
+uvicorn==0.35.0
+wandb==0.21.4
+wcwidth==0.2.13
+websockets==15.0.1
+wheel==0.45.1
+wrapt==1.14.2
+xxhash==3.5.0
+yarl==1.20.1
+zipp==3.23.0
+lerobot==0.3.4
+minLoRA==0.1.0
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.2
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.45.1
+zipp==3.19.2

all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/wandb-metadata.json

@@ -0,0 +1,202 @@
+{
+  "os":  "Linux-5.15.0-140-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.10.18",
+  "startedAt":  "2025-09-30T09:01:55.101472Z",
+  "args":  [
+    "qwen2_7b",
+    "save_folder=/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1",
+    "--vision_backbone",
+    "openai",
+    "--action_head",
+    "l1_regression",
+    "--seq_len",
+    "1600",
+    "--ft_llm",
+    "--checkpoint",
+    "/vast/users/xiaodan/zhangjian/molmo_data/Molmo-7B-D-0924",
+    "--device_train_microbatch_size",
+    "16",
+    "--global_batch_size",
+    "126",
+    "--dataset",
+    "vla_dataset_realworld",
+    "--llm_learning_rate",
+    "5e-5",
+    "--wandb_entity",
+    "henryeap",
+    "--wandb_project",
+    "a1-realworld",
+    "--wandb_run_name",
+    "realworld",
+    "--save_overwrite"
+  ],
+  "program":  "/vast/users/xiaodan/zhangjian/A1/launch_scripts/train_vla.py",
+  "codePath":  "launch_scripts/train_vla.py",
+  "codePathLocal":  "launch_scripts/train_vla.py",
+  "git":  {
+    "remote":  "https://github.com/Spatialtemporal-AI/A1.git",
+    "commit":  "f2afcc15e05f491a8e50add64395fc1db0a1188d"
+  },
+  "email":  "ihenrykwok@outlook.com",
+  "root":  "/vast/users/xiaodan/workspace/minghao.guo/warehouse_a1/ckpt/all_l1/wandb",
+  "host":  "auh7-1b-gpu-306",
+  "executable":  "/vast/users/xiaodan/miniconda3/envs/a1/bin/python3.10",
+  "cpu_count":  64,
+  "cpu_count_logical":  128,
+  "gpu":  "Instinct MI210",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "470343073792",
+      "used":  "50128465920"
+    }
+  },
+  "memory":  {
+    "total":  "2434611519488"
+  },
+  "gpu_amd":  [
+    {
+      "id":  "6",
+      "uniqueId":  "0x12140cd9e24f12e9",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "3",
+      "uniqueId":  "0x95be8fdc770fcfd7",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "7",
+      "uniqueId":  "0xa0442ab3bdd405c1",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "4",
+      "uniqueId":  "0x24ee801b7c402006",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "2",
+      "uniqueId":  "0xaabcddaa244a3d6e",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "0",
+      "uniqueId":  "0x82728d7f9bd937e4",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "5",
+      "uniqueId":  "0x413935505e32b8da",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    },
+    {
+      "id":  "1",
+      "uniqueId":  "0x27087f06439a527d",
+      "vbiosVersion":  "113-D67301V-073",
+      "performanceLevel":  "auto",
+      "maxPower":  "300.0",
+      "series":  "Instinct MI210",
+      "model":  "0x740f",
+      "vendor":  "Advanced Micro Devices, Inc. [AMD/ATI]",
+      "sku":  "D67301V",
+      "sclkRange":  "500Mhz - 1700Mhz",
+      "mclkRange":  "400Mhz - 1600Mhz"
+    }
+  ],
+  "slurm":  {
+    "cluster_name":  "ai-04r",
+    "conf":  "/etc/slurm/slurm.conf",
+    "cpus_on_node":  "128",
+    "gpus_on_node":  "8",
+    "gtids":  "0",
+    "job_account":  "faculty-acc",
+    "job_cpus_per_node":  "128",
+    "job_end_time":  "1759482071",
+    "job_gid":  "2000",
+    "job_gpus":  "0,1,2,3,4,5,6,7",
+    "job_id":  "1973",
+    "job_name":  "mh_realworld",
+    "job_nodelist":  "auh7-1b-gpu-306",
+    "job_num_nodes":  "1",
+    "job_partition":  "faculty",
+    "job_qos":  "xdqos",
+    "job_start_time":  "1759222871",
+    "job_uid":  "2013",
+    "job_user":  "xiaodan",
+    "jobid":  "1973",
+    "localid":  "0",
+    "nnodes":  "1",
+    "nodeid":  "0",
+    "nodelist":  "auh7-1b-gpu-306",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "oom_kill_step":  "0",
+    "prio_process":  "0",
+    "procid":  "0",
+    "submit_dir":  "/vast/users/xiaodan/zhangjian/A1/launch_scripts",
+    "submit_host":  "auh-1b-cpu-login-001",
+    "task_pid":  "594412",
+    "tasks_per_node":  "1",
+    "topology_addr":  "auh7-1b-gpu-306",
+    "topology_addr_pattern":  "node"
+  },
+  "writerId":  "e39r496xjes4qj7ky2l3e9tlyr84a0v3"
+}

all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_runtime":19,"_wandb":{"runtime":19}}

all_l1/wandb/wandb/run-20250930_090155-w2yi62pb/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2025-09-30T09:01:55.162904331Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmprwjho3ja/port-594589.txt","pid":594589,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-30T09:01:55.164089452Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":594589}
+{"time":"2025-09-30T09:01:55.164075022Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-594589-594975-1877083663/socket","Net":"unix"}}
+{"time":"2025-09-30T09:01:55.335963017Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-30T09:01:55.342714526Z","level":"INFO","msg":"handleInformInit: received","streamId":"w2yi62pb","id":"1(@)"}
+{"time":"2025-09-30T09:01:56.270439585Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"w2yi62pb","id":"1(@)"}
+{"time":"2025-09-30T09:02:15.777414547Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-09-30T09:02:15.777645561Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-09-30T09:02:15.777654891Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-30T09:02:15.777680572Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-09-30T09:02:15.778959994Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-594589-594975-1877083663/socket","Net":"unix"}}
+{"time":"2025-09-30T09:02:17.890949248Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-09-30T09:02:17.890967089Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-09-30T09:02:17.890978859Z","level":"INFO","msg":"server is closed"}