diff --git a/.gitattributes b/.gitattributes index 3b9bb08db7a813330e3a1bb4c39356c8ae933584..cee7af04eb7497b49ee5858f4c50b25204420970 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2088,3 +2088,6 @@ act/act-g1-real/act-pick-bottle-and-turn-and-pour-into-cup.g1-real.cosin.lr1.0e- act/act-g1-real/act-pull-the-tray-out-of-chips-can-and-throw-the-can-into-trash-bin.g1-real.cosin.lr1.0e-04.b64.gpus2.2601231421/wandb/run-20260123_142144-7kwywvv2/run-7kwywvv2.wandb filter=lfs diff=lfs merge=lfs -text act/act-g1-real/act-rotate-to-pour-ham-into-plate-and-push-the-cart-forward.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240510/wandb/run-20260124_051018-nfks4wzd/run-nfks4wzd.wandb filter=lfs diff=lfs merge=lfs -text act/act-g1-real/act-spray-the-bowl-and-wipe-it-and-stack-it-up.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240214/wandb/run-20260124_021447-qcb38sob/run-qcb38sob.wandb filter=lfs diff=lfs merge=lfs -text +act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/run-e5jhds0w.wandb filter=lfs diff=lfs merge=lfs -text +act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/run-fk8kij49.wandb filter=lfs diff=lfs merge=lfs -text +act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/run-1i9wg620.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/argv.txt b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/argv.txt new file mode 100644 index 0000000000000000000000000000000000000000..3ed9776b317c973b2b0d68528f7a92f7949eca1d --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/argv.txt @@ -0,0 +1,44 @@ +scripts/train.py +data:gr00t +data.transform.repack:gr00t-we +data.transform.action-state:maxmin +data.transform.model:act +model:act +--seed=2026 +--exp=act-push-cart-grasp-and-place-grapes-on-plate +--train.name=act-g1-real +--log.report-to=wandb +--wandb.entity=jliu530-soochow-university +--train.data_parallel=ddp +--train.mixed_precision=bf16 +--train.train-batch-size=32 +--train.gradient_accumulation_steps=1 +--train.validation_steps=500 +--train.val_num_batches=20 +--train.max-training-steps=42000 +--train.learning-rate=1e-4 +--train.max-grad-norm=1.0 +--train.lr_scheduler_kwargs.weight_decay=1e-6 +--train.lr_scheduler_kwargs.betas 0.95 0.999 +--train.lr_scheduler_type=cosine +--train.warmup-steps=1000 +--train.warmup-ratio=None +--train.checkpointing-steps=5000 +--data.root_dir=/hfm/data/real_teleop_g1/lerobot +--data.dataset_paths=Push_cart_grasp_and_place_grapes_on_plate +--data.data_config=we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig +--data.transform.repack.pad-action-dim=36 +--data.transform.repack.pad-state-dim=32 +--data.transform.action-state.stat-path=dataset_statistics/g1-stats-Push_cart_grasp_and_place_grapes_on_plate.json +--data.transform.action-state.stat-action-key=action +--data.transform.action-state.stat-state-key=states +--data.transform.action-state.normalize-state +--data.transform.action-state.action-norm-type=bounds +--data.transform.model.img-aug +--data.action-chunk-size=100 +--model.chunk-size=100 +--model.n-action-steps=100 +--model.action-dim=36 +--model.state-dim=32 +--model.use-vae +--model.kl-weight=10.0 diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_40000/model.safetensors b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_40000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..13d98db2c75e731a452f4c3dc00ab732e726536f --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_40000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2ff781e36bdf19304a69bdf22045f12e1a8e5d7cd9f8e493668adf01ce127d9 +size 206929288 diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_40000/optimizer.bin b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_40000/optimizer.bin new file mode 100644 index 0000000000000000000000000000000000000000..48cdc063a08fdb36239c4d35372ca9d96a9cac62 --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_40000/optimizer.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f6680180bf962dafe624b286e91f3ae91365679db9feb18b9de088c90197029 +size 413365323 diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_40000/random_states_0.pkl b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_40000/random_states_0.pkl new file mode 100644 index 0000000000000000000000000000000000000000..ac0b3048a6a999f13a98916f8ff17cd8e4ab468b --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_40000/random_states_0.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05a0e8ad201ae0ac8099501ba357f55b0b4cdb7fb78a60025ccd5242ba986d69 +size 15017 diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_40000/random_states_1.pkl b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_40000/random_states_1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..41a74ce7a682376b3e158e8b7842aaffa3cac0bd --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_40000/random_states_1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a7fa639504fcdf108f13cd9c981376c5035fd55546e113fe83c26db9fc87577 +size 15017 diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_40000/scheduler.bin b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_40000/scheduler.bin new file mode 100644 index 0000000000000000000000000000000000000000..b8ad95dbb23291fa1ddb3b17e1a9e1abf4dcd17a --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_40000/scheduler.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eb924d1839878754c8fee832541dbd45925c8f97a4199d20416424aca1da736 +size 1465 diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_41999/model.safetensors b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_41999/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..814f26d388266c8f3ad87467bc25ee3b0577de41 --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_41999/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55ba3430c54a0b9b532441ba42fd95797fec6ad0f5c5e9f86d47be386fd7c98f +size 206929288 diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_41999/optimizer.bin b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_41999/optimizer.bin new file mode 100644 index 0000000000000000000000000000000000000000..e998786db8ea6d14af11ef4e39fc39401cfd646a --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_41999/optimizer.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b54152a2eb4fee0f21abe334a115d85fecdf1298bc5573ab73c5a1ba72855d7b +size 413365323 diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_41999/random_states_0.pkl b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_41999/random_states_0.pkl new file mode 100644 index 0000000000000000000000000000000000000000..96d5fcd83eff84d98246e7f71e40a64d648f8012 --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_41999/random_states_0.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22aa2ee85fb66be9843ccbc11f659a521a0de164d5e50e42ed379f4351f82093 +size 15017 diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_41999/random_states_1.pkl b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_41999/random_states_1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..66a06b91677b87ca27cc43ce48cecd9ee8945ce3 --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_41999/random_states_1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884ab7db353c3691df248d0bc1fdbc547574afc9125ddddb95ceaf29f8afde21 +size 15017 diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_41999/scheduler.bin b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_41999/scheduler.bin new file mode 100644 index 0000000000000000000000000000000000000000..a1771d0fdb797a3bc3b6512b03e0142fc5e9defb --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_41999/scheduler.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e535d787f14eca99ed019dbfc496ad1979b2294c53840353f973c96848844e1e +size 1465 diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/dataset_statistics.json b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/dataset_statistics.json new file mode 100644 index 0000000000000000000000000000000000000000..4dc46432389f983e88dc2503950f409e302b2fcd --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/dataset_statistics.json @@ -0,0 +1,558 @@ +{ + "states": { + "mean": [ + -0.639490008354187, + 0.5598652958869934, + 0.7508902549743652, + -0.25175440311431885, + -0.3999335765838623, + -0.337761253118515, + -0.5562001466751099, + -0.47207823395729065, + -0.5420713424682617, + -0.7273601293563843, + 0.4445062279701233, + 0.6221323013305664, + 0.44212180376052856, + 0.6947574615478516, + 0.12124833464622498, + 0.2101212739944458, + 0.23638580739498138, + 0.050116632133722305, + -0.16650019586086273, + -0.36662817001342773, + -0.23764868080615997, + -0.5449934005737305, + -0.19021503627300262, + 0.05621454492211342, + 0.5941070914268494, + -0.23858405649662018, + -0.23731473088264465, + 0.04377191141247749, + -0.003370167687535286, + 0.03685297444462776, + -0.07056427001953125, + 0.7320583462715149 + ], + "std": [ + 0.21672311425209045, + 0.13823239505290985, + 0.15728947520256042, + 0.2361675202846527, + 0.3464930057525635, + 0.3317983150482178, + 0.46160775423049927, + 0.2563662827014923, + 0.10304629057645798, + 0.14716456830501556, + 0.3559660017490387, + 0.5002177357673645, + 0.33975711464881897, + 0.5306226015090942, + 0.36043867468833923, + 0.12918461859226227, + 0.15602730214595795, + 0.38895076513290405, + 0.2221171259880066, + 0.32731714844703674, + 0.20227091014385223, + 0.48788419365882874, + 0.1574118733406067, + 0.28795915842056274, + 0.4946692883968353, + 0.3728179633617401, + 0.3018190264701843, + 0.27890101075172424, + 0.048964355140924454, + 0.09605813026428223, + 0.24454477429389954, + 0.01829754002392292 + ], + "min": [ + -1.0231640338897705, + -0.0695418044924736, + 0.24746619164943695, + -1.4132853746414185, + -1.8126834630966187, + -1.3577090501785278, + -1.7228399515151978, + -1.0409873723983765, + -0.9193933606147766, + -1.5848426818847656, + -0.07512053102254868, + 0.023398026823997498, + -0.05005142092704773, + 0.018349293619394302, + -0.896072506904602, + -0.10484998673200607, + -0.3023500144481659, + -1.0332319736480713, + -1.0468820333480835, + -1.5026646852493286, + -0.9785187244415283, + -1.8509875535964966, + -1.010833501815796, + -0.6269427537918091, + -1.0290495157241821, + -1.522499918937683, + -1.3486835956573486, + -1.1306359767913818, + -0.17483314871788025, + -0.3118287920951843, + -1.1479005813598633, + 0.6552329659461975 + ], + "max": [ + 0.0012606654781848192, + 0.9533419013023376, + 1.441234827041626, + 0.10255199670791626, + -0.021535230800509453, + 0.2322024405002594, + -0.022155510261654854, + 0.18327920138835907, + 0.06926733255386353, + -0.24854479730129242, + 1.4151289463043213, + 1.7675095796585083, + 1.333852767944336, + 1.720163106918335, + 1.2428480386734009, + 0.7350764274597168, + 1.2933615446090698, + 1.1574125289916992, + 0.929772138595581, + 0.8751207590103149, + 0.44843629002571106, + 0.7306901812553406, + 0.12389291822910309, + 0.8440968990325928, + 1.3799595832824707, + 1.1950310468673706, + 1.071639060974121, + 1.2805979251861572, + 0.21328531205654144, + 0.3196083903312683, + 0.48254090547561646, + 0.801028311252594 + ], + "q01": [ + -1.0003288984298706, + 0.24154996499419212, + 0.5464459806680679, + -1.138906866312027, + -1.7269256114959717, + -1.0811239778995514, + -1.6471525728702545, + -0.9866711795330048, + -0.7845048010349274, + -1.2068439722061157, + 0.003995673265308142, + 0.03462587296962738, + -0.002932884090114385, + 0.01911628618836403, + -0.5482873171567917, + -0.061538998037576675, + -0.12132530100643635, + -0.9794377833604813, + -0.793193906545639, + -1.2216891944408417, + -0.7400441318750381, + -1.5305383801460266, + -0.608064591884613, + -0.4274803102016449, + -0.5353982746601105, + -1.2221932709217072, + -0.953227236866951, + -0.6563884764909744, + -0.12044951878488064, + -0.19737506285309792, + -0.8513251841068268, + 0.6925749778747559 + ], + "q99": [ + -0.12671788036823273, + 0.8919035047292709, + 1.2140001058578491, + 0.028299483936280012, + -0.031822274439036846, + 0.22735120356082916, + -0.0303517933934927, + -0.01155597297474742, + -0.28130418062210083, + -0.5209185928106308, + 1.15076744556427, + 1.722438097000122, + 0.9948239773511887, + 1.7053217887878418, + 1.0441855192184448, + 0.5283785164356232, + 0.7163720279932022, + 0.9164786338806152, + 0.2938561961054802, + 0.5599221885204315, + 0.2007444053888321, + 0.43314584344625473, + 0.09484615363180637, + 0.7030006349086761, + 1.3659979104995728, + 0.6106891632080078, + 0.5404655784368515, + 0.8005002290010452, + 0.12078485824167728, + 0.248464647680521, + 0.28620126843452454, + 0.7687181681394577 + ] + }, + "action": { + "mean": [ + -0.6448636651039124, + 0.5748050212860107, + 0.7545458078384399, + -0.2644195258617401, + -0.4068398177623749, + -0.36320579051971436, + -0.5641849637031555, + -0.46346697211265564, + -0.5778801441192627, + -0.7395340204238892, + 0.4690419137477875, + 0.6349577307701111, + 0.46419069170951843, + 0.7127190828323364, + 0.1161443442106247, + 0.2099175900220871, + 0.2267233282327652, + 0.04004364460706711, + -0.1716204285621643, + -0.37982314825057983, + -0.25060322880744934, + -0.5431874990463257, + -0.1895122081041336, + 0.05029662325978279, + 0.5510207414627075, + -0.22751697897911072, + -0.24778352677822113, + 0.05329873412847519, + -0.0033551636151969433, + 0.03679672256112099, + -0.07058019936084747, + 0.7320395708084106, + 0.028576020151376724, + 0.0, + 0.0, + 0.0 + ], + "std": [ + 0.2308892458677292, + 0.14882966876029968, + 0.1803825944662094, + 0.2700367271900177, + 0.35940539836883545, + 0.3603762984275818, + 0.48626625537872314, + 0.280373215675354, + 0.13059958815574646, + 0.17301921546459198, + 0.39215287566185, + 0.5196966528892517, + 0.37940067052841187, + 0.5541192889213562, + 0.36498698592185974, + 0.13551504909992218, + 0.16307957470417023, + 0.3947526216506958, + 0.2238609492778778, + 0.32385894656181335, + 0.20268261432647705, + 0.49303504824638367, + 0.17156723141670227, + 0.29902419447898865, + 0.5568225979804993, + 0.39549964666366577, + 0.33107200264930725, + 0.28752776980400085, + 0.04897480458021164, + 0.09612710028886795, + 0.24457372725009918, + 0.018295763060450554, + 0.09581305831670761, + 0.0, + 0.0, + 0.0 + ], + "min": [ + -1.0481975078582764, + -0.19984650611877441, + 0.4533447325229645, + -1.5717962980270386, + -1.7463293075561523, + -1.5717962980270386, + -1.7463293075561523, + -1.0481975078582764, + -0.9210000038146973, + -1.6845287084579468, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -0.9514403939247131, + -0.3264637887477875, + -0.4328526556491852, + -1.0471998453140259, + -1.0728868246078491, + -1.6144294738769531, + -0.9963729381561279, + -1.9203201532363892, + -1.1714298725128174, + -0.7144220471382141, + -1.0471998453140259, + -1.5777469873428345, + -1.614428997039795, + -1.1877036094665527, + -0.17483314871788025, + -0.3118287920951843, + -1.1479005813598633, + 0.6552329659461975, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "max": [ + 0.09929836541414261, + 0.9210000038146973, + 1.630710482597351, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0010000000474974513, + 0.30392226576805115, + 0.08567330241203308, + -0.40413898229599, + 1.517154335975647, + 1.7463293075561523, + 1.5717962980270386, + 1.7463293075561523, + 1.2656242847442627, + 0.8192498087882996, + 1.313675045967102, + 1.2117120027542114, + 0.9534142017364502, + 0.9147428870201111, + 0.5678492188453674, + 0.7814759612083435, + 0.3561345934867859, + 0.9256001710891724, + 1.4109472036361694, + 1.4722950458526611, + 1.1215534210205078, + 1.3663387298583984, + 0.21328531205654144, + 0.3196083903312683, + 0.48254090547561646, + 0.801028311252594, + 0.3499999940395355, + 0.0, + 0.0, + 0.0 + ], + "q01": [ + -1.0481975078582764, + 0.20623961091041565, + 0.5448732972145081, + -1.178023099899292, + -1.7463293075561523, + -1.1693772077560425, + -1.7463293075561523, + -1.04752117395401, + -0.8733360916376114, + -1.2716453075408936, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -0.5568297356367111, + -0.13642824068665504, + -0.14339743554592133, + -0.9750052690505981, + -0.8009233474731445, + -1.208129346370697, + -0.7669829726219177, + -1.5875315070152283, + -0.6462412625551224, + -0.4931211695075035, + -0.7749188095331192, + -1.2683663070201874, + -1.0628002285957336, + -0.6928522735834122, + -0.12044951878488064, + -0.19743520393967628, + -0.8513251841068268, + 0.6925749778747559, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "q99": [ + -0.14091050624847412, + 0.9210000038146973, + 1.2425841093063354, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0028508438263088465, + -0.25891827791929245, + -0.5224902182817459, + 1.2241601049900055, + 1.7463293075561523, + 1.071914792060852, + 1.7463293075561523, + 1.0505727529525757, + 0.5396184623241425, + 0.711058720946312, + 0.9422783255577087, + 0.3070615381002426, + 0.5665820986032486, + 0.20188817009329796, + 0.44927291572093964, + 0.1739325113594532, + 0.713741660118103, + 1.3824210166931152, + 0.7019023001194, + 0.5921306908130646, + 0.822442814707756, + 0.12078485824167728, + 0.248464647680521, + 0.28622647374868393, + 0.7687181681394577, + 0.3499999940395355, + 0.0, + 0.0, + 0.0 + ] + }, + "timestamp": { + "mean": [ + 18.629594802856445 + ], + "std": [ + 11.722174644470215 + ], + "min": [ + 0.0 + ], + "max": [ + 78.06666564941406 + ], + "q01": [ + 0.3333333432674408 + ], + "q99": [ + 49.10833263397217 + ] + }, + "frame_index": { + "mean": [ + 558.8878173828125 + ], + "std": [ + 351.66522216796875 + ], + "min": [ + 0.0 + ], + "max": [ + 2342.0 + ], + "q01": [ + 10.0 + ], + "q99": [ + 1473.25 + ] + }, + "episode_index": { + "mean": [ + 40.089988708496094 + ], + "std": [ + 22.798812866210938 + ], + "min": [ + 0.0 + ], + "max": [ + 79.0 + ], + "q01": [ + 0.0 + ], + "q99": [ + 79.0 + ] + }, + "index": { + "mean": [ + 558.8878173828125 + ], + "std": [ + 351.66522216796875 + ], + "min": [ + 0.0 + ], + "max": [ + 2342.0 + ], + "q01": [ + 10.0 + ], + "q99": [ + 1473.25 + ] + }, + "task_index": { + "mean": [ + 0.0 + ], + "std": [ + 0.0 + ], + "min": [ + 0.0 + ], + "max": [ + 0.0 + ], + "q01": [ + 0.0 + ], + "q99": [ + 0.0 + ] + }, + "next.done": { + "mean": [ + 0.0009208527044393122 + ], + "std": [ + 0.030331579968333244 + ], + "min": [ + 0.0 + ], + "max": [ + 1.0 + ], + "q01": [ + 0.0 + ], + "q99": [ + 0.0 + ] + } +} \ No newline at end of file diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/envs.txt b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/envs.txt new file mode 100644 index 0000000000000000000000000000000000000000..a470889be27478fc4dbc10c9fd51d480987bf194 --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/envs.txt @@ -0,0 +1,17 @@ +OMP_NUM_THREADS=32 +HF_HOME=/hfm/cache +TORCH_HOME=/hfm/cache +HF_TOKEN=hf_...adRb +HF_LEROBOT_HOME=Not Set +WE_HOME=Not Set +DATA_HOME=Not Set +UV_CACHE_DIR=Not Set +WANDB_API_KEY=Not Set +PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python +CUDA_VISIBLE_DEVICES=6,7 +WORLD_SIZE=2 +LOCAL_WORLD_SIZE=2 +RANK=0 +LOCAL_RANK=0 +MASTER_ADDR=nebula99 +MASTER_PORT=33685 diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/run_config.json b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/run_config.json new file mode 100644 index 0000000000000000000000000000000000000000..36616369f27a4b9b4269adbd6d6873c8ac962700 --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/run_config.json @@ -0,0 +1,310 @@ +{ + "exp": "act-push-cart-grasp-and-place-grapes-on-plate", + "seed": 2026, + "auto_tag_run": false, + "eval": false, + "debug": false, + "timestamp": "2601240341", + "log": { + "logging_dir": "logs", + "report_to": "wandb", + "log_freq": 100 + }, + "wandb": { + "project": "hfm", + "entity": "jliu530-soochow-university", + "group": "act-g1-real", + "id": "e5jhds0w", + "name": "act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341", + "resume": "allow" + }, + "train": { + "num_workers": 8, + "overfit_single_batch": false, + "name": "act-g1-real", + "resume_from_checkpoint": null, + "skip_resumed_steps": false, + "hf_token": ".hf_token", + "lora": false, + "output_dir": ".runs", + "gradient_accumulation_steps": 1, + "mixed_precision": "bf16", + "max_grad_norm": 1.0, + "train_batch_size": 32, + "val_batch_size": 16, + "val_num_batches": 20, + "checkpointing_steps": 5000, + "max_checkpoints_to_keep": null, + "validation_steps": 500, + "learning_rate": 0.0001, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": { + "betas": [ + 0.95, + 0.999 + ], + "weight_decay": 1e-6, + "eps": 1e-8 + }, + "scheduler_specific_kwargs": {}, + "data_parallel": "ddp", + "sharding_strategy": "full-shard", + "deepspeed_config": "/home/jliu/we_learn/src/InternVLA/config/deepseeds/zero3.json", + "enable_gradient_checkpointing": true, + "enable_mixed_precision_training": true, + "reduce_in_full_precision": true, + "max_training_steps": 42000, + "num_train_epochs": null, + "warmup_steps": 1000, + "warmup_ratio": null + }, + "data": { + "transform": { + "repack": { + "dataset_name": "we-g1-teleop", + "conditions": [], + "image_keys": [ + "video" + ], + "state_key": "state", + "action_key": "action", + "instruction_key": "annotation.human.task_description", + "pad_action_dim": 36, + "pad_state_dim": 32 + }, + "action_state": { + "stat_path": "dataset_statistics/g1-stats-Push_cart_grasp_and_place_grapes_on_plate.json", + "action_norm_type": "bounds", + "stat_action_key": "action", + "stat_state_key": "states", + "use_norm_mask": false, + "action_norm_masks": [ + true, + true, + true, + true, + true, + true, + false + ], + "action_min": [ + -1.0481975078582764, + -0.19984650611877441, + 0.4533447325229645, + -1.5717962980270386, + -1.7463293075561523, + -1.5717962980270386, + -1.7463293075561523, + -1.0481975078582764, + -0.9210000038146973, + -1.6845287084579468, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -0.9514403939247131, + -0.3264637887477875, + -0.4328526556491852, + -1.0471998453140259, + -1.0728868246078491, + -1.6144294738769531, + -0.9963729381561279, + -1.9203201532363892, + -1.1714298725128174, + -0.7144220471382141, + -1.0471998453140259, + -1.5777469873428345, + -1.614428997039795, + -1.1877036094665527, + -0.17483314871788025, + -0.3118287920951843, + -1.1479005813598633, + 0.6552329659461975, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "action_max": [ + 0.09929836541414261, + 0.9210000038146973, + 1.630710482597351, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0010000000474974513, + 0.30392226576805115, + 0.08567330241203308, + -0.40413898229599, + 1.517154335975647, + 1.7463293075561523, + 1.5717962980270386, + 1.7463293075561523, + 1.2656242847442627, + 0.8192498087882996, + 1.313675045967102, + 1.2117120027542114, + 0.9534142017364502, + 0.9147428870201111, + 0.5678492188453674, + 0.7814759612083435, + 0.3561345934867859, + 0.9256001710891724, + 1.4109472036361694, + 1.4722950458526611, + 1.1215534210205078, + 1.3663387298583984, + 0.21328531205654144, + 0.3196083903312683, + 0.48254090547561646, + 0.801028311252594, + 0.3499999940395355, + 0.0, + 0.0, + 0.0 + ], + "state_min": [ + -1.0231640338897705, + -0.0695418044924736, + 0.24746619164943695, + -1.4132853746414185, + -1.8126834630966187, + -1.3577090501785278, + -1.7228399515151978, + -1.0409873723983765, + -0.9193933606147766, + -1.5848426818847656, + -0.07512053102254868, + 0.023398026823997498, + -0.05005142092704773, + 0.018349293619394302, + -0.896072506904602, + -0.10484998673200607, + -0.3023500144481659, + -1.0332319736480713, + -1.0468820333480835, + -1.5026646852493286, + -0.9785187244415283, + -1.8509875535964966, + -1.010833501815796, + -0.6269427537918091, + -1.0290495157241821, + -1.522499918937683, + -1.3486835956573486, + -1.1306359767913818, + -0.17483314871788025, + -0.3118287920951843, + -1.1479005813598633, + 0.6552329659461975 + ], + "state_max": [ + 0.0012606654781848192, + 0.9533419013023376, + 1.441234827041626, + 0.10255199670791626, + -0.021535230800509453, + 0.2322024405002594, + -0.022155510261654854, + 0.18327920138835907, + 0.06926733255386353, + -0.24854479730129242, + 1.4151289463043213, + 1.7675095796585083, + 1.333852767944336, + 1.720163106918335, + 1.2428480386734009, + 0.7350764274597168, + 1.2933615446090698, + 1.1574125289916992, + 0.929772138595581, + 0.8751207590103149, + 0.44843629002571106, + 0.7306901812553406, + 0.12389291822910309, + 0.8440968990325928, + 1.3799595832824707, + 1.1950310468673706, + 1.071639060974121, + 1.2805979251861572, + 0.21328531205654144, + 0.3196083903312683, + 0.48254090547561646, + 0.801028311252594 + ], + "normalize_state": true, + "pad_action_dim": null, + "pad_state_dim": null + }, + "model": { + "resize": { + "size": [ + 256, + 480 + ] + }, + "center_crop": { + "size": [ + 224, + 224 + ] + }, + "color_jitter": { + "brightness": 0.2, + "contrast": [ + 0.8, + 1.2 + ], + "saturation": [ + 0.8, + 1.2 + ], + "hue": 0.05 + }, + "normalize": { + "mean": [ + 0.485, + 0.456, + 0.406 + ], + "std": [ + 0.229, + 0.224, + 0.225 + ] + }, + "img_aug": true + } + }, + "action_chunk_size": 100, + "root_dir": "/hfm/data/real_teleop_g1/lerobot", + "dataset_paths": [ + "Push_cart_grasp_and_place_grapes_on_plate" + ], + "data_config": "we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig", + "embodiment_tag": "new_embodiment", + "video_backend": "decord", + "balance_dataset_weights": true, + "balance_trajectory_weights": true, + "preload_all": false + }, + "model": { + "n_obs_steps": 1, + "chunk_size": 100, + "n_action_steps": 100, + "action_dim": 36, + "state_dim": 32, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "pre_norm": false, + "dropout": 0.1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "kl_weight": 10.0, + "temporal_ensemble_coeff": null + } +} \ No newline at end of file diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/debug-internal.log b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..870beb0f482b5d8efd2dac93c4bfd13d9d80276e --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/debug-internal.log @@ -0,0 +1,12 @@ +{"time":"2026-01-24T03:42:06.71737015Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"} +{"time":"2026-01-24T03:42:07.071896945Z","level":"INFO","msg":"stream: created new stream","id":"e5jhds0w"} +{"time":"2026-01-24T03:42:07.072202083Z","level":"INFO","msg":"stream: started","id":"e5jhds0w"} +{"time":"2026-01-24T03:42:07.073834424Z","level":"INFO","msg":"handler: started","stream_id":"e5jhds0w"} +{"time":"2026-01-24T03:42:07.073894488Z","level":"INFO","msg":"writer: started","stream_id":"e5jhds0w"} +{"time":"2026-01-24T03:42:07.074099822Z","level":"INFO","msg":"sender: started","stream_id":"e5jhds0w"} +{"time":"2026-01-24T09:50:49.440830292Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-01-24T09:50:49.813586925Z","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-01-24T09:50:49.825379014Z","level":"INFO","msg":"stream: closing","id":"e5jhds0w"} +{"time":"2026-01-24T09:50:49.825410233Z","level":"INFO","msg":"handler: closed","stream_id":"e5jhds0w"} +{"time":"2026-01-24T09:50:49.825511015Z","level":"INFO","msg":"sender: closed","stream_id":"e5jhds0w"} +{"time":"2026-01-24T09:50:49.825543212Z","level":"INFO","msg":"stream: closed","id":"e5jhds0w"} diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/debug.log b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ccb6d8ff766bdc97f1424d25cf3777c15e508fdc --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/debug.log @@ -0,0 +1,27 @@ +2026-01-24 03:42:06,424 INFO MainThread:2926353 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1 +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_setup.py:_flush():80] Configure stats pid to 2926353 +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_setup.py:_flush():80] Loading settings from /home/jliu/.config/wandb/settings +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_setup.py:_flush():80] Loading settings from /home/jliu/we_learn/wandb/settings +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_setup.py:_flush():80] Loading settings from environment variables +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /home/jliu/we_learn/.runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/logs/debug.log +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /home/jliu/we_learn/.runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/logs/debug-internal.log +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_init.py:init():841] calling init triggers +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_init.py:init():846] wandb.init called with sweep_config: {} +config: {'_wandb': {}} +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_init.py:init():889] starting backend +2026-01-24 03:42:06,699 INFO MainThread:2926353 [wandb_init.py:init():892] sending inform_init request +2026-01-24 03:42:06,712 INFO MainThread:2926353 [wandb_init.py:init():900] backend started and connected +2026-01-24 03:42:06,715 INFO MainThread:2926353 [wandb_init.py:init():970] updated telemetry +2026-01-24 03:42:06,724 INFO MainThread:2926353 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout +2026-01-24 03:42:07,408 INFO MainThread:2926353 [wandb_init.py:init():1041] starting run threads in backend +2026-01-24 03:42:07,584 INFO MainThread:2926353 [wandb_run.py:_console_start():2521] atexit reg +2026-01-24 03:42:07,584 INFO MainThread:2926353 [wandb_run.py:_redirect():2369] redirect: wrap_raw +2026-01-24 03:42:07,584 INFO MainThread:2926353 [wandb_run.py:_redirect():2438] Wrapping output streams. +2026-01-24 03:42:07,584 INFO MainThread:2926353 [wandb_run.py:_redirect():2461] Redirects installed. +2026-01-24 03:42:07,590 INFO MainThread:2926353 [wandb_init.py:init():1081] run started, returning control to user process +2026-01-24 03:42:07,593 INFO MainThread:2926353 [wandb_run.py:_config_callback():1396] config_cb None None {'exp': 'act-push-cart-grasp-and-place-grapes-on-plate', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2601240341', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'hfm', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'act-g1-real', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/home/jliu/we_learn/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 42000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'we-g1-teleop', 'conditions': [], 'image_keys': ['video'], 'state_key': 'state', 'action_key': 'action', 'instruction_key': 'annotation.human.task_description', 'pad_action_dim': 36, 'pad_state_dim': 32}, 'action_state': {'stat_path': 'dataset_statistics/g1-stats-Push_cart_grasp_and_place_grapes_on_plate.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-1.0481975078582764, -0.19984650611877441, 0.4533447325229645, -1.5717962980270386, -1.7463293075561523, -1.5717962980270386, -1.7463293075561523, -1.0481975078582764, -0.9210000038146973, -1.6845287084579468, -0.0010000000474974513, -0.0010000000474974513, -0.0010000000474974513, -0.0010000000474974513, -0.9514403939247131, -0.3264637887477875, -0.4328526556491852, -1.0471998453140259, -1.0728868246078491, -1.6144294738769531, -0.9963729381561279, -1.9203201532363892, -1.1714298725128174, -0.7144220471382141, -1.0471998453140259, -1.5777469873428345, -1.614428997039795, -1.1877036094665527, -0.17483314871788025, -0.3118287920951843, -1.1479005813598633, 0.6552329659461975, 0.0, 0.0, 0.0, 0.0], 'action_max': [0.09929836541414261, 0.9210000038146973, 1.630710482597351, 0.0010000000474974513, 0.0010000000474974513, 0.0010000000474974513, 0.0010000000474974513, 0.30392226576805115, 0.08567330241203308, -0.40413898229599, 1.517154335975647, 1.7463293075561523, 1.5717962980270386, 1.7463293075561523, 1.2656242847442627, 0.8192498087882996, 1.313675045967102, 1.2117120027542114, 0.9534142017364502, 0.9147428870201111, 0.5678492188453674, 0.7814759612083435, 0.3561345934867859, 0.9256001710891724, 1.4109472036361694, 1.4722950458526611, 1.1215534210205078, 1.3663387298583984, 0.21328531205654144, 0.3196083903312683, 0.48254090547561646, 0.801028311252594, 0.3499999940395355, 0.0, 0.0, 0.0], 'state_min': [-1.0231640338897705, -0.0695418044924736, 0.24746619164943695, -1.4132853746414185, -1.8126834630966187, -1.3577090501785278, -1.7228399515151978, -1.0409873723983765, -0.9193933606147766, -1.5848426818847656, -0.07512053102254868, 0.023398026823997498, -0.05005142092704773, 0.018349293619394302, -0.896072506904602, -0.10484998673200607, -0.3023500144481659, -1.0332319736480713, -1.0468820333480835, -1.5026646852493286, -0.9785187244415283, -1.8509875535964966, -1.010833501815796, -0.6269427537918091, -1.0290495157241821, -1.522499918937683, -1.3486835956573486, -1.1306359767913818, -0.17483314871788025, -0.3118287920951843, -1.1479005813598633, 0.6552329659461975], 'state_max': [0.0012606654781848192, 0.9533419013023376, 1.441234827041626, 0.10255199670791626, -0.021535230800509453, 0.2322024405002594, -0.022155510261654854, 0.18327920138835907, 0.06926733255386353, -0.24854479730129242, 1.4151289463043213, 1.7675095796585083, 1.333852767944336, 1.720163106918335, 1.2428480386734009, 0.7350764274597168, 1.2933615446090698, 1.1574125289916992, 0.929772138595581, 0.8751207590103149, 0.44843629002571106, 0.7306901812553406, 0.12389291822910309, 0.8440968990325928, 1.3799595832824707, 1.1950310468673706, 1.071639060974121, 1.2805979251861572, 0.21328531205654144, 0.3196083903312683, 0.48254090547561646, 0.801028311252594], 'normalize_state': True, 'pad_action_dim': None, 'pad_state_dim': None}, 'model': {'resize': {'size': [256, 480]}, 'center_crop': {'size': [224, 224]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}}, 'action_chunk_size': 100, 'root_dir': '/hfm/data/real_teleop_g1/lerobot', 'dataset_paths': ['Push_cart_grasp_and_place_grapes_on_plate'], 'data_config': 'we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig', 'embodiment_tag': 'new_embodiment', 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'preload_all': False}, 'model': {'n_obs_steps': 1, 'chunk_size': 100, 'n_action_steps': 100, 'action_dim': 36, 'state_dim': 32, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'pre_norm': False, 'dropout': 0.1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'kl_weight': 10.0, 'temporal_ensemble_coeff': None}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/hfm/cache', 'TORCH_HOME': '/hfm/cache', 'HF_TOKEN': 'hf_...adRb', 'HF_LEROBOT_HOME': 'Not Set', 'WE_HOME': 'Not Set', 'DATA_HOME': 'Not Set', 'UV_CACHE_DIR': 'Not Set', 'WANDB_API_KEY': 'Not Set', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '6,7', 'WORLD_SIZE': '2', 'LOCAL_WORLD_SIZE': '2', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '33685'}} +2026-01-24 09:50:48,903 INFO MainThread:2926353 [wandb_run.py:_finish():2287] finishing run jliu530-soochow-university/hfm/e5jhds0w +2026-01-24 09:50:48,905 INFO MainThread:2926353 [wandb_run.py:_atexit_cleanup():2486] got exitcode: 0 +2026-01-24 09:50:48,906 INFO MainThread:2926353 [wandb_run.py:_restore():2468] restore +2026-01-24 09:50:48,906 INFO MainThread:2926353 [wandb_run.py:_restore():2474] restore done +2026-01-24 09:50:49,823 INFO MainThread:2926353 [wandb_run.py:_footer_sync_info():3862] logging synced files diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/files/config.yaml b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b45c23e882758277d494ff6e27fc98f70f9d14f0 --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/files/config.yaml @@ -0,0 +1,457 @@ +_wandb: + value: + cli_version: 0.23.1 + e: + ugbf489zo1bfr2u0ckxloaeb6v5bk0va: + args: + - data:gr00t + - data.transform.repack:gr00t-we + - data.transform.action-state:maxmin + - data.transform.model:act + - model:act + - --seed=2026 + - --exp=act-push-cart-grasp-and-place-grapes-on-plate + - --train.name=act-g1-real + - --log.report-to=wandb + - --wandb.entity=jliu530-soochow-university + - --train.data_parallel=ddp + - --train.mixed_precision=bf16 + - --train.train-batch-size=32 + - --train.gradient_accumulation_steps=1 + - --train.validation_steps=500 + - --train.val_num_batches=20 + - --train.max-training-steps=42000 + - --train.learning-rate=1e-4 + - --train.max-grad-norm=1.0 + - --train.lr_scheduler_kwargs.weight_decay=1e-6 + - --train.lr_scheduler_kwargs.betas + - "0.95" + - "0.999" + - --train.lr_scheduler_type=cosine + - --train.warmup-steps=1000 + - --train.warmup-ratio=None + - --train.checkpointing-steps=5000 + - --data.root_dir=/hfm/data/real_teleop_g1/lerobot + - --data.dataset_paths=Push_cart_grasp_and_place_grapes_on_plate + - --data.data_config=we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig + - --data.transform.repack.pad-action-dim=36 + - --data.transform.repack.pad-state-dim=32 + - --data.transform.action-state.stat-path=dataset_statistics/g1-stats-Push_cart_grasp_and_place_grapes_on_plate.json + - --data.transform.action-state.stat-action-key=action + - --data.transform.action-state.stat-state-key=states + - --data.transform.action-state.normalize-state + - --data.transform.action-state.action-norm-type=bounds + - --data.transform.model.img-aug + - --data.action-chunk-size=100 + - --model.chunk-size=100 + - --model.n-action-steps=100 + - --model.action-dim=36 + - --model.state-dim=32 + - --model.use-vae + - --model.kl-weight=10.0 + codePath: scripts/train.py + codePathLocal: scripts/train.py + cpu_count: 128 + cpu_count_logical: 128 + cudaVersion: "12.9" + disk: + /: + total: "105089261568" + used: "95563231232" + email: jliu530@163.com + executable: /home/jliu/we_learn/.venv-pusht/bin/python3 + git: + commit: df0aa6074e8b16d0e2be002a9a9c1971edf209ec + remote: https://github.com/songlin/we_learn.git + gpu: NVIDIA A100-SXM4-80GB + gpu_count: 8 + gpu_nvidia: + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-37370f00-90f5-4e03-f763-7a35649e6783 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-d4b40383-188b-cc16-9180-20c1a71a777f + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-0f3d827c-66cf-04da-b182-4ed0414a2549 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-164a49bb-43d3-2250-59e2-1b40eee0757f + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-c0471f40-ae19-5371-a1af-da406ca30f83 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-d9040f59-13d3-6c37-4ea0-c6c751424527 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-f834d0e8-a118-24c7-5461-30fce111bea6 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-6e687915-3247-735c-6641-544540804e79 + host: nebula99 + memory: + total: "1623177732096" + os: Linux-6.8.0-88-generic-x86_64-with-glibc2.39 + program: /home/jliu/we_learn/scripts/train.py + python: CPython 3.10.19 + root: /home/jliu/we_learn/.runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341 + startedAt: "2026-01-24T03:42:06.420627Z" + writerId: ugbf489zo1bfr2u0ckxloaeb6v5bk0va + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 2 + - 3 + - 11 + - 41 + - 49 + - 71 + "2": + - 1 + - 2 + - 3 + - 11 + - 41 + - 49 + - 51 + - 71 + - 80 + - 83 + "3": + - 2 + - 13 + - 61 + "4": 3.10.19 + "5": 0.23.1 + "6": 4.44.2 + "12": 0.23.1 + "13": linux-x86_64 +auto_tag_run: + value: false +data: + value: + action_chunk_size: 100 + balance_dataset_weights: true + balance_trajectory_weights: true + data_config: we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig + dataset_paths: + - Push_cart_grasp_and_place_grapes_on_plate + embodiment_tag: new_embodiment + preload_all: false + root_dir: /hfm/data/real_teleop_g1/lerobot + transform: + action_state: + action_max: + - 0.09929836541414261 + - 0.9210000038146973 + - 1.630710482597351 + - 0.0010000000474974513 + - 0.0010000000474974513 + - 0.0010000000474974513 + - 0.0010000000474974513 + - 0.30392226576805115 + - 0.08567330241203308 + - -0.40413898229599 + - 1.517154335975647 + - 1.7463293075561523 + - 1.5717962980270386 + - 1.7463293075561523 + - 1.2656242847442627 + - 0.8192498087882996 + - 1.313675045967102 + - 1.2117120027542114 + - 0.9534142017364502 + - 0.9147428870201111 + - 0.5678492188453674 + - 0.7814759612083435 + - 0.3561345934867859 + - 0.9256001710891724 + - 1.4109472036361694 + - 1.4722950458526611 + - 1.1215534210205078 + - 1.3663387298583984 + - 0.21328531205654144 + - 0.3196083903312683 + - 0.48254090547561646 + - 0.801028311252594 + - 0.3499999940395355 + - 0 + - 0 + - 0 + action_min: + - -1.0481975078582764 + - -0.19984650611877441 + - 0.4533447325229645 + - -1.5717962980270386 + - -1.7463293075561523 + - -1.5717962980270386 + - -1.7463293075561523 + - -1.0481975078582764 + - -0.9210000038146973 + - -1.6845287084579468 + - -0.0010000000474974513 + - -0.0010000000474974513 + - -0.0010000000474974513 + - -0.0010000000474974513 + - -0.9514403939247131 + - -0.3264637887477875 + - -0.4328526556491852 + - -1.0471998453140259 + - -1.0728868246078491 + - -1.6144294738769531 + - -0.9963729381561279 + - -1.9203201532363892 + - -1.1714298725128174 + - -0.7144220471382141 + - -1.0471998453140259 + - -1.5777469873428345 + - -1.614428997039795 + - -1.1877036094665527 + - -0.17483314871788025 + - -0.3118287920951843 + - -1.1479005813598633 + - 0.6552329659461975 + - 0 + - 0 + - 0 + - 0 + action_norm_masks: + - true + - true + - true + - true + - true + - true + - false + action_norm_type: bounds + normalize_state: true + pad_action_dim: null + pad_state_dim: null + stat_action_key: action + stat_path: dataset_statistics/g1-stats-Push_cart_grasp_and_place_grapes_on_plate.json + stat_state_key: states + state_max: + - 0.0012606654781848192 + - 0.9533419013023376 + - 1.441234827041626 + - 0.10255199670791626 + - -0.021535230800509453 + - 0.2322024405002594 + - -0.022155510261654854 + - 0.18327920138835907 + - 0.06926733255386353 + - -0.24854479730129242 + - 1.4151289463043213 + - 1.7675095796585083 + - 1.333852767944336 + - 1.720163106918335 + - 1.2428480386734009 + - 0.7350764274597168 + - 1.2933615446090698 + - 1.1574125289916992 + - 0.929772138595581 + - 0.8751207590103149 + - 0.44843629002571106 + - 0.7306901812553406 + - 0.12389291822910309 + - 0.8440968990325928 + - 1.3799595832824707 + - 1.1950310468673706 + - 1.071639060974121 + - 1.2805979251861572 + - 0.21328531205654144 + - 0.3196083903312683 + - 0.48254090547561646 + - 0.801028311252594 + state_min: + - -1.0231640338897705 + - -0.0695418044924736 + - 0.24746619164943695 + - -1.4132853746414185 + - -1.8126834630966187 + - -1.3577090501785278 + - -1.7228399515151978 + - -1.0409873723983765 + - -0.9193933606147766 + - -1.5848426818847656 + - -0.07512053102254868 + - 0.023398026823997498 + - -0.05005142092704773 + - 0.018349293619394302 + - -0.896072506904602 + - -0.10484998673200607 + - -0.3023500144481659 + - -1.0332319736480713 + - -1.0468820333480835 + - -1.5026646852493286 + - -0.9785187244415283 + - -1.8509875535964966 + - -1.010833501815796 + - -0.6269427537918091 + - -1.0290495157241821 + - -1.522499918937683 + - -1.3486835956573486 + - -1.1306359767913818 + - -0.17483314871788025 + - -0.3118287920951843 + - -1.1479005813598633 + - 0.6552329659461975 + use_norm_mask: false + model: + center_crop: + size: + - 224 + - 224 + color_jitter: + brightness: 0.2 + contrast: + - 0.8 + - 1.2 + hue: 0.05 + saturation: + - 0.8 + - 1.2 + img_aug: true + normalize: + mean: + - 0.485 + - 0.456 + - 0.406 + std: + - 0.229 + - 0.224 + - 0.225 + resize: + size: + - 256 + - 480 + repack: + action_key: action + conditions: [] + dataset_name: we-g1-teleop + image_keys: + - video + instruction_key: annotation.human.task_description + pad_action_dim: 36 + pad_state_dim: 32 + state_key: state + video_backend: decord +debug: + value: false +environment_variables: + value: + CUDA_VISIBLE_DEVICES: 6,7 + DATA_HOME: Not Set + HF_HOME: /hfm/cache + HF_LEROBOT_HOME: Not Set + HF_TOKEN: hf_...adRb + LOCAL_RANK: "0" + LOCAL_WORLD_SIZE: "2" + MASTER_ADDR: nebula99 + MASTER_PORT: "33685" + OMP_NUM_THREADS: "32" + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python + RANK: "0" + TORCH_HOME: /hfm/cache + UV_CACHE_DIR: Not Set + WANDB_API_KEY: Not Set + WE_HOME: Not Set + WORLD_SIZE: "2" +eval: + value: false +exp: + value: act-push-cart-grasp-and-place-grapes-on-plate +log: + value: + log_freq: 100 + logging_dir: logs + report_to: wandb +model: + value: + action_dim: 36 + chunk_size: 100 + dim_feedforward: 3200 + dim_model: 512 + dropout: 0.1 + feedforward_activation: relu + kl_weight: 10 + latent_dim: 32 + n_action_steps: 100 + n_decoder_layers: 1 + n_encoder_layers: 4 + n_heads: 8 + n_obs_steps: 1 + n_vae_encoder_layers: 4 + pre_norm: false + state_dim: 32 + temporal_ensemble_coeff: null + use_vae: true +seed: + value: 2026 +timestamp: + value: "2601240341" +train: + value: + checkpointing_steps: 5000 + data_parallel: ddp + deepspeed_config: /home/jliu/we_learn/src/InternVLA/config/deepseeds/zero3.json + enable_gradient_checkpointing: true + enable_mixed_precision_training: true + gradient_accumulation_steps: 1 + hf_token: .hf_token + learning_rate: 0.0001 + lora: false + lr_scheduler_kwargs: + betas: + - 0.95 + - 0.999 + eps: 1e-08 + weight_decay: 1e-06 + lr_scheduler_type: cosine + max_checkpoints_to_keep: null + max_grad_norm: 1 + max_training_steps: 42000 + mixed_precision: bf16 + name: act-g1-real + num_train_epochs: null + num_workers: 8 + output_dir: .runs + overfit_single_batch: false + reduce_in_full_precision: true + resume_from_checkpoint: null + sharding_strategy: full-shard + skip_resumed_steps: false + train_batch_size: 32 + val_batch_size: 16 + val_num_batches: 20 + validation_steps: 500 + warmup_ratio: null + warmup_steps: 1000 +wandb: + value: + entity: jliu530-soochow-university + group: null + id: null + name: null + project: hfm + resume: allow diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/files/output.log b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4c807d49e683a7249bdd874486a0838421f45293 --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/files/output.log @@ -0,0 +1,263 @@ +[03:42:07 01/24] INFO  | >> [*] Saved configuration to ]8;id=805427;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=514750;file:///home/jliu/we_learn/scripts/train.py#129\129]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-   +  04.b64.gpus2.2601240341   +  INFO  | >> [*] Training configurations: ]8;id=2657;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=644039;file:///home/jliu/we_learn/scripts/train.py#182\182]8;;\ +  INFO  | >> |=> training task: 'act-g1-real' ]8;id=471497;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=12046;file:///home/jliu/we_learn/scripts/train.py#183\183]8;;\ +  INFO  | >> |=> run name: ]8;id=329497;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=220436;file:///home/jliu/we_learn/scripts/train.py#184\184]8;;\ +  act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.26012   +  40341   +  INFO  | >> |=> seed: 2026 ]8;id=844246;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=394625;file:///home/jliu/we_learn/scripts/train.py#185\185]8;;\ +  INFO  | >> |=> mixed precision: torch.bfloat16 ]8;id=356929;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=94020;file:///home/jliu/we_learn/scripts/train.py#186\186]8;;\ +  INFO  | >> |=> warmup steps: 1000 ]8;id=147732;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=919050;file:///home/jliu/we_learn/scripts/train.py#187\187]8;;\ +  INFO  | >> |=> validation steps: 500 ]8;id=25941;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=888976;file:///home/jliu/we_learn/scripts/train.py#188\188]8;;\ +  INFO  | >> |=> checkpoint steps: 5000 ]8;id=95089;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=418144;file:///home/jliu/we_learn/scripts/train.py#189\189]8;;\ +  INFO  | >> |=> max gradient norm: 1.0 ]8;id=122104;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=446109;file:///home/jliu/we_learn/scripts/train.py#190\190]8;;\ +/home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/albumentations/__init__.py:13: UserWarning: A new version of Albumentations is available: 2.0.8 (you have 1.4.18). Upgrade using: pip install -U albumentations. To disable automatic update checks, set the environment variable NO_ALBUMENTATIONS_UPDATE to 1. + check_for_updates() +Loading external config: we.data.gr00t.data_config.WeUnitreeG1LocoDataConfig +Initialized dataset Push_cart_grasp_and_place_grapes_on_plate with EmbodimentTag.NEW_EMBODIMENT +Loading external config: we.data.gr00t.data_config.WeUnitreeG1LocoDataConfig +Initialized dataset Push_cart_grasp_and_place_grapes_on_plate with EmbodimentTag.NEW_EMBODIMENT +[03:42:08 01/24] INFO  | >> |=> Num training samples: ]8;id=917958;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=514883;file:///home/jliu/we_learn/scripts/train.py#193\193]8;;\ +  INFO  | >> |=> Training dataset size: 86,876 ]8;id=440366;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=591396;file:///home/jliu/we_learn/scripts/train.py#194\194]8;;\ +  INFO  | >> |=> Val dataset size: 86,876 ]8;id=26608;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=603472;file:///home/jliu/we_learn/scripts/train.py#196\196]8;;\ +  INFO  | >> [*] Initialize optimizers and schedulers... ]8;id=738892;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=548632;file:///home/jliu/we_learn/scripts/train.py#199\199]8;;\ +  INFO  | >> [*] ***** Running training ***** ]8;id=450268;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=492873;file:///home/jliu/we_learn/scripts/train.py#203\203]8;;\ +  INFO  | >> |=> Num training examples = 86876 ]8;id=928955;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=127940;file:///home/jliu/we_learn/scripts/train.py#204\204]8;;\ +  INFO  | >> |=> Max training Epochs = 31 ]8;id=749231;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=974614;file:///home/jliu/we_learn/scripts/train.py#205\205]8;;\ +  INFO  | >> |=> Total optimization steps = 42000 ]8;id=322570;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=574775;file:///home/jliu/we_learn/scripts/train.py#206\206]8;;\ +  INFO  | >> |=> Num steps Per Epoch = 1358 ]8;id=843992;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=927413;file:///home/jliu/we_learn/scripts/train.py#207\207]8;;\ +  INFO  | >> |=> Effective training epochs = 61.86 ]8;id=282104;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=9727;file:///home/jliu/we_learn/scripts/train.py#208\208]8;;\ +  INFO  | >> |=> Global train batch size (w. parallel, distributed & accumulation) = 64 ]8;id=480486;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=842580;file:///home/jliu/we_learn/scripts/train.py#209\209]8;;\ +  INFO  | >> |=> Device train batch size = 32 ]8;id=209544;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=272745;file:///home/jliu/we_learn/scripts/train.py#210\210]8;;\ +  INFO  | >> |=> Gradient Accumulation steps = 1 ]8;id=866305;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=521307;file:///home/jliu/we_learn/scripts/train.py#211\211]8;;\ +  INFO  | >> |=> Num processes (GPUs) = 2 ]8;id=35253;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=871789;file:///home/jliu/we_learn/scripts/train.py#212\212]8;;\ +[03:42:11 01/24] INFO  | >> TensorFlow version 2.20.0 available. ]8;id=459749;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/datasets/config.py\config.py]8;;\:]8;id=722818;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/datasets/config.py#112\112]8;;\ +  INFO  | >> [*] Accelerator runs in: ]8;id=966756;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=191940;file:///home/jliu/we_learn/scripts/train.py#219\219]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-   +  04.b64.gpus2.2601240341   +Traing steps: 24%|█████████ | 10000/42000 [1:28:20<5:01:37, 1.77it/s, loss=0.0759, lr=8.9e-05] + +[04:27:09 01/24] INFO  | >> Saving current state to ]8;id=680052;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=279409;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus   +  2.2601240341/checkpoints/ckpt_5000   +  INFO  | >> Model weights saved in ]8;id=898249;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=431290;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_5000/model.safetensors   +[04:27:10 01/24] INFO  | >> Optimizer state saved in ]8;id=224;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=473149;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_5000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=639771;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=832328;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_5000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=788377;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=572992;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_5000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=374604;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=162278;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_5000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=806908;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=971194;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_5000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_5000 +[05:10:33 01/24] INFO  | >> Saving current state to ]8;id=755470;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=983054;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus   +  2.2601240341/checkpoints/ckpt_10000   +  INFO  | >> Model weights saved in ]8;id=616494;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=137590;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_10000/model.safetensors   +[05:10:34 01/24] INFO  | >> Optimizer state saved in ]8;id=304800;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=88534;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_10000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=141920;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=821494;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_10000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=532364;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=432054;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_10000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=846104;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=206251;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_10000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=930895;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=77000;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_10000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_10000 +[05:55:03 01/24] INFO  | >> Saving current state to ]8;id=370569;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=979940;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus   +  2.2601240341/checkpoints/ckpt_15000   +[05:55:04 01/24] INFO  | >> Model weights saved in ]8;id=535180;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=499147;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_15000/model.safetensors   +[05:55:05 01/24] INFO  | >> Optimizer state saved in ]8;id=341880;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=958353;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_15000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=599385;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=648655;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_15000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=544854;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=217040;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_15000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=627278;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=244660;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_15000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=418546;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=35811;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_15000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_15000 +[06:38:30 01/24] INFO  | >> Saving current state to ]8;id=47434;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=679719;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus   +  2.2601240341/checkpoints/ckpt_20000   +[06:38:31 01/24] INFO  | >> Model weights saved in ]8;id=473768;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=910263;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_20000/model.safetensors   +  INFO  | >> Optimizer state saved in ]8;id=745480;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=711599;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_20000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=555678;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=229348;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_20000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=144411;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=168181;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_20000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=286058;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=446016;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_20000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=401798;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=319720;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_20000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_20000 +[07:21:57 01/24] INFO  | >> Saving current state to ]8;id=486450;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=521773;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus   +  2.2601240341/checkpoints/ckpt_25000   +[07:21:58 01/24] INFO  | >> Model weights saved in ]8;id=490419;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=185558;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_25000/model.safetensors   +[07:21:59 01/24] INFO  | >> Optimizer state saved in ]8;id=125939;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=552863;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_25000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=855797;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=99159;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_25000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=999999;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=97642;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_25000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=427821;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=571536;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_25000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=702964;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=191314;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_25000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_25000 +[08:05:40 01/24] INFO  | >> Saving current state to ]8;id=368785;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=589302;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus   +  2.2601240341/checkpoints/ckpt_30000   +[08:05:41 01/24] INFO  | >> Model weights saved in ]8;id=227061;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=115021;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_30000/model.safetensors   +[08:05:42 01/24] INFO  | >> Optimizer state saved in ]8;id=787738;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=595634;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_30000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=590171;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=281123;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_30000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=819504;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=276522;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_30000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=717799;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=63361;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_30000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=194505;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=677321;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_30000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_30000 +[08:49:29 01/24] INFO  | >> Saving current state to ]8;id=290231;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=616676;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus   +  2.2601240341/checkpoints/ckpt_35000   +[08:49:30 01/24] INFO  | >> Model weights saved in ]8;id=329883;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=169230;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_35000/model.safetensors   +  INFO  | >> Optimizer state saved in ]8;id=431638;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=908584;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_35000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=434664;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=16179;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_35000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=324973;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=128996;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_35000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=981583;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=899091;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_35000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=636753;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=656123;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_35000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_35000 +[09:33:20 01/24] INFO  | >> Saving current state to ]8;id=969664;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=423824;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus   +  2.2601240341/checkpoints/ckpt_40000   +[09:33:21 01/24] INFO  | >> Model weights saved in ]8;id=485144;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=446823;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_40000/model.safetensors   +[09:33:22 01/24] INFO  | >> Optimizer state saved in ]8;id=392321;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=903139;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_40000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=407945;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=306059;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_40000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=771109;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=749912;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_40000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=729075;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=346604;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_40000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=960350;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=128256;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_40000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_40000 +[09:50:32 01/24] INFO  | >> Saving current state to ]8;id=614052;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=946822;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus   +  2.2601240341/checkpoints/ckpt_41999   +[09:50:33 01/24] INFO  | >> Model weights saved in ]8;id=588107;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=58872;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_41999/model.safetensors   +[09:50:34 01/24] INFO  | >> Optimizer state saved in ]8;id=128644;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=124028;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_41999/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=625950;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=679365;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_41999/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=606210;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=132432;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_41999/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=250930;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=814323;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_41999/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=131290;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=192379;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_41999/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/checkpoints/ckpt_41999 +Training has reached maximum steps. +[09:50:46 01/24] INFO  | >> Saving current state to ]8;id=980067;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=567626;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus   +  2.2601240341/checkpoints/ckpt_41999   +[09:50:47 01/24] INFO  | >> Model weights saved in ]8;id=267359;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=824815;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_41999/model.safetensors   +[09:50:48 01/24] INFO  | >> Optimizer state saved in ]8;id=290278;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=567436;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_41999/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=145392;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=990094;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_41999/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=188012;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=579843;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_41999/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=539017;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=859689;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_41999/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=791184;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=779188;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpu   +  s2.2601240341/checkpoints/ckpt_41999/random_states_0.pkl   +  INFO  | >> [*] Finalized ACT Trainer. Epoch losses: [0.0, 0.3679253160953522, 0.15888100862503052, ]8;id=658356;file:///home/jliu/we_learn/src/we/learn/trainers/act_g1_real.py\act_g1_real.py]8;;\:]8;id=254601;file:///home/jliu/we_learn/src/we/learn/trainers/act_g1_real.py#331\331]8;;\ +  0.09571532905101776, 0.08934973180294037]   +  INFO  | >> [*] Happy Ending! ]8;id=903984;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=158174;file:///home/jliu/we_learn/scripts/train.py#314\314]8;;\ diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/files/requirements.txt b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..20111d90fafa72be7424aebca4417875b5f6e178 --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/files/requirements.txt @@ -0,0 +1,237 @@ +markdown-it-py==4.0.0 +tensorflow-datasets==4.9.9 +pynput==1.8.1 +nvidia-cufile-cu12==1.13.1.3 +nvidia-nccl-cu12==2.27.3 +pandas==2.3.3 +sniffio==1.3.1 +keras==3.12.0 +expandvars==1.1.2 +isort==6.0.1 +accelerate==1.12.0 +diffusers==0.30.2 +lazy_loader==0.4 +safetensors==0.6.2 +iniconfig==2.1.0 +Jinja2==3.1.6 +pymunk==7.2.0 +datasets==4.4.1 +libclang==18.1.1 +fasteners==0.20 +httpcore==1.0.9 +immutabledict==4.2.2 +protobuf==6.33.2 +einx==0.3.0 +requests==2.32.5 +cffi==2.0.0 +optree==0.18.0 +astunparse==1.6.3 +transforms3d==0.4.2 +xxhash==3.5.0 +wcwidth==0.2.14 +click==8.3.0 +sympy==1.14.0 +annotated-types==0.7.0 +nvidia-nvjitlink-cu12==12.8.93 +numpydantic==1.6.7 +Pygments==2.19.2 +mergedeep==1.3.4 +black==25.9.0 +albumentations==1.4.18 +gymnasium==0.29.1 +multiprocess==0.70.16 +nvidia-cusparse-cu12==12.5.8.93 +certifi==2025.8.3 +array_record==0.8.1 +pyyaml-include==1.4.1 +tzdata==2025.2 +dm-tree==0.1.8 +cycler==0.12.1 +fastapi==0.128.0 +wrapt==2.0.1 +pydantic_core==2.27.2 +tqdm==4.67.1 +starlette==0.50.0 +OpenEXR==3.4.4 +av==16.0.1 +argparse==1.4.0 +pyparsing==3.3.2 +yarl==1.20.1 +dotenv==0.9.9 +omegaconf==2.3.0 +opt_einsum==3.4.0 +trimesh==4.11.1 +imageio==2.34.2 +soupsieve==2.8 +gitdb==4.0.12 +beautifulsoup4==4.14.2 +argcomplete==3.6.2 +tabulate==0.9.0 +mypy_extensions==1.1.0 +propcache==0.3.2 +nvidia-cusparselt-cu12==0.7.1 +annotated-doc==0.0.4 +PySocks==1.7.1 +simple-parsing==0.1.8 +yacs==0.1.8 +pycparser==2.23 +nvidia-cudnn-cu12==9.10.2.21 +tensorboard==2.20.0 +transformers==4.44.2 +opencv-python-headless==4.11.0.86 +websockets==15.0.1 +ruamel.yaml==0.18.15 +torchvision==0.23.0 +matplotlib==3.10.8 +rerun-sdk==0.22.1 +evdev==1.9.2 +blinker==1.9.0 +h11==0.16.0 +pyzmq==27.1.0 +nvidia-curand-cu12==10.3.9.90 +idna==3.10 +plotly==6.2.0 +aiohappyeyeballs==2.6.1 +antlr4-python3-runtime==4.9.3 +platformdirs==4.4.0 +filelock==3.19.1 +pytokens==0.1.10 +hf-xet==1.1.10 +aiohttp-cors==0.8.1 +decord==0.6.0 +docker-pycreds==0.4.0 +tifffile==2025.5.10 +pytz==2025.2 +portalocker==3.2.0 +nvidia-cuda-cupti-cu12==12.8.90 +scikit-image==0.25.2 +ruamel.yaml.clib==0.2.14 +python-xlib==0.33 +narwhals==2.6.0 +shellingham==1.5.4 +python-dotenv==1.2.1 +triton==3.4.0 +draccus==0.10.0 +ml_dtypes==0.5.4 +wheel==0.46.3 +kiwisolver==1.4.9 +tomli==2.2.1 +termcolor==3.1.0 +we.learn==0.1.0 +typing-inspect==0.9.0 +exceptiongroup==1.3.0 +Farama-Notifications==0.0.4 +setproctitle==1.3.7 +hf_transfer==0.1.9 +albucore==0.0.17 +nvidia-cusolver-cu12==11.7.3.90 +shtab==1.7.2 +imageio-ffmpeg==0.6.0 +typeguard==4.4.4 +contourpy==1.3.2 +nvidia-nvtx-cu12==12.8.90 +zipp==3.23.0 +inquirerpy==0.3.4 +torchcodec==0.6.0 +tensorboard-data-server==0.7.2 +importlib_metadata==8.7.0 +fvcore==0.1.5.post20221221 +llvmlite==0.46.0 +lerobot==0.1.0 +setuptools==80.9.0 +pluggy==1.6.0 +asciitree==0.3.3 +MarkupSafe==3.0.3 +wandb==0.23.1 +Flask==3.1.2 +pydantic==2.10.6 +httpx==0.28.1 +pfzy==0.3.4 +frozendict==2.4.6 +numpy==1.26.4 +gast==0.7.0 +pathspec==0.12.1 +ruff==0.13.2 +packaging==25.0 +promise==2.3 +tensorflow-addons==0.19.0 +importlib_resources==6.5.2 +torchaudio==2.8.0 +iopath==0.1.10 +google-pasta==0.2.0 +mypy==1.18.2 +orderly-set==5.5.0 +pyarrow==21.0.0 +vuer==0.0.68 +toml==0.10.2 +PyYAML==6.0.3 +torch==2.8.0 +numcodecs==0.13.1 +pydantic-yaml==1.6.0 +numba==0.63.1 +eval_type_backport==0.2.2 +psutil==7.1.0 +fsspec==2025.3.0 +gdown==5.2.0 +GitPython==3.1.45 +huggingface-hub==0.35.3 +fonttools==4.61.1 +frozenlist==1.7.0 +Markdown==3.10.1 +typing_extensions==4.15.0 +tensorflow==2.20.0 +h5py==3.14.0 +tyro==0.9.32 +cloudpickle==3.1.1 +cmake==4.1.0 +mdurl==0.1.2 +typer==0.19.2 +anyio==4.11.0 +namex==0.1.0 +pipablepytorch3d==0.7.6 +nvidia-cuda-nvrtc-cu12==12.8.93 +docstring_parser==0.17.0 +killport==1.2.0 +nvidia-cuda-runtime-cu12==12.8.90 +aiosignal==1.4.0 +dill==0.3.8 +nvidia-cublas-cu12==12.8.4.1 +zarr==2.18.3 +msgpack==1.1.1 +flatbuffers==25.12.19 +regex==2025.9.18 +python-dateutil==2.9.0.post0 +tokenizers==0.19.1 +pytest==8.4.2 +charset-normalizer==3.4.3 +sentry-sdk==2.39.0 +rich==14.1.0 +itsdangerous==2.2.0 +nvidia-cufft-cu12==11.3.3.83 +scipy==1.15.3 +simplejpeg==1.9.0 +uvicorn==0.40.0 +grpcio==1.76.0 +six==1.17.0 +absl-py==2.3.1 +jsonlines==4.0.0 +urllib3==2.5.0 +opencv-python==4.11.0.86 +etils==1.13.0 +mpmath==1.3.0 +einops==0.8.1 +smmap==5.0.2 +tensorflow-graphics==2021.12.3 +tensorflow-metadata==1.17.3 +multidict==6.6.4 +Werkzeug==3.1.3 +pillow==11.3.0 +prompt_toolkit==3.0.52 +waterbear==2.6.8 +async-timeout==4.0.3 +networkx==3.4.2 +attrs==25.3.0 +params_proto==2.13.2 +deepdiff==8.6.1 +aiohttp==3.10.5 +we.learn==0.1.0 diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/files/wandb-metadata.json b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a146ea805a95742e8337a90d1a1062d3d6211397 --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/files/wandb-metadata.json @@ -0,0 +1,136 @@ +{ + "os": "Linux-6.8.0-88-generic-x86_64-with-glibc2.39", + "python": "CPython 3.10.19", + "startedAt": "2026-01-24T03:42:06.420627Z", + "args": [ + "data:gr00t", + "data.transform.repack:gr00t-we", + "data.transform.action-state:maxmin", + "data.transform.model:act", + "model:act", + "--seed=2026", + "--exp=act-push-cart-grasp-and-place-grapes-on-plate", + "--train.name=act-g1-real", + "--log.report-to=wandb", + "--wandb.entity=jliu530-soochow-university", + "--train.data_parallel=ddp", + "--train.mixed_precision=bf16", + "--train.train-batch-size=32", + "--train.gradient_accumulation_steps=1", + "--train.validation_steps=500", + "--train.val_num_batches=20", + "--train.max-training-steps=42000", + "--train.learning-rate=1e-4", + "--train.max-grad-norm=1.0", + "--train.lr_scheduler_kwargs.weight_decay=1e-6", + "--train.lr_scheduler_kwargs.betas", + "0.95", + "0.999", + "--train.lr_scheduler_type=cosine", + "--train.warmup-steps=1000", + "--train.warmup-ratio=None", + "--train.checkpointing-steps=5000", + "--data.root_dir=/hfm/data/real_teleop_g1/lerobot", + "--data.dataset_paths=Push_cart_grasp_and_place_grapes_on_plate", + "--data.data_config=we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig", + "--data.transform.repack.pad-action-dim=36", + "--data.transform.repack.pad-state-dim=32", + "--data.transform.action-state.stat-path=dataset_statistics/g1-stats-Push_cart_grasp_and_place_grapes_on_plate.json", + "--data.transform.action-state.stat-action-key=action", + "--data.transform.action-state.stat-state-key=states", + "--data.transform.action-state.normalize-state", + "--data.transform.action-state.action-norm-type=bounds", + "--data.transform.model.img-aug", + "--data.action-chunk-size=100", + "--model.chunk-size=100", + "--model.n-action-steps=100", + "--model.action-dim=36", + "--model.state-dim=32", + "--model.use-vae", + "--model.kl-weight=10.0" + ], + "program": "/home/jliu/we_learn/scripts/train.py", + "codePath": "scripts/train.py", + "codePathLocal": "scripts/train.py", + "git": { + "remote": "https://github.com/songlin/we_learn.git", + "commit": "df0aa6074e8b16d0e2be002a9a9c1971edf209ec" + }, + "email": "jliu530@163.com", + "root": "/home/jliu/we_learn/.runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341", + "host": "nebula99", + "executable": "/home/jliu/we_learn/.venv-pusht/bin/python3", + "cpu_count": 128, + "cpu_count_logical": 128, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "105089261568", + "used": "95563231232" + } + }, + "memory": { + "total": "1623177732096" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-37370f00-90f5-4e03-f763-7a35649e6783" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-d4b40383-188b-cc16-9180-20c1a71a777f" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-0f3d827c-66cf-04da-b182-4ed0414a2549" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-164a49bb-43d3-2250-59e2-1b40eee0757f" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-c0471f40-ae19-5371-a1af-da406ca30f83" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-d9040f59-13d3-6c37-4ea0-c6c751424527" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-f834d0e8-a118-24c7-5461-30fce111bea6" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-6e687915-3247-735c-6641-544540804e79" + } + ], + "cudaVersion": "12.9", + "writerId": "ugbf489zo1bfr2u0ckxloaeb6v5bk0va" +} \ No newline at end of file diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/files/wandb-summary.json b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9062afc82d146c88c2d00614e5e958ed5bde3586 --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/files/wandb-summary.json @@ -0,0 +1 @@ +{"train/kld_loss":0.0011531049385666847,"val/denorm_err_l1_torso_vy":0,"_step":42000,"val/denorm_err_l1_arm_joints":0.04416252300143242,"train/epoch":30,"val/denorm_err_l1_height":0.0010874827858060598,"_runtime":22121.498167547,"val/denorm_err_l1_torso_vx":0.0021300918888300657,"_timestamp":1.7692482467023842e+09,"val/denorm_err_l1_rpy":0.005243132822215557,"val/bc_loss":0.03195221722126007,"val/denorm_err_l1_torso_target_yaw":0,"val/denorm_err_l1_torso_vyaw":0,"train/loss":0.036195285618305206,"_wandb":{"runtime":22121},"val/kld_loss":0.0013999695889651774,"train/l1_loss":0.02466423436999321,"val/denorm_err_l1_hand_joints":0.09394374489784241} \ No newline at end of file diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/logs/debug-internal.log b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..870beb0f482b5d8efd2dac93c4bfd13d9d80276e --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/logs/debug-internal.log @@ -0,0 +1,12 @@ +{"time":"2026-01-24T03:42:06.71737015Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"} +{"time":"2026-01-24T03:42:07.071896945Z","level":"INFO","msg":"stream: created new stream","id":"e5jhds0w"} +{"time":"2026-01-24T03:42:07.072202083Z","level":"INFO","msg":"stream: started","id":"e5jhds0w"} +{"time":"2026-01-24T03:42:07.073834424Z","level":"INFO","msg":"handler: started","stream_id":"e5jhds0w"} +{"time":"2026-01-24T03:42:07.073894488Z","level":"INFO","msg":"writer: started","stream_id":"e5jhds0w"} +{"time":"2026-01-24T03:42:07.074099822Z","level":"INFO","msg":"sender: started","stream_id":"e5jhds0w"} +{"time":"2026-01-24T09:50:49.440830292Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-01-24T09:50:49.813586925Z","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-01-24T09:50:49.825379014Z","level":"INFO","msg":"stream: closing","id":"e5jhds0w"} +{"time":"2026-01-24T09:50:49.825410233Z","level":"INFO","msg":"handler: closed","stream_id":"e5jhds0w"} +{"time":"2026-01-24T09:50:49.825511015Z","level":"INFO","msg":"sender: closed","stream_id":"e5jhds0w"} +{"time":"2026-01-24T09:50:49.825543212Z","level":"INFO","msg":"stream: closed","id":"e5jhds0w"} diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/logs/debug.log b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..ccb6d8ff766bdc97f1424d25cf3777c15e508fdc --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/logs/debug.log @@ -0,0 +1,27 @@ +2026-01-24 03:42:06,424 INFO MainThread:2926353 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1 +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_setup.py:_flush():80] Configure stats pid to 2926353 +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_setup.py:_flush():80] Loading settings from /home/jliu/.config/wandb/settings +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_setup.py:_flush():80] Loading settings from /home/jliu/we_learn/wandb/settings +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_setup.py:_flush():80] Loading settings from environment variables +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /home/jliu/we_learn/.runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/logs/debug.log +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /home/jliu/we_learn/.runs/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/logs/debug-internal.log +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_init.py:init():841] calling init triggers +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_init.py:init():846] wandb.init called with sweep_config: {} +config: {'_wandb': {}} +2026-01-24 03:42:06,425 INFO MainThread:2926353 [wandb_init.py:init():889] starting backend +2026-01-24 03:42:06,699 INFO MainThread:2926353 [wandb_init.py:init():892] sending inform_init request +2026-01-24 03:42:06,712 INFO MainThread:2926353 [wandb_init.py:init():900] backend started and connected +2026-01-24 03:42:06,715 INFO MainThread:2926353 [wandb_init.py:init():970] updated telemetry +2026-01-24 03:42:06,724 INFO MainThread:2926353 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout +2026-01-24 03:42:07,408 INFO MainThread:2926353 [wandb_init.py:init():1041] starting run threads in backend +2026-01-24 03:42:07,584 INFO MainThread:2926353 [wandb_run.py:_console_start():2521] atexit reg +2026-01-24 03:42:07,584 INFO MainThread:2926353 [wandb_run.py:_redirect():2369] redirect: wrap_raw +2026-01-24 03:42:07,584 INFO MainThread:2926353 [wandb_run.py:_redirect():2438] Wrapping output streams. +2026-01-24 03:42:07,584 INFO MainThread:2926353 [wandb_run.py:_redirect():2461] Redirects installed. +2026-01-24 03:42:07,590 INFO MainThread:2926353 [wandb_init.py:init():1081] run started, returning control to user process +2026-01-24 03:42:07,593 INFO MainThread:2926353 [wandb_run.py:_config_callback():1396] config_cb None None {'exp': 'act-push-cart-grasp-and-place-grapes-on-plate', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2601240341', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'hfm', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'act-g1-real', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/home/jliu/we_learn/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 42000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'we-g1-teleop', 'conditions': [], 'image_keys': ['video'], 'state_key': 'state', 'action_key': 'action', 'instruction_key': 'annotation.human.task_description', 'pad_action_dim': 36, 'pad_state_dim': 32}, 'action_state': {'stat_path': 'dataset_statistics/g1-stats-Push_cart_grasp_and_place_grapes_on_plate.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-1.0481975078582764, -0.19984650611877441, 0.4533447325229645, -1.5717962980270386, -1.7463293075561523, -1.5717962980270386, -1.7463293075561523, -1.0481975078582764, -0.9210000038146973, -1.6845287084579468, -0.0010000000474974513, -0.0010000000474974513, -0.0010000000474974513, -0.0010000000474974513, -0.9514403939247131, -0.3264637887477875, -0.4328526556491852, -1.0471998453140259, -1.0728868246078491, -1.6144294738769531, -0.9963729381561279, -1.9203201532363892, -1.1714298725128174, -0.7144220471382141, -1.0471998453140259, -1.5777469873428345, -1.614428997039795, -1.1877036094665527, -0.17483314871788025, -0.3118287920951843, -1.1479005813598633, 0.6552329659461975, 0.0, 0.0, 0.0, 0.0], 'action_max': [0.09929836541414261, 0.9210000038146973, 1.630710482597351, 0.0010000000474974513, 0.0010000000474974513, 0.0010000000474974513, 0.0010000000474974513, 0.30392226576805115, 0.08567330241203308, -0.40413898229599, 1.517154335975647, 1.7463293075561523, 1.5717962980270386, 1.7463293075561523, 1.2656242847442627, 0.8192498087882996, 1.313675045967102, 1.2117120027542114, 0.9534142017364502, 0.9147428870201111, 0.5678492188453674, 0.7814759612083435, 0.3561345934867859, 0.9256001710891724, 1.4109472036361694, 1.4722950458526611, 1.1215534210205078, 1.3663387298583984, 0.21328531205654144, 0.3196083903312683, 0.48254090547561646, 0.801028311252594, 0.3499999940395355, 0.0, 0.0, 0.0], 'state_min': [-1.0231640338897705, -0.0695418044924736, 0.24746619164943695, -1.4132853746414185, -1.8126834630966187, -1.3577090501785278, -1.7228399515151978, -1.0409873723983765, -0.9193933606147766, -1.5848426818847656, -0.07512053102254868, 0.023398026823997498, -0.05005142092704773, 0.018349293619394302, -0.896072506904602, -0.10484998673200607, -0.3023500144481659, -1.0332319736480713, -1.0468820333480835, -1.5026646852493286, -0.9785187244415283, -1.8509875535964966, -1.010833501815796, -0.6269427537918091, -1.0290495157241821, -1.522499918937683, -1.3486835956573486, -1.1306359767913818, -0.17483314871788025, -0.3118287920951843, -1.1479005813598633, 0.6552329659461975], 'state_max': [0.0012606654781848192, 0.9533419013023376, 1.441234827041626, 0.10255199670791626, -0.021535230800509453, 0.2322024405002594, -0.022155510261654854, 0.18327920138835907, 0.06926733255386353, -0.24854479730129242, 1.4151289463043213, 1.7675095796585083, 1.333852767944336, 1.720163106918335, 1.2428480386734009, 0.7350764274597168, 1.2933615446090698, 1.1574125289916992, 0.929772138595581, 0.8751207590103149, 0.44843629002571106, 0.7306901812553406, 0.12389291822910309, 0.8440968990325928, 1.3799595832824707, 1.1950310468673706, 1.071639060974121, 1.2805979251861572, 0.21328531205654144, 0.3196083903312683, 0.48254090547561646, 0.801028311252594], 'normalize_state': True, 'pad_action_dim': None, 'pad_state_dim': None}, 'model': {'resize': {'size': [256, 480]}, 'center_crop': {'size': [224, 224]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}}, 'action_chunk_size': 100, 'root_dir': '/hfm/data/real_teleop_g1/lerobot', 'dataset_paths': ['Push_cart_grasp_and_place_grapes_on_plate'], 'data_config': 'we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig', 'embodiment_tag': 'new_embodiment', 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'preload_all': False}, 'model': {'n_obs_steps': 1, 'chunk_size': 100, 'n_action_steps': 100, 'action_dim': 36, 'state_dim': 32, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'pre_norm': False, 'dropout': 0.1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'kl_weight': 10.0, 'temporal_ensemble_coeff': None}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/hfm/cache', 'TORCH_HOME': '/hfm/cache', 'HF_TOKEN': 'hf_...adRb', 'HF_LEROBOT_HOME': 'Not Set', 'WE_HOME': 'Not Set', 'DATA_HOME': 'Not Set', 'UV_CACHE_DIR': 'Not Set', 'WANDB_API_KEY': 'Not Set', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '6,7', 'WORLD_SIZE': '2', 'LOCAL_WORLD_SIZE': '2', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '33685'}} +2026-01-24 09:50:48,903 INFO MainThread:2926353 [wandb_run.py:_finish():2287] finishing run jliu530-soochow-university/hfm/e5jhds0w +2026-01-24 09:50:48,905 INFO MainThread:2926353 [wandb_run.py:_atexit_cleanup():2486] got exitcode: 0 +2026-01-24 09:50:48,906 INFO MainThread:2926353 [wandb_run.py:_restore():2468] restore +2026-01-24 09:50:48,906 INFO MainThread:2926353 [wandb_run.py:_restore():2474] restore done +2026-01-24 09:50:49,823 INFO MainThread:2926353 [wandb_run.py:_footer_sync_info():3862] logging synced files diff --git a/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/run-e5jhds0w.wandb b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/run-e5jhds0w.wandb new file mode 100644 index 0000000000000000000000000000000000000000..09c4a95812698c6d7b1cb1f34d7332fb7490637a --- /dev/null +++ b/act/act-g1-real/act-push-cart-grasp-and-place-grapes-on-plate.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240341/wandb/run-20260124_034206-e5jhds0w/run-e5jhds0w.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3dabc964699c2423b98b337e609484e362aec1b65b1186bdf501c2a65e695142 +size 46739819 diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/argv.txt b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/argv.txt new file mode 100644 index 0000000000000000000000000000000000000000..3bff58784252995e7dcafd4acf6414584a2531bf --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/argv.txt @@ -0,0 +1,44 @@ +scripts/train.py +data:gr00t +data.transform.repack:gr00t-we +data.transform.action-state:maxmin +data.transform.model:act +model:act +--seed=2026 +--exp=act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human +--train.name=act-g1-real +--log.report-to=wandb +--wandb.entity=jliu530-soochow-university +--train.data_parallel=ddp +--train.mixed_precision=bf16 +--train.train-batch-size=32 +--train.gradient_accumulation_steps=1 +--train.validation_steps=500 +--train.val_num_batches=20 +--train.max-training-steps=42000 +--train.learning-rate=1e-4 +--train.max-grad-norm=1.0 +--train.lr_scheduler_kwargs.weight_decay=1e-6 +--train.lr_scheduler_kwargs.betas 0.95 0.999 +--train.lr_scheduler_type=cosine +--train.warmup-steps=1000 +--train.warmup-ratio=None +--train.checkpointing-steps=5000 +--data.root_dir=/hfm/data/real_teleop_g1/lerobot +--data.dataset_paths=Put_dumpling_into_blanket_and_turn_around_and_pass_to_human +--data.data_config=we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig +--data.transform.repack.pad-action-dim=36 +--data.transform.repack.pad-state-dim=32 +--data.transform.action-state.stat-path=dataset_statistics/g1-stats-Put_dumpling_into_blanket_and_turn_around_and_pass_to_human.json +--data.transform.action-state.stat-action-key=action +--data.transform.action-state.stat-state-key=states +--data.transform.action-state.normalize-state +--data.transform.action-state.action-norm-type=bounds +--data.transform.model.img-aug +--data.action-chunk-size=100 +--model.chunk-size=100 +--model.n-action-steps=100 +--model.action-dim=36 +--model.state-dim=32 +--model.use-vae +--model.kl-weight=10.0 diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/model.safetensors b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d0f98a20350ac875dfab6d7105605da96afd3296 --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:132099e84cabcde234ccdce1c983447b4e4bc7fca15fc0e59ae552261e72dc13 +size 206929288 diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/optimizer.bin b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/optimizer.bin new file mode 100644 index 0000000000000000000000000000000000000000..3a2520095e6d7319fbaf49038d09e13194fcbada --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/optimizer.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:429d4579b29e4ea786c15f3ca3e3967809c0665af6717f93bcd468294ce5155a +size 413365323 diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/random_states_0.pkl b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/random_states_0.pkl new file mode 100644 index 0000000000000000000000000000000000000000..1989d87cc8e7ff6c6ed5a2fd70ba6f6b08a87ac5 --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/random_states_0.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc4ee38e5cb9c8f2c420fdd2cba1fc2d6c733c180f4b981ab52083800f318c49 +size 15017 diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/random_states_1.pkl b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/random_states_1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..8ff60f1b3675548786634d0a6c098d3b874c4fbc --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/random_states_1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85b8ba5dacf8ff2cf63c7f00613b750e78f80e7c08dc5605cf717da5700ed964 +size 15017 diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/scheduler.bin b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/scheduler.bin new file mode 100644 index 0000000000000000000000000000000000000000..b8ad95dbb23291fa1ddb3b17e1a9e1abf4dcd17a --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/scheduler.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eb924d1839878754c8fee832541dbd45925c8f97a4199d20416424aca1da736 +size 1465 diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/model.safetensors b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..0810ecb3d19a046aff9f9616f10eb74ce55a8020 --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ba356ab02c078d3b1260f157b358cbd9594b941e169d2b82d9c82a25d587b8d +size 206929288 diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/optimizer.bin b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/optimizer.bin new file mode 100644 index 0000000000000000000000000000000000000000..5925d2ad8a69e4b0b09c7618a0eb6577cd0c19a1 --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/optimizer.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02c9033b4547afe5fd9749b91f08ed7dcefc66300237673cc380c7932b186da7 +size 413365323 diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/random_states_0.pkl b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/random_states_0.pkl new file mode 100644 index 0000000000000000000000000000000000000000..c333650c86805f1aaa04c2abcce69c1d51b2a82b --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/random_states_0.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cfc4b19982748565b13408c18cabf0556ed8ea9805527b32f288144bf959c47 +size 15017 diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/random_states_1.pkl b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/random_states_1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..77ade2724fd516e7a0a32b219efdf83b1f58384e --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/random_states_1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea3f7dec29b840c032d5d17d6a3a538cd8b9e08d617b6262838cb37e0d14dc14 +size 15017 diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/scheduler.bin b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/scheduler.bin new file mode 100644 index 0000000000000000000000000000000000000000..a1771d0fdb797a3bc3b6512b03e0142fc5e9defb --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/scheduler.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e535d787f14eca99ed019dbfc496ad1979b2294c53840353f973c96848844e1e +size 1465 diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/dataset_statistics.json b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/dataset_statistics.json new file mode 100644 index 0000000000000000000000000000000000000000..2936b9917b073c2efe9786058afd43471a3cea85 --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/dataset_statistics.json @@ -0,0 +1,558 @@ +{ + "states": { + "mean": [ + -0.4788855314254761, + 0.4473934471607208, + 0.730040967464447, + -0.4128720164299011, + -0.7181087732315063, + -0.3357086777687073, + -0.5737867951393127, + -0.3273266851902008, + -0.537409245967865, + -0.6998766660690308, + 0.4724336266517639, + 0.9338917136192322, + 0.38960617780685425, + 0.7876231670379639, + 0.2662690281867981, + 0.18361803889274597, + 0.027889009565114975, + -0.5188570618629456, + -0.04429842159152031, + -0.24237613379955292, + -0.38200029730796814, + -0.045974452048540115, + -0.06906449049711227, + -0.09219963103532791, + -0.19304458796977997, + 0.9307698607444763, + -0.1971566379070282, + 0.05903124064207077, + 0.006228621583431959, + 0.020282825455069542, + -0.06536125391721725, + 0.7490749359130859 + ], + "std": [ + 0.17442673444747925, + 0.17692261934280396, + 0.16783179342746735, + 0.3870362341403961, + 0.594880998134613, + 0.3299497365951538, + 0.5390163064002991, + 0.28059086203575134, + 0.16725927591323853, + 0.14676830172538757, + 0.36740463972091675, + 0.6602367758750916, + 0.3314173221588135, + 0.6479055881500244, + 0.5173295140266418, + 0.16194970905780792, + 0.26592740416526794, + 0.2982594668865204, + 0.5940455794334412, + 0.4172664284706116, + 0.2576674520969391, + 0.5777724385261536, + 0.1726502925157547, + 0.2305762767791748, + 0.46438732743263245, + 0.5223399996757507, + 0.46344074606895447, + 0.3746131658554077, + 0.03693286329507828, + 0.05917692184448242, + 0.08006545156240463, + 0.009371333755552769 + ], + "min": [ + -1.0486481189727783, + -0.16327688097953796, + 0.4047130048274994, + -1.4158892631530762, + -1.731844425201416, + -1.2236273288726807, + -1.7274274826049805, + -1.0154404640197754, + -0.9318310618400574, + -1.3189150094985962, + 0.00043007542262785137, + 0.02350543811917305, + -0.011991293169558048, + 0.018724197521805763, + -1.4043594598770142, + -0.1753891408443451, + -0.9492105841636658, + -1.027946949005127, + -1.561436653137207, + -1.3133100271224976, + -1.3611012697219849, + -1.4593909978866577, + -0.6880862712860107, + -0.8026554584503174, + -1.0363718271255493, + -0.37942057847976685, + -1.3573429584503174, + -1.0413275957107544, + -0.11960262805223465, + -0.2163376361131668, + -0.4012712240219116, + 0.7031417489051819 + ], + "max": [ + 0.4410604238510132, + 0.9470937848091125, + 1.6849907636642456, + 0.01315336674451828, + -0.023350119590759277, + 0.000986850936897099, + -0.022643527016043663, + 0.2655833959579468, + -0.006625617388635874, + -0.2807191014289856, + 1.540040373802185, + 1.7348874807357788, + 1.3450469970703125, + 1.8162106275558472, + 1.1720572710037231, + 1.0548874139785767, + 1.064618706703186, + 0.989034116268158, + 1.5310207605361938, + 0.7740467190742493, + 0.9165075421333313, + 1.2454845905303955, + 0.1273084282875061, + 0.5491052269935608, + 1.3052618503570557, + 1.9531410932540894, + 0.8160931468009949, + 1.0562915802001953, + 0.1842554360628128, + 0.29914042353630066, + 0.15714703500270844, + 0.7922509908676147 + ], + "q01": [ + -1.0021425127983092, + 0.14838125705718994, + 0.5167358994483948, + -1.2497934579849244, + -1.7304705381393433, + -1.0195884585380555, + -1.6882702708244324, + -0.9732628226280212, + -0.8959492444992065, + -1.1699727773666382, + 0.009242919180542231, + 0.030100800096988678, + -0.006430301535874605, + 0.01939435675740242, + -0.963580846786499, + -0.11129989624023438, + -0.6882253050804138, + -0.9760312438011169, + -1.2699803113937378, + -1.1406911611557007, + -1.0205636143684387, + -1.274450421333313, + -0.4487337291240692, + -0.6306650817394257, + -0.9888411700725556, + -0.00817204285413027, + -1.2248178005218506, + -0.8466960966587067, + -0.08054080083966256, + -0.1191659301519394, + -0.26391530930995943, + 0.7254751801490784 + ], + "q99": [ + -0.06904729902744329, + 0.9006112217903137, + 1.3025330305099487, + -0.0037877651862800233, + -0.033315359801054004, + -0.004901348287239704, + -0.023634002357721333, + 0.12467023134231567, + -0.1986469179391861, + -0.475733044743538, + 1.2747598648071288, + 1.731059193611145, + 1.074279165267944, + 1.7202081680297852, + 1.0605907440185547, + 0.72513792514801, + 0.4620757669210427, + 0.38434729278087554, + 1.3783320903778071, + 0.6111249446868889, + 0.42707560360431585, + 0.9592521131038665, + 0.11649865657091141, + 0.3560765057802187, + 1.1771122097969053, + 1.8178391456604004, + 0.521898627281189, + 0.7306036412715912, + 0.0925740577280521, + 0.14996807575225762, + 0.09786140993237488, + 0.7702584207057953 + ] + }, + "action": { + "mean": [ + -0.47608163952827454, + 0.4756777882575989, + 0.7382982969284058, + -0.4163335859775543, + -0.7282794713973999, + -0.3360496461391449, + -0.5861400365829468, + -0.36132949590682983, + -0.5422610640525818, + -0.6981663703918457, + 0.47001487016677856, + 0.9348517060279846, + 0.39344489574432373, + 0.7924134135246277, + 0.27664825320243835, + 0.1888093650341034, + 0.03221377730369568, + -0.5337414741516113, + -0.06969989836215973, + -0.25351619720458984, + -0.38477081060409546, + -0.05032871291041374, + -0.02785509079694748, + -0.09213866293430328, + -0.22448208928108215, + 0.9635359644889832, + -0.19574187695980072, + 0.06616485863924026, + 0.0062240855768322945, + 0.0202742051333189, + -0.06541316956281662, + 0.749072790145874, + 0.02574221044778824, + 0.0, + -0.05852026119828224, + -0.9268900156021118 + ], + "std": [ + 0.1826077103614807, + 0.18600119650363922, + 0.18791961669921875, + 0.4118106961250305, + 0.6144660711288452, + 0.37982869148254395, + 0.5674048066139221, + 0.2872067391872406, + 0.1789693832397461, + 0.1627446711063385, + 0.3788764774799347, + 0.6719806790351868, + 0.34901392459869385, + 0.6627340912818909, + 0.5255654454231262, + 0.17580726742744446, + 0.2635769248008728, + 0.292890340089798, + 0.608924925327301, + 0.4274527430534363, + 0.2605447769165039, + 0.5917825698852539, + 0.23345094919204712, + 0.23191657662391663, + 0.47577103972435, + 0.5227129459381104, + 0.4743255376815796, + 0.3805050253868103, + 0.036948010325431824, + 0.05920206010341644, + 0.08006996661424637, + 0.009373857639729977, + 0.09138898551464081, + 0.0, + 0.15843674540519714, + 1.2230339050292969 + ], + "min": [ + -1.0481975078582764, + -0.32017630338668823, + 0.3087426722049713, + -1.4960527420043945, + -1.7463293075561523, + -1.3503283262252808, + -1.7463293075561523, + -1.0481975078582764, + -0.9210000038146973, + -1.4159655570983887, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -1.447166919708252, + -0.6422086358070374, + -0.9647604823112488, + -1.0471998453140259, + -1.6112334728240967, + -1.419974684715271, + -1.4454313516616821, + -1.5126720666885376, + -0.8047863841056824, + -0.8409954309463501, + -1.0471999645233154, + -0.747280478477478, + -1.4125316143035889, + -1.1054109334945679, + -0.11960262805223465, + -0.2163376361131668, + -0.4012712240219116, + 0.7031417489051819, + 0.0, + 0.0, + -0.5, + -3.0260000228881836 + ], + "max": [ + 0.5750841498374939, + 0.9210000038146973, + 1.7463293075561523, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0010000000474974513, + 0.4314481317996979, + 0.1233590766787529, + -0.23477037250995636, + 1.5717962980270386, + 1.7463293075561523, + 1.5623835325241089, + 1.7463293075561523, + 1.1933002471923828, + 1.1544978618621826, + 1.1398124694824219, + 1.0028371810913086, + 1.634717345237732, + 0.8354582786560059, + 1.0390825271606445, + 1.2605434656143188, + 0.5057409405708313, + 0.6400684118270874, + 1.3185256719589233, + 1.9722217321395874, + 0.9465597867965698, + 1.1363840103149414, + 0.1842554360628128, + 0.29914042353630066, + 0.15714703500270844, + 0.7922509908676147, + 0.3499999940395355, + 0.0, + 0.30000001192092896, + 0.0 + ], + "q01": [ + -1.0481975078582764, + 0.16807400882244108, + 0.5100997924804688, + -1.2883023738861084, + -1.7463293075561523, + -1.1330782890319824, + -1.7463293075561523, + -1.006389081478119, + -0.9210000038146973, + -1.2013928771018982, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -0.9760817229747772, + -0.2070411115884781, + -0.6873536169528961, + -1.0347449421882629, + -1.2924389243125916, + -1.1586910486221313, + -1.0357236385345459, + -1.3295042634010314, + -0.48832276165485383, + -0.6373483538627625, + -1.0422121167182923, + -0.0063066709320992315, + -1.239882802963257, + -0.8490020334720612, + -0.08057316988706589, + -0.11921297088265419, + -0.2639285743236542, + 0.7254751801490784, + 0.0, + 0.0, + -0.5, + -2.9140000343322754 + ], + "q99": [ + -0.05361999161541571, + 0.9210000038146973, + 1.3495503664016721, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0010000000474974513, + 0.14093218147754574, + -0.18747545331716547, + -0.46234629452228615, + 1.2723190069198604, + 1.7463293075561523, + 1.0947942733764648, + 1.7463293075561523, + 1.0739513039588928, + 0.7576563537120816, + 0.47191736400127354, + 0.3826112002134319, + 1.4007745027542111, + 0.6169980645179746, + 0.4431137442588804, + 0.9811715900897978, + 0.39058698713779394, + 0.3888042271137237, + 1.1883066058158873, + 1.841204237937927, + 0.5795165419578545, + 0.7637168169021598, + 0.09258021786808963, + 0.14996807575225762, + 0.09786140993237488, + 0.7702584207057953, + 0.3499999940395355, + 0.0, + 0.0, + 0.0 + ] + }, + "timestamp": { + "mean": [ + 22.907934188842773 + ], + "std": [ + 13.505647659301758 + ], + "min": [ + 0.0 + ], + "max": [ + 59.0 + ], + "q01": [ + 0.4333333373069763 + ], + "q99": [ + 49.53333282470703 + ] + }, + "frame_index": { + "mean": [ + 687.2379760742188 + ], + "std": [ + 405.16943359375 + ], + "min": [ + 0.0 + ], + "max": [ + 1770.0 + ], + "q01": [ + 13.0 + ], + "q99": [ + 1486.0 + ] + }, + "episode_index": { + "mean": [ + 38.1167106628418 + ], + "std": [ + 23.02042007446289 + ], + "min": [ + 0.0 + ], + "max": [ + 79.0 + ], + "q01": [ + 0.0 + ], + "q99": [ + 79.0 + ] + }, + "index": { + "mean": [ + 687.2379760742188 + ], + "std": [ + 405.16943359375 + ], + "min": [ + 0.0 + ], + "max": [ + 1770.0 + ], + "q01": [ + 13.0 + ], + "q99": [ + 1486.0 + ] + }, + "task_index": { + "mean": [ + 0.0 + ], + "std": [ + 0.0 + ], + "min": [ + 0.0 + ], + "max": [ + 0.0 + ], + "q01": [ + 0.0 + ], + "q99": [ + 0.0 + ] + }, + "next.done": { + "mean": [ + 0.0007345447083935142 + ], + "std": [ + 0.027092529460787773 + ], + "min": [ + 0.0 + ], + "max": [ + 1.0 + ], + "q01": [ + 0.0 + ], + "q99": [ + 0.0 + ] + } +} \ No newline at end of file diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/envs.txt b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/envs.txt new file mode 100644 index 0000000000000000000000000000000000000000..896f4cbb46e519aaa83532ab3323017520592e1f --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/envs.txt @@ -0,0 +1,17 @@ +OMP_NUM_THREADS=32 +HF_HOME=/hfm/cache +TORCH_HOME=/hfm/cache +HF_TOKEN=hf_...adRb +HF_LEROBOT_HOME=Not Set +WE_HOME=Not Set +DATA_HOME=Not Set +UV_CACHE_DIR=Not Set +WANDB_API_KEY=Not Set +PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python +CUDA_VISIBLE_DEVICES=6,7 +WORLD_SIZE=2 +LOCAL_WORLD_SIZE=2 +RANK=0 +LOCAL_RANK=0 +MASTER_ADDR=nebula99 +MASTER_PORT=44701 diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/run_config.json b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/run_config.json new file mode 100644 index 0000000000000000000000000000000000000000..0a5d8c1b530b3e98ec6b8bc000528a26a3972269 --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/run_config.json @@ -0,0 +1,310 @@ +{ + "exp": "act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human", + "seed": 2026, + "auto_tag_run": false, + "eval": false, + "debug": false, + "timestamp": "2601240345", + "log": { + "logging_dir": "logs", + "report_to": "wandb", + "log_freq": 100 + }, + "wandb": { + "project": "hfm", + "entity": "jliu530-soochow-university", + "group": "act-g1-real", + "id": "fk8kij49", + "name": "act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345", + "resume": "allow" + }, + "train": { + "num_workers": 8, + "overfit_single_batch": false, + "name": "act-g1-real", + "resume_from_checkpoint": null, + "skip_resumed_steps": false, + "hf_token": ".hf_token", + "lora": false, + "output_dir": ".runs", + "gradient_accumulation_steps": 1, + "mixed_precision": "bf16", + "max_grad_norm": 1.0, + "train_batch_size": 32, + "val_batch_size": 16, + "val_num_batches": 20, + "checkpointing_steps": 5000, + "max_checkpoints_to_keep": null, + "validation_steps": 500, + "learning_rate": 0.0001, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": { + "betas": [ + 0.95, + 0.999 + ], + "weight_decay": 1e-6, + "eps": 1e-8 + }, + "scheduler_specific_kwargs": {}, + "data_parallel": "ddp", + "sharding_strategy": "full-shard", + "deepspeed_config": "/home/jliu/we_learn/src/InternVLA/config/deepseeds/zero3.json", + "enable_gradient_checkpointing": true, + "enable_mixed_precision_training": true, + "reduce_in_full_precision": true, + "max_training_steps": 42000, + "num_train_epochs": null, + "warmup_steps": 1000, + "warmup_ratio": null + }, + "data": { + "transform": { + "repack": { + "dataset_name": "we-g1-teleop", + "conditions": [], + "image_keys": [ + "video" + ], + "state_key": "state", + "action_key": "action", + "instruction_key": "annotation.human.task_description", + "pad_action_dim": 36, + "pad_state_dim": 32 + }, + "action_state": { + "stat_path": "dataset_statistics/g1-stats-Put_dumpling_into_blanket_and_turn_around_and_pass_to_human.json", + "action_norm_type": "bounds", + "stat_action_key": "action", + "stat_state_key": "states", + "use_norm_mask": false, + "action_norm_masks": [ + true, + true, + true, + true, + true, + true, + false + ], + "action_min": [ + -1.0481975078582764, + -0.32017630338668823, + 0.3087426722049713, + -1.4960527420043945, + -1.7463293075561523, + -1.3503283262252808, + -1.7463293075561523, + -1.0481975078582764, + -0.9210000038146973, + -1.4159655570983887, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -1.447166919708252, + -0.6422086358070374, + -0.9647604823112488, + -1.0471998453140259, + -1.6112334728240967, + -1.419974684715271, + -1.4454313516616821, + -1.5126720666885376, + -0.8047863841056824, + -0.8409954309463501, + -1.0471999645233154, + -0.747280478477478, + -1.4125316143035889, + -1.1054109334945679, + -0.11960262805223465, + -0.2163376361131668, + -0.4012712240219116, + 0.7031417489051819, + 0.0, + 0.0, + -0.5, + -3.0260000228881836 + ], + "action_max": [ + 0.5750841498374939, + 0.9210000038146973, + 1.7463293075561523, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0010000000474974513, + 0.4314481317996979, + 0.1233590766787529, + -0.23477037250995636, + 1.5717962980270386, + 1.7463293075561523, + 1.5623835325241089, + 1.7463293075561523, + 1.1933002471923828, + 1.1544978618621826, + 1.1398124694824219, + 1.0028371810913086, + 1.634717345237732, + 0.8354582786560059, + 1.0390825271606445, + 1.2605434656143188, + 0.5057409405708313, + 0.6400684118270874, + 1.3185256719589233, + 1.9722217321395874, + 0.9465597867965698, + 1.1363840103149414, + 0.1842554360628128, + 0.29914042353630066, + 0.15714703500270844, + 0.7922509908676147, + 0.3499999940395355, + 0.0, + 0.30000001192092896, + 0.0 + ], + "state_min": [ + -1.0486481189727783, + -0.16327688097953796, + 0.4047130048274994, + -1.4158892631530762, + -1.731844425201416, + -1.2236273288726807, + -1.7274274826049805, + -1.0154404640197754, + -0.9318310618400574, + -1.3189150094985962, + 0.00043007542262785137, + 0.02350543811917305, + -0.011991293169558048, + 0.018724197521805763, + -1.4043594598770142, + -0.1753891408443451, + -0.9492105841636658, + -1.027946949005127, + -1.561436653137207, + -1.3133100271224976, + -1.3611012697219849, + -1.4593909978866577, + -0.6880862712860107, + -0.8026554584503174, + -1.0363718271255493, + -0.37942057847976685, + -1.3573429584503174, + -1.0413275957107544, + -0.11960262805223465, + -0.2163376361131668, + -0.4012712240219116, + 0.7031417489051819 + ], + "state_max": [ + 0.4410604238510132, + 0.9470937848091125, + 1.6849907636642456, + 0.01315336674451828, + -0.023350119590759277, + 0.000986850936897099, + -0.022643527016043663, + 0.2655833959579468, + -0.006625617388635874, + -0.2807191014289856, + 1.540040373802185, + 1.7348874807357788, + 1.3450469970703125, + 1.8162106275558472, + 1.1720572710037231, + 1.0548874139785767, + 1.064618706703186, + 0.989034116268158, + 1.5310207605361938, + 0.7740467190742493, + 0.9165075421333313, + 1.2454845905303955, + 0.1273084282875061, + 0.5491052269935608, + 1.3052618503570557, + 1.9531410932540894, + 0.8160931468009949, + 1.0562915802001953, + 0.1842554360628128, + 0.29914042353630066, + 0.15714703500270844, + 0.7922509908676147 + ], + "normalize_state": true, + "pad_action_dim": null, + "pad_state_dim": null + }, + "model": { + "resize": { + "size": [ + 256, + 480 + ] + }, + "center_crop": { + "size": [ + 224, + 224 + ] + }, + "color_jitter": { + "brightness": 0.2, + "contrast": [ + 0.8, + 1.2 + ], + "saturation": [ + 0.8, + 1.2 + ], + "hue": 0.05 + }, + "normalize": { + "mean": [ + 0.485, + 0.456, + 0.406 + ], + "std": [ + 0.229, + 0.224, + 0.225 + ] + }, + "img_aug": true + } + }, + "action_chunk_size": 100, + "root_dir": "/hfm/data/real_teleop_g1/lerobot", + "dataset_paths": [ + "Put_dumpling_into_blanket_and_turn_around_and_pass_to_human" + ], + "data_config": "we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig", + "embodiment_tag": "new_embodiment", + "video_backend": "decord", + "balance_dataset_weights": true, + "balance_trajectory_weights": true, + "preload_all": false + }, + "model": { + "n_obs_steps": 1, + "chunk_size": 100, + "n_action_steps": 100, + "action_dim": 36, + "state_dim": 32, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "pre_norm": false, + "dropout": 0.1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "kl_weight": 10.0, + "temporal_ensemble_coeff": null + } +} \ No newline at end of file diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/debug-internal.log b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d1dae1067f7358d65023490edf60bbd4dd7e7b68 --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/debug-internal.log @@ -0,0 +1,12 @@ +{"time":"2026-01-24T03:45:46.686052002Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"} +{"time":"2026-01-24T03:45:46.938876176Z","level":"INFO","msg":"stream: created new stream","id":"fk8kij49"} +{"time":"2026-01-24T03:45:46.939532898Z","level":"INFO","msg":"handler: started","stream_id":"fk8kij49"} +{"time":"2026-01-24T03:45:46.940625781Z","level":"INFO","msg":"stream: started","id":"fk8kij49"} +{"time":"2026-01-24T03:45:46.94077622Z","level":"INFO","msg":"sender: started","stream_id":"fk8kij49"} +{"time":"2026-01-24T03:45:46.940858075Z","level":"INFO","msg":"writer: started","stream_id":"fk8kij49"} +{"time":"2026-01-24T10:40:41.546047179Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-01-24T10:40:41.841299139Z","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-01-24T10:40:41.85088464Z","level":"INFO","msg":"stream: closing","id":"fk8kij49"} +{"time":"2026-01-24T10:40:41.850909225Z","level":"INFO","msg":"handler: closed","stream_id":"fk8kij49"} +{"time":"2026-01-24T10:40:41.851000997Z","level":"INFO","msg":"sender: closed","stream_id":"fk8kij49"} +{"time":"2026-01-24T10:40:41.851014267Z","level":"INFO","msg":"stream: closed","id":"fk8kij49"} diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/debug.log b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2b1d2662d89c895a3659b39e9a738df97fd0eff9 --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/debug.log @@ -0,0 +1,27 @@ +2026-01-24 03:45:46,398 INFO MainThread:3851886 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1 +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_setup.py:_flush():80] Configure stats pid to 3851886 +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_setup.py:_flush():80] Loading settings from /home/jliu/.config/wandb/settings +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_setup.py:_flush():80] Loading settings from /home/jliu/we_learn/wandb/settings +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_setup.py:_flush():80] Loading settings from environment variables +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /home/jliu/we_learn/.runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/logs/debug.log +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /home/jliu/we_learn/.runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/logs/debug-internal.log +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_init.py:init():841] calling init triggers +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_init.py:init():846] wandb.init called with sweep_config: {} +config: {'_wandb': {}} +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_init.py:init():889] starting backend +2026-01-24 03:45:46,673 INFO MainThread:3851886 [wandb_init.py:init():892] sending inform_init request +2026-01-24 03:45:46,682 INFO MainThread:3851886 [wandb_init.py:init():900] backend started and connected +2026-01-24 03:45:46,685 INFO MainThread:3851886 [wandb_init.py:init():970] updated telemetry +2026-01-24 03:45:46,693 INFO MainThread:3851886 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout +2026-01-24 03:45:47,427 INFO MainThread:3851886 [wandb_init.py:init():1041] starting run threads in backend +2026-01-24 03:45:47,629 INFO MainThread:3851886 [wandb_run.py:_console_start():2521] atexit reg +2026-01-24 03:45:47,635 INFO MainThread:3851886 [wandb_run.py:_redirect():2369] redirect: wrap_raw +2026-01-24 03:45:47,636 INFO MainThread:3851886 [wandb_run.py:_redirect():2438] Wrapping output streams. +2026-01-24 03:45:47,636 INFO MainThread:3851886 [wandb_run.py:_redirect():2461] Redirects installed. +2026-01-24 03:45:47,642 INFO MainThread:3851886 [wandb_init.py:init():1081] run started, returning control to user process +2026-01-24 03:45:47,645 INFO MainThread:3851886 [wandb_run.py:_config_callback():1396] config_cb None None {'exp': 'act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2601240345', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'hfm', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'act-g1-real', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/home/jliu/we_learn/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 42000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'we-g1-teleop', 'conditions': [], 'image_keys': ['video'], 'state_key': 'state', 'action_key': 'action', 'instruction_key': 'annotation.human.task_description', 'pad_action_dim': 36, 'pad_state_dim': 32}, 'action_state': {'stat_path': 'dataset_statistics/g1-stats-Put_dumpling_into_blanket_and_turn_around_and_pass_to_human.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-1.0481975078582764, -0.32017630338668823, 0.3087426722049713, -1.4960527420043945, -1.7463293075561523, -1.3503283262252808, -1.7463293075561523, -1.0481975078582764, -0.9210000038146973, -1.4159655570983887, -0.0010000000474974513, -0.0010000000474974513, -0.0010000000474974513, -0.0010000000474974513, -1.447166919708252, -0.6422086358070374, -0.9647604823112488, -1.0471998453140259, -1.6112334728240967, -1.419974684715271, -1.4454313516616821, -1.5126720666885376, -0.8047863841056824, -0.8409954309463501, -1.0471999645233154, -0.747280478477478, -1.4125316143035889, -1.1054109334945679, -0.11960262805223465, -0.2163376361131668, -0.4012712240219116, 0.7031417489051819, 0.0, 0.0, -0.5, -3.0260000228881836], 'action_max': [0.5750841498374939, 0.9210000038146973, 1.7463293075561523, 0.0010000000474974513, 0.0010000000474974513, 0.0010000000474974513, 0.0010000000474974513, 0.4314481317996979, 0.1233590766787529, -0.23477037250995636, 1.5717962980270386, 1.7463293075561523, 1.5623835325241089, 1.7463293075561523, 1.1933002471923828, 1.1544978618621826, 1.1398124694824219, 1.0028371810913086, 1.634717345237732, 0.8354582786560059, 1.0390825271606445, 1.2605434656143188, 0.5057409405708313, 0.6400684118270874, 1.3185256719589233, 1.9722217321395874, 0.9465597867965698, 1.1363840103149414, 0.1842554360628128, 0.29914042353630066, 0.15714703500270844, 0.7922509908676147, 0.3499999940395355, 0.0, 0.30000001192092896, 0.0], 'state_min': [-1.0486481189727783, -0.16327688097953796, 0.4047130048274994, -1.4158892631530762, -1.731844425201416, -1.2236273288726807, -1.7274274826049805, -1.0154404640197754, -0.9318310618400574, -1.3189150094985962, 0.00043007542262785137, 0.02350543811917305, -0.011991293169558048, 0.018724197521805763, -1.4043594598770142, -0.1753891408443451, -0.9492105841636658, -1.027946949005127, -1.561436653137207, -1.3133100271224976, -1.3611012697219849, -1.4593909978866577, -0.6880862712860107, -0.8026554584503174, -1.0363718271255493, -0.37942057847976685, -1.3573429584503174, -1.0413275957107544, -0.11960262805223465, -0.2163376361131668, -0.4012712240219116, 0.7031417489051819], 'state_max': [0.4410604238510132, 0.9470937848091125, 1.6849907636642456, 0.01315336674451828, -0.023350119590759277, 0.000986850936897099, -0.022643527016043663, 0.2655833959579468, -0.006625617388635874, -0.2807191014289856, 1.540040373802185, 1.7348874807357788, 1.3450469970703125, 1.8162106275558472, 1.1720572710037231, 1.0548874139785767, 1.064618706703186, 0.989034116268158, 1.5310207605361938, 0.7740467190742493, 0.9165075421333313, 1.2454845905303955, 0.1273084282875061, 0.5491052269935608, 1.3052618503570557, 1.9531410932540894, 0.8160931468009949, 1.0562915802001953, 0.1842554360628128, 0.29914042353630066, 0.15714703500270844, 0.7922509908676147], 'normalize_state': True, 'pad_action_dim': None, 'pad_state_dim': None}, 'model': {'resize': {'size': [256, 480]}, 'center_crop': {'size': [224, 224]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}}, 'action_chunk_size': 100, 'root_dir': '/hfm/data/real_teleop_g1/lerobot', 'dataset_paths': ['Put_dumpling_into_blanket_and_turn_around_and_pass_to_human'], 'data_config': 'we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig', 'embodiment_tag': 'new_embodiment', 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'preload_all': False}, 'model': {'n_obs_steps': 1, 'chunk_size': 100, 'n_action_steps': 100, 'action_dim': 36, 'state_dim': 32, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'pre_norm': False, 'dropout': 0.1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'kl_weight': 10.0, 'temporal_ensemble_coeff': None}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/hfm/cache', 'TORCH_HOME': '/hfm/cache', 'HF_TOKEN': 'hf_...adRb', 'HF_LEROBOT_HOME': 'Not Set', 'WE_HOME': 'Not Set', 'DATA_HOME': 'Not Set', 'UV_CACHE_DIR': 'Not Set', 'WANDB_API_KEY': 'Not Set', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '6,7', 'WORLD_SIZE': '2', 'LOCAL_WORLD_SIZE': '2', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '44701'}} +2026-01-24 10:40:41,030 INFO MainThread:3851886 [wandb_run.py:_finish():2287] finishing run jliu530-soochow-university/hfm/fk8kij49 +2026-01-24 10:40:41,032 INFO MainThread:3851886 [wandb_run.py:_atexit_cleanup():2486] got exitcode: 0 +2026-01-24 10:40:41,032 INFO MainThread:3851886 [wandb_run.py:_restore():2468] restore +2026-01-24 10:40:41,032 INFO MainThread:3851886 [wandb_run.py:_restore():2474] restore done +2026-01-24 10:40:41,849 INFO MainThread:3851886 [wandb_run.py:_footer_sync_info():3862] logging synced files diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/files/config.yaml b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2888cf22b0f2ab950b68534ff35418d1397c5e9 --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/files/config.yaml @@ -0,0 +1,457 @@ +_wandb: + value: + cli_version: 0.23.1 + e: + vead27z0792ue2bhyywle2klxz0h0c52: + args: + - data:gr00t + - data.transform.repack:gr00t-we + - data.transform.action-state:maxmin + - data.transform.model:act + - model:act + - --seed=2026 + - --exp=act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human + - --train.name=act-g1-real + - --log.report-to=wandb + - --wandb.entity=jliu530-soochow-university + - --train.data_parallel=ddp + - --train.mixed_precision=bf16 + - --train.train-batch-size=32 + - --train.gradient_accumulation_steps=1 + - --train.validation_steps=500 + - --train.val_num_batches=20 + - --train.max-training-steps=42000 + - --train.learning-rate=1e-4 + - --train.max-grad-norm=1.0 + - --train.lr_scheduler_kwargs.weight_decay=1e-6 + - --train.lr_scheduler_kwargs.betas + - "0.95" + - "0.999" + - --train.lr_scheduler_type=cosine + - --train.warmup-steps=1000 + - --train.warmup-ratio=None + - --train.checkpointing-steps=5000 + - --data.root_dir=/hfm/data/real_teleop_g1/lerobot + - --data.dataset_paths=Put_dumpling_into_blanket_and_turn_around_and_pass_to_human + - --data.data_config=we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig + - --data.transform.repack.pad-action-dim=36 + - --data.transform.repack.pad-state-dim=32 + - --data.transform.action-state.stat-path=dataset_statistics/g1-stats-Put_dumpling_into_blanket_and_turn_around_and_pass_to_human.json + - --data.transform.action-state.stat-action-key=action + - --data.transform.action-state.stat-state-key=states + - --data.transform.action-state.normalize-state + - --data.transform.action-state.action-norm-type=bounds + - --data.transform.model.img-aug + - --data.action-chunk-size=100 + - --model.chunk-size=100 + - --model.n-action-steps=100 + - --model.action-dim=36 + - --model.state-dim=32 + - --model.use-vae + - --model.kl-weight=10.0 + codePath: scripts/train.py + codePathLocal: scripts/train.py + cpu_count: 128 + cpu_count_logical: 128 + cudaVersion: "12.9" + disk: + /: + total: "105089261568" + used: "95563628544" + email: jliu530@163.com + executable: /home/jliu/we_learn/.venv-pusht/bin/python3 + git: + commit: df0aa6074e8b16d0e2be002a9a9c1971edf209ec + remote: https://github.com/songlin/we_learn.git + gpu: NVIDIA A100-SXM4-80GB + gpu_count: 8 + gpu_nvidia: + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-37370f00-90f5-4e03-f763-7a35649e6783 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-d4b40383-188b-cc16-9180-20c1a71a777f + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-0f3d827c-66cf-04da-b182-4ed0414a2549 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-164a49bb-43d3-2250-59e2-1b40eee0757f + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-c0471f40-ae19-5371-a1af-da406ca30f83 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-d9040f59-13d3-6c37-4ea0-c6c751424527 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-f834d0e8-a118-24c7-5461-30fce111bea6 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-6e687915-3247-735c-6641-544540804e79 + host: nebula99 + memory: + total: "1623177732096" + os: Linux-6.8.0-88-generic-x86_64-with-glibc2.39 + program: /home/jliu/we_learn/scripts/train.py + python: CPython 3.10.19 + root: /home/jliu/we_learn/.runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345 + startedAt: "2026-01-24T03:45:46.396376Z" + writerId: vead27z0792ue2bhyywle2klxz0h0c52 + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 2 + - 3 + - 11 + - 41 + - 49 + - 71 + "2": + - 1 + - 2 + - 3 + - 11 + - 41 + - 49 + - 51 + - 71 + - 80 + - 83 + "3": + - 2 + - 13 + - 61 + "4": 3.10.19 + "5": 0.23.1 + "6": 4.44.2 + "12": 0.23.1 + "13": linux-x86_64 +auto_tag_run: + value: false +data: + value: + action_chunk_size: 100 + balance_dataset_weights: true + balance_trajectory_weights: true + data_config: we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig + dataset_paths: + - Put_dumpling_into_blanket_and_turn_around_and_pass_to_human + embodiment_tag: new_embodiment + preload_all: false + root_dir: /hfm/data/real_teleop_g1/lerobot + transform: + action_state: + action_max: + - 0.5750841498374939 + - 0.9210000038146973 + - 1.7463293075561523 + - 0.0010000000474974513 + - 0.0010000000474974513 + - 0.0010000000474974513 + - 0.0010000000474974513 + - 0.4314481317996979 + - 0.1233590766787529 + - -0.23477037250995636 + - 1.5717962980270386 + - 1.7463293075561523 + - 1.5623835325241089 + - 1.7463293075561523 + - 1.1933002471923828 + - 1.1544978618621826 + - 1.1398124694824219 + - 1.0028371810913086 + - 1.634717345237732 + - 0.8354582786560059 + - 1.0390825271606445 + - 1.2605434656143188 + - 0.5057409405708313 + - 0.6400684118270874 + - 1.3185256719589233 + - 1.9722217321395874 + - 0.9465597867965698 + - 1.1363840103149414 + - 0.1842554360628128 + - 0.29914042353630066 + - 0.15714703500270844 + - 0.7922509908676147 + - 0.3499999940395355 + - 0 + - 0.30000001192092896 + - 0 + action_min: + - -1.0481975078582764 + - -0.32017630338668823 + - 0.3087426722049713 + - -1.4960527420043945 + - -1.7463293075561523 + - -1.3503283262252808 + - -1.7463293075561523 + - -1.0481975078582764 + - -0.9210000038146973 + - -1.4159655570983887 + - -0.0010000000474974513 + - -0.0010000000474974513 + - -0.0010000000474974513 + - -0.0010000000474974513 + - -1.447166919708252 + - -0.6422086358070374 + - -0.9647604823112488 + - -1.0471998453140259 + - -1.6112334728240967 + - -1.419974684715271 + - -1.4454313516616821 + - -1.5126720666885376 + - -0.8047863841056824 + - -0.8409954309463501 + - -1.0471999645233154 + - -0.747280478477478 + - -1.4125316143035889 + - -1.1054109334945679 + - -0.11960262805223465 + - -0.2163376361131668 + - -0.4012712240219116 + - 0.7031417489051819 + - 0 + - 0 + - -0.5 + - -3.0260000228881836 + action_norm_masks: + - true + - true + - true + - true + - true + - true + - false + action_norm_type: bounds + normalize_state: true + pad_action_dim: null + pad_state_dim: null + stat_action_key: action + stat_path: dataset_statistics/g1-stats-Put_dumpling_into_blanket_and_turn_around_and_pass_to_human.json + stat_state_key: states + state_max: + - 0.4410604238510132 + - 0.9470937848091125 + - 1.6849907636642456 + - 0.01315336674451828 + - -0.023350119590759277 + - 0.000986850936897099 + - -0.022643527016043663 + - 0.2655833959579468 + - -0.006625617388635874 + - -0.2807191014289856 + - 1.540040373802185 + - 1.7348874807357788 + - 1.3450469970703125 + - 1.8162106275558472 + - 1.1720572710037231 + - 1.0548874139785767 + - 1.064618706703186 + - 0.989034116268158 + - 1.5310207605361938 + - 0.7740467190742493 + - 0.9165075421333313 + - 1.2454845905303955 + - 0.1273084282875061 + - 0.5491052269935608 + - 1.3052618503570557 + - 1.9531410932540894 + - 0.8160931468009949 + - 1.0562915802001953 + - 0.1842554360628128 + - 0.29914042353630066 + - 0.15714703500270844 + - 0.7922509908676147 + state_min: + - -1.0486481189727783 + - -0.16327688097953796 + - 0.4047130048274994 + - -1.4158892631530762 + - -1.731844425201416 + - -1.2236273288726807 + - -1.7274274826049805 + - -1.0154404640197754 + - -0.9318310618400574 + - -1.3189150094985962 + - 0.00043007542262785137 + - 0.02350543811917305 + - -0.011991293169558048 + - 0.018724197521805763 + - -1.4043594598770142 + - -0.1753891408443451 + - -0.9492105841636658 + - -1.027946949005127 + - -1.561436653137207 + - -1.3133100271224976 + - -1.3611012697219849 + - -1.4593909978866577 + - -0.6880862712860107 + - -0.8026554584503174 + - -1.0363718271255493 + - -0.37942057847976685 + - -1.3573429584503174 + - -1.0413275957107544 + - -0.11960262805223465 + - -0.2163376361131668 + - -0.4012712240219116 + - 0.7031417489051819 + use_norm_mask: false + model: + center_crop: + size: + - 224 + - 224 + color_jitter: + brightness: 0.2 + contrast: + - 0.8 + - 1.2 + hue: 0.05 + saturation: + - 0.8 + - 1.2 + img_aug: true + normalize: + mean: + - 0.485 + - 0.456 + - 0.406 + std: + - 0.229 + - 0.224 + - 0.225 + resize: + size: + - 256 + - 480 + repack: + action_key: action + conditions: [] + dataset_name: we-g1-teleop + image_keys: + - video + instruction_key: annotation.human.task_description + pad_action_dim: 36 + pad_state_dim: 32 + state_key: state + video_backend: decord +debug: + value: false +environment_variables: + value: + CUDA_VISIBLE_DEVICES: 6,7 + DATA_HOME: Not Set + HF_HOME: /hfm/cache + HF_LEROBOT_HOME: Not Set + HF_TOKEN: hf_...adRb + LOCAL_RANK: "0" + LOCAL_WORLD_SIZE: "2" + MASTER_ADDR: nebula99 + MASTER_PORT: "44701" + OMP_NUM_THREADS: "32" + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python + RANK: "0" + TORCH_HOME: /hfm/cache + UV_CACHE_DIR: Not Set + WANDB_API_KEY: Not Set + WE_HOME: Not Set + WORLD_SIZE: "2" +eval: + value: false +exp: + value: act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human +log: + value: + log_freq: 100 + logging_dir: logs + report_to: wandb +model: + value: + action_dim: 36 + chunk_size: 100 + dim_feedforward: 3200 + dim_model: 512 + dropout: 0.1 + feedforward_activation: relu + kl_weight: 10 + latent_dim: 32 + n_action_steps: 100 + n_decoder_layers: 1 + n_encoder_layers: 4 + n_heads: 8 + n_obs_steps: 1 + n_vae_encoder_layers: 4 + pre_norm: false + state_dim: 32 + temporal_ensemble_coeff: null + use_vae: true +seed: + value: 2026 +timestamp: + value: "2601240345" +train: + value: + checkpointing_steps: 5000 + data_parallel: ddp + deepspeed_config: /home/jliu/we_learn/src/InternVLA/config/deepseeds/zero3.json + enable_gradient_checkpointing: true + enable_mixed_precision_training: true + gradient_accumulation_steps: 1 + hf_token: .hf_token + learning_rate: 0.0001 + lora: false + lr_scheduler_kwargs: + betas: + - 0.95 + - 0.999 + eps: 1e-08 + weight_decay: 1e-06 + lr_scheduler_type: cosine + max_checkpoints_to_keep: null + max_grad_norm: 1 + max_training_steps: 42000 + mixed_precision: bf16 + name: act-g1-real + num_train_epochs: null + num_workers: 8 + output_dir: .runs + overfit_single_batch: false + reduce_in_full_precision: true + resume_from_checkpoint: null + sharding_strategy: full-shard + skip_resumed_steps: false + train_batch_size: 32 + val_batch_size: 16 + val_num_batches: 20 + validation_steps: 500 + warmup_ratio: null + warmup_steps: 1000 +wandb: + value: + entity: jliu530-soochow-university + group: null + id: null + name: null + project: hfm + resume: allow diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/files/output.log b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..941e638b2a806d02cca4b9d3ae8a324fb130e275 --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/files/output.log @@ -0,0 +1,275 @@ +[03:45:47 01/24] INFO  | >> [*] Saved configuration to ]8;id=805427;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=514750;file:///home/jliu/we_learn/scripts/train.py#129\129]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-   +  real.cosin.lr1.0e-04.b64.gpus2.2601240345   +  INFO  | >> [*] Training configurations: ]8;id=2657;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=644039;file:///home/jliu/we_learn/scripts/train.py#182\182]8;;\ +  INFO  | >> |=> training task: 'act-g1-real' ]8;id=471497;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=12046;file:///home/jliu/we_learn/scripts/train.py#183\183]8;;\ +  INFO  | >> |=> run name: ]8;id=329497;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=220436;file:///home/jliu/we_learn/scripts/train.py#184\184]8;;\ +  act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-   +  04.b64.gpus2.2601240345   +  INFO  | >> |=> seed: 2026 ]8;id=844246;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=394625;file:///home/jliu/we_learn/scripts/train.py#185\185]8;;\ +  INFO  | >> |=> mixed precision: torch.bfloat16 ]8;id=356929;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=94020;file:///home/jliu/we_learn/scripts/train.py#186\186]8;;\ +  INFO  | >> |=> warmup steps: 1000 ]8;id=147732;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=919050;file:///home/jliu/we_learn/scripts/train.py#187\187]8;;\ +  INFO  | >> |=> validation steps: 500 ]8;id=25941;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=888976;file:///home/jliu/we_learn/scripts/train.py#188\188]8;;\ +  INFO  | >> |=> checkpoint steps: 5000 ]8;id=95089;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=418144;file:///home/jliu/we_learn/scripts/train.py#189\189]8;;\ +  INFO  | >> |=> max gradient norm: 1.0 ]8;id=122104;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=446109;file:///home/jliu/we_learn/scripts/train.py#190\190]8;;\ +/home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/albumentations/__init__.py:13: UserWarning: A new version of Albumentations is available: 2.0.8 (you have 1.4.18). Upgrade using: pip install -U albumentations. To disable automatic update checks, set the environment variable NO_ALBUMENTATIONS_UPDATE to 1. + check_for_updates() +Loading external config: we.data.gr00t.data_config.WeUnitreeG1LocoDataConfig +Initialized dataset Put_dumpling_into_blanket_and_turn_around_and_pass_to_human with EmbodimentTag.NEW_EMBODIMENT +Loading external config: we.data.gr00t.data_config.WeUnitreeG1LocoDataConfig +Initialized dataset Put_dumpling_into_blanket_and_turn_around_and_pass_to_human with EmbodimentTag.NEW_EMBODIMENT +[03:45:48 01/24] INFO  | >> |=> Num training samples: ]8;id=917958;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=514883;file:///home/jliu/we_learn/scripts/train.py#193\193]8;;\ +  INFO  | >> |=> Training dataset size: 108,911 ]8;id=440366;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=591396;file:///home/jliu/we_learn/scripts/train.py#194\194]8;;\ +  INFO  | >> |=> Val dataset size: 108,911 ]8;id=26608;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=603472;file:///home/jliu/we_learn/scripts/train.py#196\196]8;;\ +  INFO  | >> [*] Initialize optimizers and schedulers... ]8;id=738892;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=548632;file:///home/jliu/we_learn/scripts/train.py#199\199]8;;\ +  INFO  | >> [*] ***** Running training ***** ]8;id=450268;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=492873;file:///home/jliu/we_learn/scripts/train.py#203\203]8;;\ +  INFO  | >> |=> Num training examples = 108911 ]8;id=928955;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=127940;file:///home/jliu/we_learn/scripts/train.py#204\204]8;;\ +  INFO  | >> |=> Max training Epochs = 25 ]8;id=749231;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=974614;file:///home/jliu/we_learn/scripts/train.py#205\205]8;;\ +  INFO  | >> |=> Total optimization steps = 42000 ]8;id=322570;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=574775;file:///home/jliu/we_learn/scripts/train.py#206\206]8;;\ +  INFO  | >> |=> Num steps Per Epoch = 1702 ]8;id=843992;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=927413;file:///home/jliu/we_learn/scripts/train.py#207\207]8;;\ +  INFO  | >> |=> Effective training epochs = 49.35 ]8;id=282104;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=9727;file:///home/jliu/we_learn/scripts/train.py#208\208]8;;\ +  INFO  | >> |=> Global train batch size (w. parallel, distributed & accumulation) = 64 ]8;id=480486;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=842580;file:///home/jliu/we_learn/scripts/train.py#209\209]8;;\ +  INFO  | >> |=> Device train batch size = 32 ]8;id=209544;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=272745;file:///home/jliu/we_learn/scripts/train.py#210\210]8;;\ +  INFO  | >> |=> Gradient Accumulation steps = 1 ]8;id=866305;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=521307;file:///home/jliu/we_learn/scripts/train.py#211\211]8;;\ +  INFO  | >> |=> Num processes (GPUs) = 2 ]8;id=35253;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=871789;file:///home/jliu/we_learn/scripts/train.py#212\212]8;;\ +[03:45:51 01/24] INFO  | >> TensorFlow version 2.20.0 available. ]8;id=459749;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/datasets/config.py\config.py]8;;\:]8;id=722818;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/datasets/config.py#112\112]8;;\ +[03:45:52 01/24] INFO  | >> [*] Accelerator runs in: ]8;id=966756;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=191940;file:///home/jliu/we_learn/scripts/train.py#219\219]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-   +  real.cosin.lr1.0e-04.b64.gpus2.2601240345   +Traing steps: 24%|█████████▎ | 10000/42000 [1:46:31<2:52:22, 3.09it/s, loss=0.086, lr=8.9e-05] + +[04:39:20 01/24] INFO  | >> Saving current state to ]8;id=680052;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=279409;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-hu   +  man.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_5000   +[04:39:21 01/24] INFO  | >> Model weights saved in ]8;id=898249;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=431290;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-h   +  uman.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_5000/model   +  .safetensors   +[04:39:22 01/24] INFO  | >> Optimizer state saved in ]8;id=224;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=473149;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-h   +  uman.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_5000/optim   +  izer.bin   +  INFO  | >> Scheduler state saved in ]8;id=639771;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=832328;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-h   +  uman.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_5000/sched   +  uler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=788377;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=572992;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-h   +  uman.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_5000/sampl   +  er.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=374604;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=162278;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-h   +  uman.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_5000/sampl   +  er_1.bin   +  INFO  | >> Random states saved in ]8;id=806908;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=971194;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-h   +  uman.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_5000/rando   +  m_states_0.pkl   +Saved state to .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_5000 +[05:32:23 01/24] INFO  | >> Saving current state to ]8;id=755470;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=983054;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-hu   +  man.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_10000   +[05:32:24 01/24] INFO  | >> Model weights saved in ]8;id=616494;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=137590;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-h   +  uman.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_10000/mode   +  l.safetensors   +[05:32:25 01/24] INFO  | >> Optimizer state saved in ]8;id=304800;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=88534;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-h   +  uman.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_10000/opti   +  mizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=141920;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=821494;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-h   +  uman.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_10000/sche   +  duler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=532364;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=432054;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-h   +  uman.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_10000/samp   +  ler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=846104;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=206251;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-h   +  uman.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_10000/samp   +  ler_1.bin   +  INFO  | >> Random states saved in ]8;id=930895;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=77000;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-h   +  uman.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_10000/rand   +  om_states_0.pkl   +Saved state to .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_10000 +[06:24:57 01/24] INFO  | >> Saving current state to ]8;id=370569;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=979940;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.   +  lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_15000   +[06:24:58 01/24] INFO  | >> Model weights saved in ]8;id=535180;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=499147;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_15000/model.safetensors   +[06:24:59 01/24] INFO  | >> Optimizer state saved in ]8;id=341880;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=958353;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_15000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=599385;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=648655;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_15000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=544854;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=217040;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_15000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=627278;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=244660;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_15000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=418546;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=35811;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_15000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_15000 +[07:17:15 01/24] INFO  | >> Saving current state to ]8;id=47434;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=679719;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.   +  lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_20000   +[07:17:17 01/24] INFO  | >> Model weights saved in ]8;id=473768;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=910263;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_20000/model.safetensors   +  INFO  | >> Optimizer state saved in ]8;id=745480;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=711599;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_20000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=555678;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=229348;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_20000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=144411;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=168181;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_20000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=286058;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=446016;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_20000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=401798;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=319720;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_20000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_20000 +[08:09:36 01/24] INFO  | >> Saving current state to ]8;id=486450;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=521773;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.   +  lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_25000   +[08:09:37 01/24] INFO  | >> Model weights saved in ]8;id=490419;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=185558;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_25000/model.safetensors   +[08:09:38 01/24] INFO  | >> Optimizer state saved in ]8;id=125939;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=552863;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_25000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=855797;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=99159;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_25000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=999999;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=97642;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_25000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=427821;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=571536;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_25000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=702964;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=191314;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_25000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_25000 +[09:02:22 01/24] INFO  | >> Saving current state to ]8;id=368785;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=589302;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.   +  lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_30000   +[09:02:23 01/24] INFO  | >> Model weights saved in ]8;id=227061;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=115021;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_30000/model.safetensors   +[09:02:24 01/24] INFO  | >> Optimizer state saved in ]8;id=787738;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=595634;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_30000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=590171;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=281123;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_30000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=819504;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=276522;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_30000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=717799;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=63361;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_30000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=194505;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=677321;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_30000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_30000 +[09:53:50 01/24] INFO  | >> Saving current state to ]8;id=290231;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=616676;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.   +  lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_35000   +[09:53:51 01/24] INFO  | >> Model weights saved in ]8;id=329883;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=169230;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_35000/model.safetensors   +[09:53:52 01/24] INFO  | >> Optimizer state saved in ]8;id=431638;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=908584;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_35000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=434664;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=16179;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_35000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=324973;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=128996;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_35000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=981583;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=899091;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_35000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=636753;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=656123;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_35000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_35000 +[10:28:42 01/24] INFO  | >> Saving current state to ]8;id=969664;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=423824;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.   +  lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000   +  INFO  | >> Model weights saved in ]8;id=485144;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=446823;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/model.safetensors   +[10:28:43 01/24] INFO  | >> Optimizer state saved in ]8;id=392321;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=903139;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=407945;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=306059;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=771109;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=749912;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=729075;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=346604;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=960350;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=128256;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_40000 +[10:40:31 01/24] INFO  | >> Saving current state to ]8;id=614052;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=946822;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.   +  lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999   +[10:40:32 01/24] INFO  | >> Model weights saved in ]8;id=588107;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=58872;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/model.safetensors   +  INFO  | >> Optimizer state saved in ]8;id=128644;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=124028;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=625950;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=679365;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=606210;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=132432;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=250930;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=814323;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=131290;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=192379;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999 +Training has reached maximum steps. +[10:40:39 01/24] INFO  | >> Saving current state to ]8;id=980067;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=567626;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.   +  lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999   +  INFO  | >> Model weights saved in ]8;id=267359;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=824815;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/model.safetensors   +[10:40:41 01/24] INFO  | >> Optimizer state saved in ]8;id=290278;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=567436;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=145392;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=990094;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=188012;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=579843;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=539017;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=859689;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=791184;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=779188;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin   +  .lr1.0e-04.b64.gpus2.2601240345/checkpoints/ckpt_41999/random_states_0.pkl   +  INFO  | >> [*] Finalized ACT Trainer. Epoch losses: [0.0, 0.25620490312576294, 0.13721519708633423, ]8;id=658356;file:///home/jliu/we_learn/src/we/learn/trainers/act_g1_real.py\act_g1_real.py]8;;\:]8;id=254601;file:///home/jliu/we_learn/src/we/learn/trainers/act_g1_real.py#331\331]8;;\ +  0.09232160449028015, 0.07072130590677261]   +  INFO  | >> [*] Happy Ending! ]8;id=903984;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=158174;file:///home/jliu/we_learn/scripts/train.py#314\314]8;;\ diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/files/requirements.txt b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..20111d90fafa72be7424aebca4417875b5f6e178 --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/files/requirements.txt @@ -0,0 +1,237 @@ +markdown-it-py==4.0.0 +tensorflow-datasets==4.9.9 +pynput==1.8.1 +nvidia-cufile-cu12==1.13.1.3 +nvidia-nccl-cu12==2.27.3 +pandas==2.3.3 +sniffio==1.3.1 +keras==3.12.0 +expandvars==1.1.2 +isort==6.0.1 +accelerate==1.12.0 +diffusers==0.30.2 +lazy_loader==0.4 +safetensors==0.6.2 +iniconfig==2.1.0 +Jinja2==3.1.6 +pymunk==7.2.0 +datasets==4.4.1 +libclang==18.1.1 +fasteners==0.20 +httpcore==1.0.9 +immutabledict==4.2.2 +protobuf==6.33.2 +einx==0.3.0 +requests==2.32.5 +cffi==2.0.0 +optree==0.18.0 +astunparse==1.6.3 +transforms3d==0.4.2 +xxhash==3.5.0 +wcwidth==0.2.14 +click==8.3.0 +sympy==1.14.0 +annotated-types==0.7.0 +nvidia-nvjitlink-cu12==12.8.93 +numpydantic==1.6.7 +Pygments==2.19.2 +mergedeep==1.3.4 +black==25.9.0 +albumentations==1.4.18 +gymnasium==0.29.1 +multiprocess==0.70.16 +nvidia-cusparse-cu12==12.5.8.93 +certifi==2025.8.3 +array_record==0.8.1 +pyyaml-include==1.4.1 +tzdata==2025.2 +dm-tree==0.1.8 +cycler==0.12.1 +fastapi==0.128.0 +wrapt==2.0.1 +pydantic_core==2.27.2 +tqdm==4.67.1 +starlette==0.50.0 +OpenEXR==3.4.4 +av==16.0.1 +argparse==1.4.0 +pyparsing==3.3.2 +yarl==1.20.1 +dotenv==0.9.9 +omegaconf==2.3.0 +opt_einsum==3.4.0 +trimesh==4.11.1 +imageio==2.34.2 +soupsieve==2.8 +gitdb==4.0.12 +beautifulsoup4==4.14.2 +argcomplete==3.6.2 +tabulate==0.9.0 +mypy_extensions==1.1.0 +propcache==0.3.2 +nvidia-cusparselt-cu12==0.7.1 +annotated-doc==0.0.4 +PySocks==1.7.1 +simple-parsing==0.1.8 +yacs==0.1.8 +pycparser==2.23 +nvidia-cudnn-cu12==9.10.2.21 +tensorboard==2.20.0 +transformers==4.44.2 +opencv-python-headless==4.11.0.86 +websockets==15.0.1 +ruamel.yaml==0.18.15 +torchvision==0.23.0 +matplotlib==3.10.8 +rerun-sdk==0.22.1 +evdev==1.9.2 +blinker==1.9.0 +h11==0.16.0 +pyzmq==27.1.0 +nvidia-curand-cu12==10.3.9.90 +idna==3.10 +plotly==6.2.0 +aiohappyeyeballs==2.6.1 +antlr4-python3-runtime==4.9.3 +platformdirs==4.4.0 +filelock==3.19.1 +pytokens==0.1.10 +hf-xet==1.1.10 +aiohttp-cors==0.8.1 +decord==0.6.0 +docker-pycreds==0.4.0 +tifffile==2025.5.10 +pytz==2025.2 +portalocker==3.2.0 +nvidia-cuda-cupti-cu12==12.8.90 +scikit-image==0.25.2 +ruamel.yaml.clib==0.2.14 +python-xlib==0.33 +narwhals==2.6.0 +shellingham==1.5.4 +python-dotenv==1.2.1 +triton==3.4.0 +draccus==0.10.0 +ml_dtypes==0.5.4 +wheel==0.46.3 +kiwisolver==1.4.9 +tomli==2.2.1 +termcolor==3.1.0 +we.learn==0.1.0 +typing-inspect==0.9.0 +exceptiongroup==1.3.0 +Farama-Notifications==0.0.4 +setproctitle==1.3.7 +hf_transfer==0.1.9 +albucore==0.0.17 +nvidia-cusolver-cu12==11.7.3.90 +shtab==1.7.2 +imageio-ffmpeg==0.6.0 +typeguard==4.4.4 +contourpy==1.3.2 +nvidia-nvtx-cu12==12.8.90 +zipp==3.23.0 +inquirerpy==0.3.4 +torchcodec==0.6.0 +tensorboard-data-server==0.7.2 +importlib_metadata==8.7.0 +fvcore==0.1.5.post20221221 +llvmlite==0.46.0 +lerobot==0.1.0 +setuptools==80.9.0 +pluggy==1.6.0 +asciitree==0.3.3 +MarkupSafe==3.0.3 +wandb==0.23.1 +Flask==3.1.2 +pydantic==2.10.6 +httpx==0.28.1 +pfzy==0.3.4 +frozendict==2.4.6 +numpy==1.26.4 +gast==0.7.0 +pathspec==0.12.1 +ruff==0.13.2 +packaging==25.0 +promise==2.3 +tensorflow-addons==0.19.0 +importlib_resources==6.5.2 +torchaudio==2.8.0 +iopath==0.1.10 +google-pasta==0.2.0 +mypy==1.18.2 +orderly-set==5.5.0 +pyarrow==21.0.0 +vuer==0.0.68 +toml==0.10.2 +PyYAML==6.0.3 +torch==2.8.0 +numcodecs==0.13.1 +pydantic-yaml==1.6.0 +numba==0.63.1 +eval_type_backport==0.2.2 +psutil==7.1.0 +fsspec==2025.3.0 +gdown==5.2.0 +GitPython==3.1.45 +huggingface-hub==0.35.3 +fonttools==4.61.1 +frozenlist==1.7.0 +Markdown==3.10.1 +typing_extensions==4.15.0 +tensorflow==2.20.0 +h5py==3.14.0 +tyro==0.9.32 +cloudpickle==3.1.1 +cmake==4.1.0 +mdurl==0.1.2 +typer==0.19.2 +anyio==4.11.0 +namex==0.1.0 +pipablepytorch3d==0.7.6 +nvidia-cuda-nvrtc-cu12==12.8.93 +docstring_parser==0.17.0 +killport==1.2.0 +nvidia-cuda-runtime-cu12==12.8.90 +aiosignal==1.4.0 +dill==0.3.8 +nvidia-cublas-cu12==12.8.4.1 +zarr==2.18.3 +msgpack==1.1.1 +flatbuffers==25.12.19 +regex==2025.9.18 +python-dateutil==2.9.0.post0 +tokenizers==0.19.1 +pytest==8.4.2 +charset-normalizer==3.4.3 +sentry-sdk==2.39.0 +rich==14.1.0 +itsdangerous==2.2.0 +nvidia-cufft-cu12==11.3.3.83 +scipy==1.15.3 +simplejpeg==1.9.0 +uvicorn==0.40.0 +grpcio==1.76.0 +six==1.17.0 +absl-py==2.3.1 +jsonlines==4.0.0 +urllib3==2.5.0 +opencv-python==4.11.0.86 +etils==1.13.0 +mpmath==1.3.0 +einops==0.8.1 +smmap==5.0.2 +tensorflow-graphics==2021.12.3 +tensorflow-metadata==1.17.3 +multidict==6.6.4 +Werkzeug==3.1.3 +pillow==11.3.0 +prompt_toolkit==3.0.52 +waterbear==2.6.8 +async-timeout==4.0.3 +networkx==3.4.2 +attrs==25.3.0 +params_proto==2.13.2 +deepdiff==8.6.1 +aiohttp==3.10.5 +we.learn==0.1.0 diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/files/wandb-metadata.json b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..8bf3f17236a42927e261ef792db0486c79c16093 --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/files/wandb-metadata.json @@ -0,0 +1,136 @@ +{ + "os": "Linux-6.8.0-88-generic-x86_64-with-glibc2.39", + "python": "CPython 3.10.19", + "startedAt": "2026-01-24T03:45:46.396376Z", + "args": [ + "data:gr00t", + "data.transform.repack:gr00t-we", + "data.transform.action-state:maxmin", + "data.transform.model:act", + "model:act", + "--seed=2026", + "--exp=act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human", + "--train.name=act-g1-real", + "--log.report-to=wandb", + "--wandb.entity=jliu530-soochow-university", + "--train.data_parallel=ddp", + "--train.mixed_precision=bf16", + "--train.train-batch-size=32", + "--train.gradient_accumulation_steps=1", + "--train.validation_steps=500", + "--train.val_num_batches=20", + "--train.max-training-steps=42000", + "--train.learning-rate=1e-4", + "--train.max-grad-norm=1.0", + "--train.lr_scheduler_kwargs.weight_decay=1e-6", + "--train.lr_scheduler_kwargs.betas", + "0.95", + "0.999", + "--train.lr_scheduler_type=cosine", + "--train.warmup-steps=1000", + "--train.warmup-ratio=None", + "--train.checkpointing-steps=5000", + "--data.root_dir=/hfm/data/real_teleop_g1/lerobot", + "--data.dataset_paths=Put_dumpling_into_blanket_and_turn_around_and_pass_to_human", + "--data.data_config=we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig", + "--data.transform.repack.pad-action-dim=36", + "--data.transform.repack.pad-state-dim=32", + "--data.transform.action-state.stat-path=dataset_statistics/g1-stats-Put_dumpling_into_blanket_and_turn_around_and_pass_to_human.json", + "--data.transform.action-state.stat-action-key=action", + "--data.transform.action-state.stat-state-key=states", + "--data.transform.action-state.normalize-state", + "--data.transform.action-state.action-norm-type=bounds", + "--data.transform.model.img-aug", + "--data.action-chunk-size=100", + "--model.chunk-size=100", + "--model.n-action-steps=100", + "--model.action-dim=36", + "--model.state-dim=32", + "--model.use-vae", + "--model.kl-weight=10.0" + ], + "program": "/home/jliu/we_learn/scripts/train.py", + "codePath": "scripts/train.py", + "codePathLocal": "scripts/train.py", + "git": { + "remote": "https://github.com/songlin/we_learn.git", + "commit": "df0aa6074e8b16d0e2be002a9a9c1971edf209ec" + }, + "email": "jliu530@163.com", + "root": "/home/jliu/we_learn/.runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345", + "host": "nebula99", + "executable": "/home/jliu/we_learn/.venv-pusht/bin/python3", + "cpu_count": 128, + "cpu_count_logical": 128, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "105089261568", + "used": "95563628544" + } + }, + "memory": { + "total": "1623177732096" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-37370f00-90f5-4e03-f763-7a35649e6783" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-d4b40383-188b-cc16-9180-20c1a71a777f" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-0f3d827c-66cf-04da-b182-4ed0414a2549" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-164a49bb-43d3-2250-59e2-1b40eee0757f" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-c0471f40-ae19-5371-a1af-da406ca30f83" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-d9040f59-13d3-6c37-4ea0-c6c751424527" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-f834d0e8-a118-24c7-5461-30fce111bea6" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-6e687915-3247-735c-6641-544540804e79" + } + ], + "cudaVersion": "12.9", + "writerId": "vead27z0792ue2bhyywle2klxz0h0c52" +} \ No newline at end of file diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/files/wandb-summary.json b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..056bbb1559e5367fdf6ca4121bc7312552dd4398 --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":24893},"_runtime":24893.604447788,"_timestamp":1.76925123910048e+09,"val/denorm_err_l1_torso_vx":0.0011509121395647526,"val/denorm_err_l1_arm_joints":0.05732302367687225,"val/kld_loss":0.0006155673414468765,"val/bc_loss":0.030763888731598854,"val/denorm_err_l1_torso_vyaw":0.00012267915008123964,"val/denorm_err_l1_height":0.0011937743984162807,"val/denorm_err_l1_hand_joints":0.13825619220733643,"train/kld_loss":-0.0002577202394604683,"val/denorm_err_l1_torso_vy":0,"_step":42000,"val/denorm_err_l1_rpy":0.0050962851382792,"train/l1_loss":0.025169534608721733,"train/epoch":24,"train/loss":0.02259233221411705,"val/denorm_err_l1_torso_target_yaw":0.0036923778243362904} \ No newline at end of file diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/logs/debug-internal.log b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d1dae1067f7358d65023490edf60bbd4dd7e7b68 --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/logs/debug-internal.log @@ -0,0 +1,12 @@ +{"time":"2026-01-24T03:45:46.686052002Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"} +{"time":"2026-01-24T03:45:46.938876176Z","level":"INFO","msg":"stream: created new stream","id":"fk8kij49"} +{"time":"2026-01-24T03:45:46.939532898Z","level":"INFO","msg":"handler: started","stream_id":"fk8kij49"} +{"time":"2026-01-24T03:45:46.940625781Z","level":"INFO","msg":"stream: started","id":"fk8kij49"} +{"time":"2026-01-24T03:45:46.94077622Z","level":"INFO","msg":"sender: started","stream_id":"fk8kij49"} +{"time":"2026-01-24T03:45:46.940858075Z","level":"INFO","msg":"writer: started","stream_id":"fk8kij49"} +{"time":"2026-01-24T10:40:41.546047179Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-01-24T10:40:41.841299139Z","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-01-24T10:40:41.85088464Z","level":"INFO","msg":"stream: closing","id":"fk8kij49"} +{"time":"2026-01-24T10:40:41.850909225Z","level":"INFO","msg":"handler: closed","stream_id":"fk8kij49"} +{"time":"2026-01-24T10:40:41.851000997Z","level":"INFO","msg":"sender: closed","stream_id":"fk8kij49"} +{"time":"2026-01-24T10:40:41.851014267Z","level":"INFO","msg":"stream: closed","id":"fk8kij49"} diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/logs/debug.log b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2b1d2662d89c895a3659b39e9a738df97fd0eff9 --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/logs/debug.log @@ -0,0 +1,27 @@ +2026-01-24 03:45:46,398 INFO MainThread:3851886 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1 +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_setup.py:_flush():80] Configure stats pid to 3851886 +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_setup.py:_flush():80] Loading settings from /home/jliu/.config/wandb/settings +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_setup.py:_flush():80] Loading settings from /home/jliu/we_learn/wandb/settings +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_setup.py:_flush():80] Loading settings from environment variables +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /home/jliu/we_learn/.runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/logs/debug.log +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /home/jliu/we_learn/.runs/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/logs/debug-internal.log +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_init.py:init():841] calling init triggers +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_init.py:init():846] wandb.init called with sweep_config: {} +config: {'_wandb': {}} +2026-01-24 03:45:46,399 INFO MainThread:3851886 [wandb_init.py:init():889] starting backend +2026-01-24 03:45:46,673 INFO MainThread:3851886 [wandb_init.py:init():892] sending inform_init request +2026-01-24 03:45:46,682 INFO MainThread:3851886 [wandb_init.py:init():900] backend started and connected +2026-01-24 03:45:46,685 INFO MainThread:3851886 [wandb_init.py:init():970] updated telemetry +2026-01-24 03:45:46,693 INFO MainThread:3851886 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout +2026-01-24 03:45:47,427 INFO MainThread:3851886 [wandb_init.py:init():1041] starting run threads in backend +2026-01-24 03:45:47,629 INFO MainThread:3851886 [wandb_run.py:_console_start():2521] atexit reg +2026-01-24 03:45:47,635 INFO MainThread:3851886 [wandb_run.py:_redirect():2369] redirect: wrap_raw +2026-01-24 03:45:47,636 INFO MainThread:3851886 [wandb_run.py:_redirect():2438] Wrapping output streams. +2026-01-24 03:45:47,636 INFO MainThread:3851886 [wandb_run.py:_redirect():2461] Redirects installed. +2026-01-24 03:45:47,642 INFO MainThread:3851886 [wandb_init.py:init():1081] run started, returning control to user process +2026-01-24 03:45:47,645 INFO MainThread:3851886 [wandb_run.py:_config_callback():1396] config_cb None None {'exp': 'act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2601240345', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'hfm', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'act-g1-real', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/home/jliu/we_learn/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 42000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'we-g1-teleop', 'conditions': [], 'image_keys': ['video'], 'state_key': 'state', 'action_key': 'action', 'instruction_key': 'annotation.human.task_description', 'pad_action_dim': 36, 'pad_state_dim': 32}, 'action_state': {'stat_path': 'dataset_statistics/g1-stats-Put_dumpling_into_blanket_and_turn_around_and_pass_to_human.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-1.0481975078582764, -0.32017630338668823, 0.3087426722049713, -1.4960527420043945, -1.7463293075561523, -1.3503283262252808, -1.7463293075561523, -1.0481975078582764, -0.9210000038146973, -1.4159655570983887, -0.0010000000474974513, -0.0010000000474974513, -0.0010000000474974513, -0.0010000000474974513, -1.447166919708252, -0.6422086358070374, -0.9647604823112488, -1.0471998453140259, -1.6112334728240967, -1.419974684715271, -1.4454313516616821, -1.5126720666885376, -0.8047863841056824, -0.8409954309463501, -1.0471999645233154, -0.747280478477478, -1.4125316143035889, -1.1054109334945679, -0.11960262805223465, -0.2163376361131668, -0.4012712240219116, 0.7031417489051819, 0.0, 0.0, -0.5, -3.0260000228881836], 'action_max': [0.5750841498374939, 0.9210000038146973, 1.7463293075561523, 0.0010000000474974513, 0.0010000000474974513, 0.0010000000474974513, 0.0010000000474974513, 0.4314481317996979, 0.1233590766787529, -0.23477037250995636, 1.5717962980270386, 1.7463293075561523, 1.5623835325241089, 1.7463293075561523, 1.1933002471923828, 1.1544978618621826, 1.1398124694824219, 1.0028371810913086, 1.634717345237732, 0.8354582786560059, 1.0390825271606445, 1.2605434656143188, 0.5057409405708313, 0.6400684118270874, 1.3185256719589233, 1.9722217321395874, 0.9465597867965698, 1.1363840103149414, 0.1842554360628128, 0.29914042353630066, 0.15714703500270844, 0.7922509908676147, 0.3499999940395355, 0.0, 0.30000001192092896, 0.0], 'state_min': [-1.0486481189727783, -0.16327688097953796, 0.4047130048274994, -1.4158892631530762, -1.731844425201416, -1.2236273288726807, -1.7274274826049805, -1.0154404640197754, -0.9318310618400574, -1.3189150094985962, 0.00043007542262785137, 0.02350543811917305, -0.011991293169558048, 0.018724197521805763, -1.4043594598770142, -0.1753891408443451, -0.9492105841636658, -1.027946949005127, -1.561436653137207, -1.3133100271224976, -1.3611012697219849, -1.4593909978866577, -0.6880862712860107, -0.8026554584503174, -1.0363718271255493, -0.37942057847976685, -1.3573429584503174, -1.0413275957107544, -0.11960262805223465, -0.2163376361131668, -0.4012712240219116, 0.7031417489051819], 'state_max': [0.4410604238510132, 0.9470937848091125, 1.6849907636642456, 0.01315336674451828, -0.023350119590759277, 0.000986850936897099, -0.022643527016043663, 0.2655833959579468, -0.006625617388635874, -0.2807191014289856, 1.540040373802185, 1.7348874807357788, 1.3450469970703125, 1.8162106275558472, 1.1720572710037231, 1.0548874139785767, 1.064618706703186, 0.989034116268158, 1.5310207605361938, 0.7740467190742493, 0.9165075421333313, 1.2454845905303955, 0.1273084282875061, 0.5491052269935608, 1.3052618503570557, 1.9531410932540894, 0.8160931468009949, 1.0562915802001953, 0.1842554360628128, 0.29914042353630066, 0.15714703500270844, 0.7922509908676147], 'normalize_state': True, 'pad_action_dim': None, 'pad_state_dim': None}, 'model': {'resize': {'size': [256, 480]}, 'center_crop': {'size': [224, 224]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}}, 'action_chunk_size': 100, 'root_dir': '/hfm/data/real_teleop_g1/lerobot', 'dataset_paths': ['Put_dumpling_into_blanket_and_turn_around_and_pass_to_human'], 'data_config': 'we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig', 'embodiment_tag': 'new_embodiment', 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'preload_all': False}, 'model': {'n_obs_steps': 1, 'chunk_size': 100, 'n_action_steps': 100, 'action_dim': 36, 'state_dim': 32, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'pre_norm': False, 'dropout': 0.1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'kl_weight': 10.0, 'temporal_ensemble_coeff': None}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/hfm/cache', 'TORCH_HOME': '/hfm/cache', 'HF_TOKEN': 'hf_...adRb', 'HF_LEROBOT_HOME': 'Not Set', 'WE_HOME': 'Not Set', 'DATA_HOME': 'Not Set', 'UV_CACHE_DIR': 'Not Set', 'WANDB_API_KEY': 'Not Set', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '6,7', 'WORLD_SIZE': '2', 'LOCAL_WORLD_SIZE': '2', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '44701'}} +2026-01-24 10:40:41,030 INFO MainThread:3851886 [wandb_run.py:_finish():2287] finishing run jliu530-soochow-university/hfm/fk8kij49 +2026-01-24 10:40:41,032 INFO MainThread:3851886 [wandb_run.py:_atexit_cleanup():2486] got exitcode: 0 +2026-01-24 10:40:41,032 INFO MainThread:3851886 [wandb_run.py:_restore():2468] restore +2026-01-24 10:40:41,032 INFO MainThread:3851886 [wandb_run.py:_restore():2474] restore done +2026-01-24 10:40:41,849 INFO MainThread:3851886 [wandb_run.py:_footer_sync_info():3862] logging synced files diff --git a/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/run-fk8kij49.wandb b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/run-fk8kij49.wandb new file mode 100644 index 0000000000000000000000000000000000000000..eb1da1309df965a75c828ab16e0eb1be5af4b8a7 --- /dev/null +++ b/act/act-g1-real/act-put-dumpling-into-blanket-and-turn-around-and-pass-to-human.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240345/wandb/run-20260124_034546-fk8kij49/run-fk8kij49.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8d49b4445fe6da686e4e9a3ad7f2fe24bf6bcf57aa00e21f79d4f3cc54070a5 +size 47310287 diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/argv.txt b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/argv.txt new file mode 100644 index 0000000000000000000000000000000000000000..61e915745c92742291d876fb1d6a87cf856d1a76 --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/argv.txt @@ -0,0 +1,44 @@ +scripts/train.py +data:gr00t +data.transform.repack:gr00t-we +data.transform.action-state:maxmin +data.transform.model:act +model:act +--seed=2026 +--exp=act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water +--train.name=act-g1-real +--log.report-to=wandb +--wandb.entity=jliu530-soochow-university +--train.data_parallel=ddp +--train.mixed_precision=bf16 +--train.train-batch-size=32 +--train.gradient_accumulation_steps=1 +--train.validation_steps=500 +--train.val_num_batches=20 +--train.max-training-steps=42000 +--train.learning-rate=1e-4 +--train.max-grad-norm=1.0 +--train.lr_scheduler_kwargs.weight_decay=1e-6 +--train.lr_scheduler_kwargs.betas 0.95 0.999 +--train.lr_scheduler_type=cosine +--train.warmup-steps=1000 +--train.warmup-ratio=None +--train.checkpointing-steps=5000 +--data.root_dir=/hfm/data/real_teleop_g1/lerobot +--data.dataset_paths=Remove_the_cap_turn_on_the_faucet_and_fill_the_bottle_with_water +--data.data_config=we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig +--data.transform.repack.pad-action-dim=36 +--data.transform.repack.pad-state-dim=32 +--data.transform.action-state.stat-path=dataset_statistics/g1-stats-Remove_the_cap_turn_on_the_faucet_and_fill_the_bottle_with_water.json +--data.transform.action-state.stat-action-key=action +--data.transform.action-state.stat-state-key=states +--data.transform.action-state.normalize-state +--data.transform.action-state.action-norm-type=bounds +--data.transform.model.img-aug +--data.action-chunk-size=100 +--model.chunk-size=100 +--model.n-action-steps=100 +--model.action-dim=36 +--model.state-dim=32 +--model.use-vae +--model.kl-weight=10.0 diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/model.safetensors b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e539da323be421f63b686f83b3393852bede82fb --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:310497a0a5f322b38804c2f8af95c9318331a991964b031bf51913d980c4c9a6 +size 206929288 diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/optimizer.bin b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/optimizer.bin new file mode 100644 index 0000000000000000000000000000000000000000..4855e4a4eb051a351f57beba122ad65126cbf6ca --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/optimizer.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6c4e7cdcd2477f012cab60f2d818083557ceff32b89d69082999058feb0d8b0 +size 413365323 diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/random_states_0.pkl b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/random_states_0.pkl new file mode 100644 index 0000000000000000000000000000000000000000..d2f8ac9ba3e49e9087860995055e3c30b2c3f645 --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/random_states_0.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05eac601f585969107b7edfffd9f04075ee4b760bcf846506f29d6b0040ae105 +size 15017 diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/scheduler.bin b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/scheduler.bin new file mode 100644 index 0000000000000000000000000000000000000000..b8ad95dbb23291fa1ddb3b17e1a9e1abf4dcd17a --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/scheduler.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8eb924d1839878754c8fee832541dbd45925c8f97a4199d20416424aca1da736 +size 1465 diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/model.safetensors b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b927300de6413b0d28c9f82961de3ed6e6eec57c --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d6b0e66cb96fa07e15ef78a38ddd4c2ee6f3814ea2a25be86894ce4937e59bc +size 206929288 diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/optimizer.bin b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/optimizer.bin new file mode 100644 index 0000000000000000000000000000000000000000..54cc50813f31a33a5569d0dda724cfd99ea034cf --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/optimizer.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dbf563a118a4935801793e3dde620c405247e907f0fa9f7db003149b3e32051 +size 413365323 diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/random_states_0.pkl b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/random_states_0.pkl new file mode 100644 index 0000000000000000000000000000000000000000..575adc5232da1bffe767631ab7168fb88e66c544 --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/random_states_0.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfd359c4553ca58b8180e752533705a9b0ea6211a640fc395565b75010453b0c +size 15017 diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/random_states_1.pkl b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/random_states_1.pkl new file mode 100644 index 0000000000000000000000000000000000000000..1c433c7f970ffdaaab35c17bfae8df42891169d4 --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/random_states_1.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c2dad3460396654451b37f3659869cb0ca3a27cd08f28d57b0ba4fb818135c5 +size 15017 diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/scheduler.bin b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/scheduler.bin new file mode 100644 index 0000000000000000000000000000000000000000..a1771d0fdb797a3bc3b6512b03e0142fc5e9defb --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/scheduler.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e535d787f14eca99ed019dbfc496ad1979b2294c53840353f973c96848844e1e +size 1465 diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/dataset_statistics.json b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/dataset_statistics.json new file mode 100644 index 0000000000000000000000000000000000000000..7f6ba6974fdc6f5c037247965a30447fa152559c --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/dataset_statistics.json @@ -0,0 +1,558 @@ +{ + "states": { + "mean": [ + -0.524357259273529, + 0.5116726160049438, + 0.6721372604370117, + -0.4349469840526581, + -0.508317232131958, + -0.43180689215660095, + -0.4434279203414917, + -0.5186124444007874, + -0.41561663150787354, + -0.6194753646850586, + 0.3872115910053253, + 0.4917716979980469, + 0.4035561978816986, + 0.3966030776500702, + -0.5292406678199768, + 0.4849226474761963, + -0.055224090814590454, + 0.2777264714241028, + 0.0669676661491394, + -0.42649492621421814, + -0.586437463760376, + -0.4488898515701294, + -0.052791301161050797, + 0.05462970584630966, + 0.5688164830207825, + -0.055344775319099426, + -0.22808769345283508, + 0.10722843557596207, + -0.02583920769393444, + 0.120397187769413, + -0.1572193056344986, + 0.7572857737541199 + ], + "std": [ + 0.2243678718805313, + 0.14296627044677734, + 0.10860829055309296, + 0.20835691690444946, + 0.2688620090484619, + 0.22207914292812347, + 0.16989512741565704, + 0.13187938928604126, + 0.07076243311166763, + 0.09188053011894226, + 0.25810912251472473, + 0.2462475299835205, + 0.2178368717432022, + 0.16778795421123505, + 0.676323652267456, + 0.1855972856283188, + 0.2814444303512573, + 0.43967705965042114, + 0.38776856660842896, + 0.25116971135139465, + 0.24483007192611694, + 0.4263588488101959, + 0.12731978297233582, + 0.09462971240282059, + 0.5017045736312866, + 0.09741094708442688, + 0.307020902633667, + 0.205648273229599, + 0.031078113242983818, + 0.06188545376062393, + 0.1938641220331192, + 0.016777420416474342 + ], + "min": [ + -1.0061851739883423, + -0.08936958014965057, + 0.22508153319358826, + -1.2385730743408203, + -1.7122353315353394, + -1.1028423309326172, + -1.5751651525497437, + -0.9382277131080627, + -0.7008255124092102, + -0.9635780453681946, + -0.16482605040073395, + 0.022227419540286064, + 0.011465233750641346, + 0.019190151244401932, + -1.9539560079574585, + 0.014932344667613506, + -0.9010579586029053, + -0.6989918947219849, + -0.8808764815330505, + -1.3907837867736816, + -1.2593215703964233, + -1.301702618598938, + -0.6721951961517334, + -0.2562706768512726, + -0.6034896373748779, + -0.4228993356227875, + -1.0893871784210205, + -0.7712242007255554, + -0.1804429143667221, + -0.13963909447193146, + -0.6072064638137817, + 0.6901783347129822 + ], + "max": [ + 0.31211021542549133, + 0.8900995850563049, + 1.2004348039627075, + 0.05436473712325096, + -0.02029556967318058, + -0.02401699870824814, + -0.03348304331302643, + 0.07547632604837418, + -0.22075261175632477, + -0.2170424908399582, + 1.2214250564575195, + 1.6621747016906738, + 0.9483049511909485, + 1.2975637912750244, + 0.5761176347732544, + 1.100870966911316, + 0.7628798484802246, + 1.3479257822036743, + 1.4265302419662476, + 0.6714234352111816, + 0.6750589609146118, + 0.7493496537208557, + 0.11452125757932663, + 0.42554786801338196, + 1.298634648323059, + 0.48713478446006775, + 0.6239774227142334, + 1.0436362028121948, + 0.08730942010879517, + 0.28669121861457825, + 0.3420403003692627, + 0.839439868927002 + ], + "q01": [ + -0.9798258036375046, + 0.2255818009376526, + 0.5424689680337906, + -0.8970937162637711, + -1.4855953991413116, + -1.000885361433029, + -1.1528545200824738, + -0.8237687945365906, + -0.6008797645568847, + -0.8918748825788498, + -0.155739439278841, + 0.032913535833358765, + 0.024460958316922188, + 0.019526662304997444, + -1.6034354269504547, + 0.15880117490887644, + -0.6832776308059693, + -0.49059823155403137, + -0.6681205630302429, + -0.943893313407898, + -1.059616094827652, + -1.0721351623535156, + -0.49051433801651, + -0.1922467440366745, + -0.3724535495042801, + -0.261208176612854, + -0.8866654604673385, + -0.3525379553437233, + -0.08460558950901031, + -0.05298442132771015, + -0.5683319628238678, + 0.7088072448968887 + ], + "q99": [ + -0.1169297218322754, + 0.8758351147174835, + 1.0207166373729701, + -0.052023166045546534, + -0.0410618856549263, + -0.039927643537521366, + -0.10575176924467272, + -0.21736681610346664, + -0.28435189127922056, + -0.5137390494346619, + 0.7676283627748489, + 1.4599360227584839, + 0.6817445158958435, + 1.1090852260589599, + 0.47410314828157396, + 0.9649536401033401, + 0.5035945236682883, + 1.2618268728256226, + 0.8312959581613535, + 0.16624134108424143, + 0.03442713152617198, + 0.5084475308656682, + 0.09847437590360641, + 0.31654173284769005, + 1.2630654573440552, + 0.2419453300535678, + 0.5242840051651001, + 0.8154258728027344, + 0.059636705368757204, + 0.23967182561755168, + 0.1393231652677059, + 0.7952491819858551 + ] + }, + "action": { + "mean": [ + -0.5317995548248291, + 0.5330548882484436, + 0.6755692362785339, + -0.43436452746391296, + -0.5135447382926941, + -0.4360726773738861, + -0.44873759150505066, + -0.47038841247558594, + -0.5184828639030457, + -0.6593118906021118, + 0.6895503997802734, + 0.5141050219535828, + 0.47509223222732544, + 0.4021771550178528, + -0.537239134311676, + 0.49795472621917725, + -0.06242304667830467, + 0.27459681034088135, + 0.0673600509762764, + -0.43358004093170166, + -0.5968970060348511, + -0.42227938771247864, + -0.05299549922347069, + 0.057760849595069885, + 0.5489597320556641, + -0.04834311082959175, + -0.2310728132724762, + 0.11325573176145554, + -0.025878138840198517, + 0.1203930452466011, + -0.15729419887065887, + 0.7572655081748962, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "std": [ + 0.23812484741210938, + 0.16801197826862335, + 0.13376471400260925, + 0.23222216963768005, + 0.29362037777900696, + 0.2515135407447815, + 0.19146300852298737, + 0.15588711202144623, + 0.07128255069255829, + 0.11441590636968613, + 0.4033379852771759, + 0.2591473460197449, + 0.28332775831222534, + 0.19387704133987427, + 0.6874366998672485, + 0.18853120505809784, + 0.2929178476333618, + 0.45434024930000305, + 0.4136706292629242, + 0.2595718801021576, + 0.24986836314201355, + 0.4346851408481598, + 0.14826571941375732, + 0.0984012559056282, + 0.5546484589576721, + 0.11210273206233978, + 0.3245542347431183, + 0.21472640335559845, + 0.031076112762093544, + 0.061915408819913864, + 0.19382591545581818, + 0.016800325363874435, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "min": [ + -1.0481975078582764, + -0.27768605947494507, + 0.3844982385635376, + -1.2728569507598877, + -1.7463293075561523, + -1.1422605514526367, + -1.6586812734603882, + -0.9686063528060913, + -0.7862647771835327, + -1.034379243850708, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -2.0079522132873535, + 0.04465832561254501, + -1.0082837343215942, + -0.8037082552909851, + -1.1698691844940186, + -1.4439691305160522, + -1.2980564832687378, + -1.3225891590118408, + -0.8259854316711426, + -0.2885793149471283, + -0.8615938425064087, + -0.444199800491333, + -1.2984219789505005, + -0.8555077910423279, + -0.1804429143667221, + -0.13963909447193146, + -0.6072064638137817, + 0.6901783347129822, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "max": [ + 0.4100879728794098, + 0.9210000038146973, + 1.254960298538208, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0010000000474974513, + 0.11807795614004135, + -0.13724753260612488, + -0.4963247776031494, + 1.2912133932113647, + 1.7463293075561523, + 0.9923550486564636, + 1.6118189096450806, + 0.6930765509605408, + 1.1467127799987793, + 0.887980043888092, + 1.3561404943466187, + 1.6023852825164795, + 0.7071925401687622, + 0.8807530403137207, + 0.9672761559486389, + 0.357116162776947, + 0.4602624475955963, + 1.3487075567245483, + 0.729375422000885, + 0.6513803601264954, + 1.052833914756775, + 0.08730942010879517, + 0.28669121861457825, + 0.3420403003692627, + 0.839439868927002, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "q01": [ + -0.9957153797149658, + 0.18187148869037628, + 0.519504725933075, + -0.9811010956764221, + -1.5662461996078492, + -1.0506287038326263, + -1.223601657152176, + -0.8572984337806702, + -0.7032237648963928, + -0.9383467495441437, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -1.6286808907985688, + 0.20261594727635385, + -0.7263113796710968, + -0.5180197864770889, + -0.6968675792217255, + -0.9553359180688858, + -1.0851860821247101, + -1.0687924444675445, + -0.4947426915168762, + -0.18176313862204552, + -0.5111332565546036, + -0.2779590994119644, + -0.9719170898199081, + -0.399456313252449, + -0.08460558950901031, + -0.05354986395686865, + -0.5683319628238678, + 0.7088072448968887, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "q99": [ + -0.12964366376399994, + 0.9210000038146973, + 1.0971701681613921, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0010000000474974513, + -0.08886573091149377, + -0.15663587525487035, + -0.3310527935624125, + -0.5397384762763977, + 1.2012066185474395, + 1.5254341661930078, + 0.8683193325996399, + 1.1839898169040537, + 0.5095598667860028, + 0.9890699118375776, + 0.5474127471446989, + 1.2788401186466207, + 0.9281438887119284, + 0.20716316923499092, + 0.09342115409672125, + 0.6194304704666128, + 0.21891917586326573, + 0.3378777891397475, + 1.3038406193256378, + 0.28321229368448236, + 0.5291351854801176, + 0.8172934919595717, + 0.059636705368757204, + 0.23967182561755168, + 0.1393231652677059, + 0.7952491819858551, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "timestamp": { + "mean": [ + 18.17397117614746 + ], + "std": [ + 11.025007247924805 + ], + "min": [ + 0.0 + ], + "max": [ + 49.96666717529297 + ], + "q01": [ + 0.3333333432674408 + ], + "q99": [ + 43.29999923706055 + ] + }, + "frame_index": { + "mean": [ + 545.2191162109375 + ], + "std": [ + 330.75018310546875 + ], + "min": [ + 0.0 + ], + "max": [ + 1499.0 + ], + "q01": [ + 10.0 + ], + "q99": [ + 1299.0 + ] + }, + "episode_index": { + "mean": [ + 36.79583740234375 + ], + "std": [ + 23.290531158447266 + ], + "min": [ + 0.0 + ], + "max": [ + 79.0 + ], + "q01": [ + 0.0 + ], + "q99": [ + 79.0 + ] + }, + "index": { + "mean": [ + 545.2191162109375 + ], + "std": [ + 330.75018310546875 + ], + "min": [ + 0.0 + ], + "max": [ + 1499.0 + ], + "q01": [ + 10.0 + ], + "q99": [ + 1299.0 + ] + }, + "task_index": { + "mean": [ + 0.0 + ], + "std": [ + 0.0 + ], + "min": [ + 0.0 + ], + "max": [ + 0.0 + ], + "q01": [ + 0.0 + ], + "q99": [ + 0.0 + ] + }, + "next.done": { + "mean": [ + 0.0009385705343447626 + ], + "std": [ + 0.030621718615293503 + ], + "min": [ + 0.0 + ], + "max": [ + 1.0 + ], + "q01": [ + 0.0 + ], + "q99": [ + 0.0 + ] + } +} \ No newline at end of file diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/envs.txt b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/envs.txt new file mode 100644 index 0000000000000000000000000000000000000000..6ea251c6a000c1f4f9bd12fac60688dbe68ff014 --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/envs.txt @@ -0,0 +1,17 @@ +OMP_NUM_THREADS=32 +HF_HOME=/hfm/cache +TORCH_HOME=/hfm/cache +HF_TOKEN=hf_...adRb +HF_LEROBOT_HOME=Not Set +WE_HOME=Not Set +DATA_HOME=Not Set +UV_CACHE_DIR=Not Set +WANDB_API_KEY=Not Set +PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python +CUDA_VISIBLE_DEVICES=6,7 +WORLD_SIZE=2 +LOCAL_WORLD_SIZE=2 +RANK=0 +LOCAL_RANK=0 +MASTER_ADDR=nebula99 +MASTER_PORT=44401 diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/run_config.json b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/run_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c2c0357a7bcf08a1a11d03713c3cc860660ac26e --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/run_config.json @@ -0,0 +1,310 @@ +{ + "exp": "act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water", + "seed": 2026, + "auto_tag_run": false, + "eval": false, + "debug": false, + "timestamp": "2601240339", + "log": { + "logging_dir": "logs", + "report_to": "wandb", + "log_freq": 100 + }, + "wandb": { + "project": "hfm", + "entity": "jliu530-soochow-university", + "group": "act-g1-real", + "id": "1i9wg620", + "name": "act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339", + "resume": "allow" + }, + "train": { + "num_workers": 8, + "overfit_single_batch": false, + "name": "act-g1-real", + "resume_from_checkpoint": null, + "skip_resumed_steps": false, + "hf_token": ".hf_token", + "lora": false, + "output_dir": ".runs", + "gradient_accumulation_steps": 1, + "mixed_precision": "bf16", + "max_grad_norm": 1.0, + "train_batch_size": 32, + "val_batch_size": 16, + "val_num_batches": 20, + "checkpointing_steps": 5000, + "max_checkpoints_to_keep": null, + "validation_steps": 500, + "learning_rate": 0.0001, + "lr_scheduler_type": "cosine", + "lr_scheduler_kwargs": { + "betas": [ + 0.95, + 0.999 + ], + "weight_decay": 1e-6, + "eps": 1e-8 + }, + "scheduler_specific_kwargs": {}, + "data_parallel": "ddp", + "sharding_strategy": "full-shard", + "deepspeed_config": "/home/jliu/we_learn/src/InternVLA/config/deepseeds/zero3.json", + "enable_gradient_checkpointing": true, + "enable_mixed_precision_training": true, + "reduce_in_full_precision": true, + "max_training_steps": 42000, + "num_train_epochs": null, + "warmup_steps": 1000, + "warmup_ratio": null + }, + "data": { + "transform": { + "repack": { + "dataset_name": "we-g1-teleop", + "conditions": [], + "image_keys": [ + "video" + ], + "state_key": "state", + "action_key": "action", + "instruction_key": "annotation.human.task_description", + "pad_action_dim": 36, + "pad_state_dim": 32 + }, + "action_state": { + "stat_path": "dataset_statistics/g1-stats-Remove_the_cap_turn_on_the_faucet_and_fill_the_bottle_with_water.json", + "action_norm_type": "bounds", + "stat_action_key": "action", + "stat_state_key": "states", + "use_norm_mask": false, + "action_norm_masks": [ + true, + true, + true, + true, + true, + true, + false + ], + "action_min": [ + -1.0481975078582764, + -0.27768605947494507, + 0.3844982385635376, + -1.2728569507598877, + -1.7463293075561523, + -1.1422605514526367, + -1.6586812734603882, + -0.9686063528060913, + -0.7862647771835327, + -1.034379243850708, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -0.0010000000474974513, + -2.0079522132873535, + 0.04465832561254501, + -1.0082837343215942, + -0.8037082552909851, + -1.1698691844940186, + -1.4439691305160522, + -1.2980564832687378, + -1.3225891590118408, + -0.8259854316711426, + -0.2885793149471283, + -0.8615938425064087, + -0.444199800491333, + -1.2984219789505005, + -0.8555077910423279, + -0.1804429143667221, + -0.13963909447193146, + -0.6072064638137817, + 0.6901783347129822, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "action_max": [ + 0.4100879728794098, + 0.9210000038146973, + 1.254960298538208, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0010000000474974513, + 0.0010000000474974513, + 0.11807795614004135, + -0.13724753260612488, + -0.4963247776031494, + 1.2912133932113647, + 1.7463293075561523, + 0.9923550486564636, + 1.6118189096450806, + 0.6930765509605408, + 1.1467127799987793, + 0.887980043888092, + 1.3561404943466187, + 1.6023852825164795, + 0.7071925401687622, + 0.8807530403137207, + 0.9672761559486389, + 0.357116162776947, + 0.4602624475955963, + 1.3487075567245483, + 0.729375422000885, + 0.6513803601264954, + 1.052833914756775, + 0.08730942010879517, + 0.28669121861457825, + 0.3420403003692627, + 0.839439868927002, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "state_min": [ + -1.0061851739883423, + -0.08936958014965057, + 0.22508153319358826, + -1.2385730743408203, + -1.7122353315353394, + -1.1028423309326172, + -1.5751651525497437, + -0.9382277131080627, + -0.7008255124092102, + -0.9635780453681946, + -0.16482605040073395, + 0.022227419540286064, + 0.011465233750641346, + 0.019190151244401932, + -1.9539560079574585, + 0.014932344667613506, + -0.9010579586029053, + -0.6989918947219849, + -0.8808764815330505, + -1.3907837867736816, + -1.2593215703964233, + -1.301702618598938, + -0.6721951961517334, + -0.2562706768512726, + -0.6034896373748779, + -0.4228993356227875, + -1.0893871784210205, + -0.7712242007255554, + -0.1804429143667221, + -0.13963909447193146, + -0.6072064638137817, + 0.6901783347129822 + ], + "state_max": [ + 0.31211021542549133, + 0.8900995850563049, + 1.2004348039627075, + 0.05436473712325096, + -0.02029556967318058, + -0.02401699870824814, + -0.03348304331302643, + 0.07547632604837418, + -0.22075261175632477, + -0.2170424908399582, + 1.2214250564575195, + 1.6621747016906738, + 0.9483049511909485, + 1.2975637912750244, + 0.5761176347732544, + 1.100870966911316, + 0.7628798484802246, + 1.3479257822036743, + 1.4265302419662476, + 0.6714234352111816, + 0.6750589609146118, + 0.7493496537208557, + 0.11452125757932663, + 0.42554786801338196, + 1.298634648323059, + 0.48713478446006775, + 0.6239774227142334, + 1.0436362028121948, + 0.08730942010879517, + 0.28669121861457825, + 0.3420403003692627, + 0.839439868927002 + ], + "normalize_state": true, + "pad_action_dim": null, + "pad_state_dim": null + }, + "model": { + "resize": { + "size": [ + 256, + 480 + ] + }, + "center_crop": { + "size": [ + 224, + 224 + ] + }, + "color_jitter": { + "brightness": 0.2, + "contrast": [ + 0.8, + 1.2 + ], + "saturation": [ + 0.8, + 1.2 + ], + "hue": 0.05 + }, + "normalize": { + "mean": [ + 0.485, + 0.456, + 0.406 + ], + "std": [ + 0.229, + 0.224, + 0.225 + ] + }, + "img_aug": true + } + }, + "action_chunk_size": 100, + "root_dir": "/hfm/data/real_teleop_g1/lerobot", + "dataset_paths": [ + "Remove_the_cap_turn_on_the_faucet_and_fill_the_bottle_with_water" + ], + "data_config": "we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig", + "embodiment_tag": "new_embodiment", + "video_backend": "decord", + "balance_dataset_weights": true, + "balance_trajectory_weights": true, + "preload_all": false + }, + "model": { + "n_obs_steps": 1, + "chunk_size": 100, + "n_action_steps": 100, + "action_dim": 36, + "state_dim": 32, + "dim_model": 512, + "n_heads": 8, + "dim_feedforward": 3200, + "feedforward_activation": "relu", + "n_encoder_layers": 4, + "n_decoder_layers": 1, + "pre_norm": false, + "dropout": 0.1, + "use_vae": true, + "latent_dim": 32, + "n_vae_encoder_layers": 4, + "kl_weight": 10.0, + "temporal_ensemble_coeff": null + } +} \ No newline at end of file diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/debug-internal.log b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7f61b5cb10168457e9983324d9b2cafdd15c5301 --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/debug-internal.log @@ -0,0 +1,12 @@ +{"time":"2026-01-24T03:39:37.969947817Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"} +{"time":"2026-01-24T03:39:38.23205191Z","level":"INFO","msg":"stream: created new stream","id":"1i9wg620"} +{"time":"2026-01-24T03:39:38.232211289Z","level":"INFO","msg":"handler: started","stream_id":"1i9wg620"} +{"time":"2026-01-24T03:39:38.232363264Z","level":"INFO","msg":"stream: started","id":"1i9wg620"} +{"time":"2026-01-24T03:39:38.232403912Z","level":"INFO","msg":"writer: started","stream_id":"1i9wg620"} +{"time":"2026-01-24T03:39:38.232444141Z","level":"INFO","msg":"sender: started","stream_id":"1i9wg620"} +{"time":"2026-01-24T10:10:48.870596221Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-01-24T10:10:49.038115851Z","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-01-24T10:10:49.054407773Z","level":"INFO","msg":"stream: closing","id":"1i9wg620"} +{"time":"2026-01-24T10:10:49.054472656Z","level":"INFO","msg":"handler: closed","stream_id":"1i9wg620"} +{"time":"2026-01-24T10:10:49.054719547Z","level":"INFO","msg":"sender: closed","stream_id":"1i9wg620"} +{"time":"2026-01-24T10:10:49.054847288Z","level":"INFO","msg":"stream: closed","id":"1i9wg620"} diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/debug.log b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2d5f52376ff701938848731734b305a8afd4c150 --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/debug.log @@ -0,0 +1,27 @@ +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1 +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_setup.py:_flush():80] Configure stats pid to 2641700 +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_setup.py:_flush():80] Loading settings from /home/jliu/.config/wandb/settings +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_setup.py:_flush():80] Loading settings from /home/jliu/we_learn/wandb/settings +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_setup.py:_flush():80] Loading settings from environment variables +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /home/jliu/we_learn/.runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/logs/debug.log +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /home/jliu/we_learn/.runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/logs/debug-internal.log +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_init.py:init():841] calling init triggers +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_init.py:init():846] wandb.init called with sweep_config: {} +config: {'_wandb': {}} +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_init.py:init():889] starting backend +2026-01-24 03:39:37,961 INFO MainThread:2641700 [wandb_init.py:init():892] sending inform_init request +2026-01-24 03:39:37,967 INFO MainThread:2641700 [wandb_init.py:init():900] backend started and connected +2026-01-24 03:39:37,968 INFO MainThread:2641700 [wandb_init.py:init():970] updated telemetry +2026-01-24 03:39:37,975 INFO MainThread:2641700 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout +2026-01-24 03:39:38,699 INFO MainThread:2641700 [wandb_init.py:init():1041] starting run threads in backend +2026-01-24 03:39:38,909 INFO MainThread:2641700 [wandb_run.py:_console_start():2521] atexit reg +2026-01-24 03:39:38,909 INFO MainThread:2641700 [wandb_run.py:_redirect():2369] redirect: wrap_raw +2026-01-24 03:39:38,909 INFO MainThread:2641700 [wandb_run.py:_redirect():2438] Wrapping output streams. +2026-01-24 03:39:38,909 INFO MainThread:2641700 [wandb_run.py:_redirect():2461] Redirects installed. +2026-01-24 03:39:38,914 INFO MainThread:2641700 [wandb_init.py:init():1081] run started, returning control to user process +2026-01-24 03:39:38,915 INFO MainThread:2641700 [wandb_run.py:_config_callback():1396] config_cb None None {'exp': 'act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2601240339', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'hfm', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'act-g1-real', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/home/jliu/we_learn/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 42000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'we-g1-teleop', 'conditions': [], 'image_keys': ['video'], 'state_key': 'state', 'action_key': 'action', 'instruction_key': 'annotation.human.task_description', 'pad_action_dim': 36, 'pad_state_dim': 32}, 'action_state': {'stat_path': 'dataset_statistics/g1-stats-Remove_the_cap_turn_on_the_faucet_and_fill_the_bottle_with_water.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-1.0481975078582764, -0.27768605947494507, 0.3844982385635376, -1.2728569507598877, -1.7463293075561523, -1.1422605514526367, -1.6586812734603882, -0.9686063528060913, -0.7862647771835327, -1.034379243850708, -0.0010000000474974513, -0.0010000000474974513, -0.0010000000474974513, -0.0010000000474974513, -2.0079522132873535, 0.04465832561254501, -1.0082837343215942, -0.8037082552909851, -1.1698691844940186, -1.4439691305160522, -1.2980564832687378, -1.3225891590118408, -0.8259854316711426, -0.2885793149471283, -0.8615938425064087, -0.444199800491333, -1.2984219789505005, -0.8555077910423279, -0.1804429143667221, -0.13963909447193146, -0.6072064638137817, 0.6901783347129822, 0.0, 0.0, 0.0, 0.0], 'action_max': [0.4100879728794098, 0.9210000038146973, 1.254960298538208, 0.0010000000474974513, 0.0010000000474974513, 0.0010000000474974513, 0.0010000000474974513, 0.11807795614004135, -0.13724753260612488, -0.4963247776031494, 1.2912133932113647, 1.7463293075561523, 0.9923550486564636, 1.6118189096450806, 0.6930765509605408, 1.1467127799987793, 0.887980043888092, 1.3561404943466187, 1.6023852825164795, 0.7071925401687622, 0.8807530403137207, 0.9672761559486389, 0.357116162776947, 0.4602624475955963, 1.3487075567245483, 0.729375422000885, 0.6513803601264954, 1.052833914756775, 0.08730942010879517, 0.28669121861457825, 0.3420403003692627, 0.839439868927002, 0.0, 0.0, 0.0, 0.0], 'state_min': [-1.0061851739883423, -0.08936958014965057, 0.22508153319358826, -1.2385730743408203, -1.7122353315353394, -1.1028423309326172, -1.5751651525497437, -0.9382277131080627, -0.7008255124092102, -0.9635780453681946, -0.16482605040073395, 0.022227419540286064, 0.011465233750641346, 0.019190151244401932, -1.9539560079574585, 0.014932344667613506, -0.9010579586029053, -0.6989918947219849, -0.8808764815330505, -1.3907837867736816, -1.2593215703964233, -1.301702618598938, -0.6721951961517334, -0.2562706768512726, -0.6034896373748779, -0.4228993356227875, -1.0893871784210205, -0.7712242007255554, -0.1804429143667221, -0.13963909447193146, -0.6072064638137817, 0.6901783347129822], 'state_max': [0.31211021542549133, 0.8900995850563049, 1.2004348039627075, 0.05436473712325096, -0.02029556967318058, -0.02401699870824814, -0.03348304331302643, 0.07547632604837418, -0.22075261175632477, -0.2170424908399582, 1.2214250564575195, 1.6621747016906738, 0.9483049511909485, 1.2975637912750244, 0.5761176347732544, 1.100870966911316, 0.7628798484802246, 1.3479257822036743, 1.4265302419662476, 0.6714234352111816, 0.6750589609146118, 0.7493496537208557, 0.11452125757932663, 0.42554786801338196, 1.298634648323059, 0.48713478446006775, 0.6239774227142334, 1.0436362028121948, 0.08730942010879517, 0.28669121861457825, 0.3420403003692627, 0.839439868927002], 'normalize_state': True, 'pad_action_dim': None, 'pad_state_dim': None}, 'model': {'resize': {'size': [256, 480]}, 'center_crop': {'size': [224, 224]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}}, 'action_chunk_size': 100, 'root_dir': '/hfm/data/real_teleop_g1/lerobot', 'dataset_paths': ['Remove_the_cap_turn_on_the_faucet_and_fill_the_bottle_with_water'], 'data_config': 'we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig', 'embodiment_tag': 'new_embodiment', 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'preload_all': False}, 'model': {'n_obs_steps': 1, 'chunk_size': 100, 'n_action_steps': 100, 'action_dim': 36, 'state_dim': 32, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'pre_norm': False, 'dropout': 0.1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'kl_weight': 10.0, 'temporal_ensemble_coeff': None}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/hfm/cache', 'TORCH_HOME': '/hfm/cache', 'HF_TOKEN': 'hf_...adRb', 'HF_LEROBOT_HOME': 'Not Set', 'WE_HOME': 'Not Set', 'DATA_HOME': 'Not Set', 'UV_CACHE_DIR': 'Not Set', 'WANDB_API_KEY': 'Not Set', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '6,7', 'WORLD_SIZE': '2', 'LOCAL_WORLD_SIZE': '2', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '44401'}} +2026-01-24 10:10:48,299 INFO MainThread:2641700 [wandb_run.py:_finish():2287] finishing run jliu530-soochow-university/hfm/1i9wg620 +2026-01-24 10:10:48,300 INFO MainThread:2641700 [wandb_run.py:_atexit_cleanup():2486] got exitcode: 0 +2026-01-24 10:10:48,301 INFO MainThread:2641700 [wandb_run.py:_restore():2468] restore +2026-01-24 10:10:48,301 INFO MainThread:2641700 [wandb_run.py:_restore():2474] restore done +2026-01-24 10:10:49,049 INFO MainThread:2641700 [wandb_run.py:_footer_sync_info():3862] logging synced files diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/files/config.yaml b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1245d9ad691ac1cc91f97d05091feedb0738d479 --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/files/config.yaml @@ -0,0 +1,457 @@ +_wandb: + value: + cli_version: 0.23.1 + e: + hslktvgyjla3ec98tcf0kvcf80yf0nx5: + args: + - data:gr00t + - data.transform.repack:gr00t-we + - data.transform.action-state:maxmin + - data.transform.model:act + - model:act + - --seed=2026 + - --exp=act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water + - --train.name=act-g1-real + - --log.report-to=wandb + - --wandb.entity=jliu530-soochow-university + - --train.data_parallel=ddp + - --train.mixed_precision=bf16 + - --train.train-batch-size=32 + - --train.gradient_accumulation_steps=1 + - --train.validation_steps=500 + - --train.val_num_batches=20 + - --train.max-training-steps=42000 + - --train.learning-rate=1e-4 + - --train.max-grad-norm=1.0 + - --train.lr_scheduler_kwargs.weight_decay=1e-6 + - --train.lr_scheduler_kwargs.betas + - "0.95" + - "0.999" + - --train.lr_scheduler_type=cosine + - --train.warmup-steps=1000 + - --train.warmup-ratio=None + - --train.checkpointing-steps=5000 + - --data.root_dir=/hfm/data/real_teleop_g1/lerobot + - --data.dataset_paths=Remove_the_cap_turn_on_the_faucet_and_fill_the_bottle_with_water + - --data.data_config=we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig + - --data.transform.repack.pad-action-dim=36 + - --data.transform.repack.pad-state-dim=32 + - --data.transform.action-state.stat-path=dataset_statistics/g1-stats-Remove_the_cap_turn_on_the_faucet_and_fill_the_bottle_with_water.json + - --data.transform.action-state.stat-action-key=action + - --data.transform.action-state.stat-state-key=states + - --data.transform.action-state.normalize-state + - --data.transform.action-state.action-norm-type=bounds + - --data.transform.model.img-aug + - --data.action-chunk-size=100 + - --model.chunk-size=100 + - --model.n-action-steps=100 + - --model.action-dim=36 + - --model.state-dim=32 + - --model.use-vae + - --model.kl-weight=10.0 + codePath: scripts/train.py + codePathLocal: scripts/train.py + cpu_count: 128 + cpu_count_logical: 128 + cudaVersion: "12.9" + disk: + /: + total: "105089261568" + used: "95562919936" + email: jliu530@163.com + executable: /home/jliu/we_learn/.venv-pusht/bin/python3 + git: + commit: df0aa6074e8b16d0e2be002a9a9c1971edf209ec + remote: https://github.com/songlin/we_learn.git + gpu: NVIDIA A100-SXM4-80GB + gpu_count: 8 + gpu_nvidia: + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-37370f00-90f5-4e03-f763-7a35649e6783 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-d4b40383-188b-cc16-9180-20c1a71a777f + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-0f3d827c-66cf-04da-b182-4ed0414a2549 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-164a49bb-43d3-2250-59e2-1b40eee0757f + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-c0471f40-ae19-5371-a1af-da406ca30f83 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-d9040f59-13d3-6c37-4ea0-c6c751424527 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-f834d0e8-a118-24c7-5461-30fce111bea6 + - architecture: Ampere + cudaCores: 6912 + memoryTotal: "85899345920" + name: NVIDIA A100-SXM4-80GB + uuid: GPU-6e687915-3247-735c-6641-544540804e79 + host: nebula99 + memory: + total: "1623177732096" + os: Linux-6.8.0-88-generic-x86_64-with-glibc2.39 + program: /home/jliu/we_learn/scripts/train.py + python: CPython 3.10.19 + root: /home/jliu/we_learn/.runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339 + startedAt: "2026-01-24T03:39:37.709638Z" + writerId: hslktvgyjla3ec98tcf0kvcf80yf0nx5 + m: [] + python_version: 3.10.19 + t: + "1": + - 1 + - 2 + - 3 + - 11 + - 41 + - 49 + - 71 + "2": + - 1 + - 2 + - 3 + - 11 + - 41 + - 49 + - 51 + - 71 + - 80 + - 83 + "3": + - 2 + - 13 + - 61 + "4": 3.10.19 + "5": 0.23.1 + "6": 4.44.2 + "12": 0.23.1 + "13": linux-x86_64 +auto_tag_run: + value: false +data: + value: + action_chunk_size: 100 + balance_dataset_weights: true + balance_trajectory_weights: true + data_config: we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig + dataset_paths: + - Remove_the_cap_turn_on_the_faucet_and_fill_the_bottle_with_water + embodiment_tag: new_embodiment + preload_all: false + root_dir: /hfm/data/real_teleop_g1/lerobot + transform: + action_state: + action_max: + - 0.4100879728794098 + - 0.9210000038146973 + - 1.254960298538208 + - 0.0010000000474974513 + - 0.0010000000474974513 + - 0.0010000000474974513 + - 0.0010000000474974513 + - 0.11807795614004135 + - -0.13724753260612488 + - -0.4963247776031494 + - 1.2912133932113647 + - 1.7463293075561523 + - 0.9923550486564636 + - 1.6118189096450806 + - 0.6930765509605408 + - 1.1467127799987793 + - 0.887980043888092 + - 1.3561404943466187 + - 1.6023852825164795 + - 0.7071925401687622 + - 0.8807530403137207 + - 0.9672761559486389 + - 0.357116162776947 + - 0.4602624475955963 + - 1.3487075567245483 + - 0.729375422000885 + - 0.6513803601264954 + - 1.052833914756775 + - 0.08730942010879517 + - 0.28669121861457825 + - 0.3420403003692627 + - 0.839439868927002 + - 0 + - 0 + - 0 + - 0 + action_min: + - -1.0481975078582764 + - -0.27768605947494507 + - 0.3844982385635376 + - -1.2728569507598877 + - -1.7463293075561523 + - -1.1422605514526367 + - -1.6586812734603882 + - -0.9686063528060913 + - -0.7862647771835327 + - -1.034379243850708 + - -0.0010000000474974513 + - -0.0010000000474974513 + - -0.0010000000474974513 + - -0.0010000000474974513 + - -2.0079522132873535 + - 0.04465832561254501 + - -1.0082837343215942 + - -0.8037082552909851 + - -1.1698691844940186 + - -1.4439691305160522 + - -1.2980564832687378 + - -1.3225891590118408 + - -0.8259854316711426 + - -0.2885793149471283 + - -0.8615938425064087 + - -0.444199800491333 + - -1.2984219789505005 + - -0.8555077910423279 + - -0.1804429143667221 + - -0.13963909447193146 + - -0.6072064638137817 + - 0.6901783347129822 + - 0 + - 0 + - 0 + - 0 + action_norm_masks: + - true + - true + - true + - true + - true + - true + - false + action_norm_type: bounds + normalize_state: true + pad_action_dim: null + pad_state_dim: null + stat_action_key: action + stat_path: dataset_statistics/g1-stats-Remove_the_cap_turn_on_the_faucet_and_fill_the_bottle_with_water.json + stat_state_key: states + state_max: + - 0.31211021542549133 + - 0.8900995850563049 + - 1.2004348039627075 + - 0.05436473712325096 + - -0.02029556967318058 + - -0.02401699870824814 + - -0.03348304331302643 + - 0.07547632604837418 + - -0.22075261175632477 + - -0.2170424908399582 + - 1.2214250564575195 + - 1.6621747016906738 + - 0.9483049511909485 + - 1.2975637912750244 + - 0.5761176347732544 + - 1.100870966911316 + - 0.7628798484802246 + - 1.3479257822036743 + - 1.4265302419662476 + - 0.6714234352111816 + - 0.6750589609146118 + - 0.7493496537208557 + - 0.11452125757932663 + - 0.42554786801338196 + - 1.298634648323059 + - 0.48713478446006775 + - 0.6239774227142334 + - 1.0436362028121948 + - 0.08730942010879517 + - 0.28669121861457825 + - 0.3420403003692627 + - 0.839439868927002 + state_min: + - -1.0061851739883423 + - -0.08936958014965057 + - 0.22508153319358826 + - -1.2385730743408203 + - -1.7122353315353394 + - -1.1028423309326172 + - -1.5751651525497437 + - -0.9382277131080627 + - -0.7008255124092102 + - -0.9635780453681946 + - -0.16482605040073395 + - 0.022227419540286064 + - 0.011465233750641346 + - 0.019190151244401932 + - -1.9539560079574585 + - 0.014932344667613506 + - -0.9010579586029053 + - -0.6989918947219849 + - -0.8808764815330505 + - -1.3907837867736816 + - -1.2593215703964233 + - -1.301702618598938 + - -0.6721951961517334 + - -0.2562706768512726 + - -0.6034896373748779 + - -0.4228993356227875 + - -1.0893871784210205 + - -0.7712242007255554 + - -0.1804429143667221 + - -0.13963909447193146 + - -0.6072064638137817 + - 0.6901783347129822 + use_norm_mask: false + model: + center_crop: + size: + - 224 + - 224 + color_jitter: + brightness: 0.2 + contrast: + - 0.8 + - 1.2 + hue: 0.05 + saturation: + - 0.8 + - 1.2 + img_aug: true + normalize: + mean: + - 0.485 + - 0.456 + - 0.406 + std: + - 0.229 + - 0.224 + - 0.225 + resize: + size: + - 256 + - 480 + repack: + action_key: action + conditions: [] + dataset_name: we-g1-teleop + image_keys: + - video + instruction_key: annotation.human.task_description + pad_action_dim: 36 + pad_state_dim: 32 + state_key: state + video_backend: decord +debug: + value: false +environment_variables: + value: + CUDA_VISIBLE_DEVICES: 6,7 + DATA_HOME: Not Set + HF_HOME: /hfm/cache + HF_LEROBOT_HOME: Not Set + HF_TOKEN: hf_...adRb + LOCAL_RANK: "0" + LOCAL_WORLD_SIZE: "2" + MASTER_ADDR: nebula99 + MASTER_PORT: "44401" + OMP_NUM_THREADS: "32" + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python + RANK: "0" + TORCH_HOME: /hfm/cache + UV_CACHE_DIR: Not Set + WANDB_API_KEY: Not Set + WE_HOME: Not Set + WORLD_SIZE: "2" +eval: + value: false +exp: + value: act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water +log: + value: + log_freq: 100 + logging_dir: logs + report_to: wandb +model: + value: + action_dim: 36 + chunk_size: 100 + dim_feedforward: 3200 + dim_model: 512 + dropout: 0.1 + feedforward_activation: relu + kl_weight: 10 + latent_dim: 32 + n_action_steps: 100 + n_decoder_layers: 1 + n_encoder_layers: 4 + n_heads: 8 + n_obs_steps: 1 + n_vae_encoder_layers: 4 + pre_norm: false + state_dim: 32 + temporal_ensemble_coeff: null + use_vae: true +seed: + value: 2026 +timestamp: + value: "2601240339" +train: + value: + checkpointing_steps: 5000 + data_parallel: ddp + deepspeed_config: /home/jliu/we_learn/src/InternVLA/config/deepseeds/zero3.json + enable_gradient_checkpointing: true + enable_mixed_precision_training: true + gradient_accumulation_steps: 1 + hf_token: .hf_token + learning_rate: 0.0001 + lora: false + lr_scheduler_kwargs: + betas: + - 0.95 + - 0.999 + eps: 1e-08 + weight_decay: 1e-06 + lr_scheduler_type: cosine + max_checkpoints_to_keep: null + max_grad_norm: 1 + max_training_steps: 42000 + mixed_precision: bf16 + name: act-g1-real + num_train_epochs: null + num_workers: 8 + output_dir: .runs + overfit_single_batch: false + reduce_in_full_precision: true + resume_from_checkpoint: null + sharding_strategy: full-shard + skip_resumed_steps: false + train_batch_size: 32 + val_batch_size: 16 + val_num_batches: 20 + validation_steps: 500 + warmup_ratio: null + warmup_steps: 1000 +wandb: + value: + entity: jliu530-soochow-university + group: null + id: null + name: null + project: hfm + resume: allow diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/files/output.log b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..341946d0f5e0c055c807b7f2796d6139c56e438c --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/files/output.log @@ -0,0 +1,255 @@ +[03:39:38 01/24] INFO  | >> [*] Saved configuration to ]8;id=805427;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=514750;file:///home/jliu/we_learn/scripts/train.py#129\129]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real   +  .cosin.lr1.0e-04.b64.gpus2.2601240339   +  INFO  | >> [*] Training configurations: ]8;id=2657;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=644039;file:///home/jliu/we_learn/scripts/train.py#182\182]8;;\ +  INFO  | >> |=> training task: 'act-g1-real' ]8;id=471497;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=12046;file:///home/jliu/we_learn/scripts/train.py#183\183]8;;\ +  INFO  | >> |=> run name: ]8;id=329497;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=220436;file:///home/jliu/we_learn/scripts/train.py#184\184]8;;\ +  act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b   +  64.gpus2.2601240339   +  INFO  | >> |=> seed: 2026 ]8;id=844246;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=394625;file:///home/jliu/we_learn/scripts/train.py#185\185]8;;\ +  INFO  | >> |=> mixed precision: torch.bfloat16 ]8;id=356929;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=94020;file:///home/jliu/we_learn/scripts/train.py#186\186]8;;\ +  INFO  | >> |=> warmup steps: 1000 ]8;id=147732;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=919050;file:///home/jliu/we_learn/scripts/train.py#187\187]8;;\ +  INFO  | >> |=> validation steps: 500 ]8;id=25941;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=888976;file:///home/jliu/we_learn/scripts/train.py#188\188]8;;\ +  INFO  | >> |=> checkpoint steps: 5000 ]8;id=95089;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=418144;file:///home/jliu/we_learn/scripts/train.py#189\189]8;;\ +  INFO  | >> |=> max gradient norm: 1.0 ]8;id=122104;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=446109;file:///home/jliu/we_learn/scripts/train.py#190\190]8;;\ +/home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/albumentations/__init__.py:13: UserWarning: A new version of Albumentations is available: 2.0.8 (you have 1.4.18). Upgrade using: pip install -U albumentations. To disable automatic update checks, set the environment variable NO_ALBUMENTATIONS_UPDATE to 1. + check_for_updates() +Loading external config: we.data.gr00t.data_config.WeUnitreeG1LocoDataConfig +Initialized dataset Remove_the_cap_turn_on_the_faucet_and_fill_the_bottle_with_water with EmbodimentTag.NEW_EMBODIMENT +Loading external config: we.data.gr00t.data_config.WeUnitreeG1LocoDataConfig +Initialized dataset Remove_the_cap_turn_on_the_faucet_and_fill_the_bottle_with_water with EmbodimentTag.NEW_EMBODIMENT +[03:39:39 01/24] INFO  | >> |=> Num training samples: ]8;id=917958;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=514883;file:///home/jliu/we_learn/scripts/train.py#193\193]8;;\ +  INFO  | >> |=> Training dataset size: 85,236 ]8;id=440366;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=591396;file:///home/jliu/we_learn/scripts/train.py#194\194]8;;\ +  INFO  | >> |=> Val dataset size: 85,236 ]8;id=26608;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=603472;file:///home/jliu/we_learn/scripts/train.py#196\196]8;;\ +  INFO  | >> [*] Initialize optimizers and schedulers... ]8;id=738892;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=548632;file:///home/jliu/we_learn/scripts/train.py#199\199]8;;\ +  INFO  | >> [*] ***** Running training ***** ]8;id=450268;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=492873;file:///home/jliu/we_learn/scripts/train.py#203\203]8;;\ +  INFO  | >> |=> Num training examples = 85236 ]8;id=928955;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=127940;file:///home/jliu/we_learn/scripts/train.py#204\204]8;;\ +  INFO  | >> |=> Max training Epochs = 32 ]8;id=749231;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=974614;file:///home/jliu/we_learn/scripts/train.py#205\205]8;;\ +  INFO  | >> |=> Total optimization steps = 42000 ]8;id=322570;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=574775;file:///home/jliu/we_learn/scripts/train.py#206\206]8;;\ +  INFO  | >> |=> Num steps Per Epoch = 1332 ]8;id=843992;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=927413;file:///home/jliu/we_learn/scripts/train.py#207\207]8;;\ +  INFO  | >> |=> Effective training epochs = 63.06 ]8;id=282104;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=9727;file:///home/jliu/we_learn/scripts/train.py#208\208]8;;\ +  INFO  | >> |=> Global train batch size (w. parallel, distributed & accumulation) = 64 ]8;id=480486;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=842580;file:///home/jliu/we_learn/scripts/train.py#209\209]8;;\ +  INFO  | >> |=> Device train batch size = 32 ]8;id=209544;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=272745;file:///home/jliu/we_learn/scripts/train.py#210\210]8;;\ +  INFO  | >> |=> Gradient Accumulation steps = 1 ]8;id=866305;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=521307;file:///home/jliu/we_learn/scripts/train.py#211\211]8;;\ +  INFO  | >> |=> Num processes (GPUs) = 2 ]8;id=35253;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=871789;file:///home/jliu/we_learn/scripts/train.py#212\212]8;;\ +[03:39:41 01/24] INFO  | >> TensorFlow version 2.20.0 available. ]8;id=459749;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/datasets/config.py\config.py]8;;\:]8;id=722818;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/datasets/config.py#112\112]8;;\ +  INFO  | >> [*] Accelerator runs in: ]8;id=966756;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=191940;file:///home/jliu/we_learn/scripts/train.py#219\219]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real   +  .cosin.lr1.0e-04.b64.gpus2.2601240339   +Traing steps: 36%|████████████████▊ | 15000/42000 [2:23:07<1:48:06, 4.16it/s, loss=0.0987, lr=7.4e-05] + +[04:28:30 01/24] INFO  | >> Saving current state to ]8;id=680052;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=279409;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-wi   +  th-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_5000   +[04:28:31 01/24] INFO  | >> Model weights saved in ]8;id=898249;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=431290;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-w   +  ith-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_5000/   +  model.safetensors   +[04:28:32 01/24] INFO  | >> Optimizer state saved in ]8;id=224;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=473149;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-w   +  ith-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_5000/   +  optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=639771;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=832328;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-w   +  ith-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_5000/   +  scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=788377;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=572992;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-w   +  ith-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_5000/   +  sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=374604;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=162278;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-w   +  ith-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_5000/   +  sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=806908;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=971194;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-w   +  ith-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_5000/   +  random_states_0.pkl   +Saved state to .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_5000 +[05:15:27 01/24] WARNING  | >> [*] Checkpoint 10000 already exists, skipping save. ]8;id=983054;file:///home/jliu/we_learn/src/we/learn/trainers/act_g1_real.py\act_g1_real.py]8;;\:]8;id=141415;file:///home/jliu/we_learn/src/we/learn/trainers/act_g1_real.py#177\177]8;;\ +Saved state to None +[06:02:49 01/24] INFO  | >> Saving current state to ]8;id=137590;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=627191;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-wi   +  th-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_15000   +[06:02:50 01/24] INFO  | >> Model weights saved in ]8;id=88534;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=438072;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-w   +  ith-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_15000   +  /model.safetensors   +[06:02:51 01/24] INFO  | >> Optimizer state saved in ]8;id=821494;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=820417;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-w   +  ith-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_15000   +  /optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=432054;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=409371;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-w   +  ith-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_15000   +  /scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=206251;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=394271;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-w   +  ith-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_15000   +  /sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=77000;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=53870;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-w   +  ith-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_15000   +  /sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=979940;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=806914;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-w   +  ith-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_15000   +  /random_states_0.pkl   +Saved state to .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_15000 +[06:49:16 01/24] INFO  | >> Saving current state to ]8;id=499147;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=343681;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.c   +  osin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_20000   +[06:49:17 01/24] INFO  | >> Model weights saved in ]8;id=958353;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=127268;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_20000/model.safetensors   +[06:49:18 01/24] INFO  | >> Optimizer state saved in ]8;id=648655;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=844039;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_20000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=217040;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=454670;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_20000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=244660;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=711457;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_20000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=35811;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=204990;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_20000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=679719;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=571229;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_20000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_20000 +[07:35:57 01/24] INFO  | >> Saving current state to ]8;id=910263;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=791370;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.c   +  osin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_25000   +[07:35:58 01/24] INFO  | >> Model weights saved in ]8;id=711599;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=920457;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_25000/model.safetensors   +  INFO  | >> Optimizer state saved in ]8;id=229348;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=823560;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_25000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=168181;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=693342;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_25000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=446016;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=558902;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_25000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=319720;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=196292;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_25000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=521773;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=818149;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_25000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_25000 +[08:22:52 01/24] INFO  | >> Saving current state to ]8;id=185558;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=341614;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.c   +  osin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_30000   +[08:22:53 01/24] INFO  | >> Model weights saved in ]8;id=552863;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=835298;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_30000/model.safetensors   +[08:22:54 01/24] INFO  | >> Optimizer state saved in ]8;id=99159;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=889056;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_30000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=97642;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=921300;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_30000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=571536;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=821364;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_30000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=191314;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=28505;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_30000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=589302;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=720536;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_30000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_30000 +[09:10:09 01/24] INFO  | >> Saving current state to ]8;id=115021;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=650939;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.c   +  osin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_35000   +[09:10:10 01/24] INFO  | >> Model weights saved in ]8;id=595634;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=766478;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_35000/model.safetensors   +[09:10:11 01/24] INFO  | >> Optimizer state saved in ]8;id=281123;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=702868;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_35000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=276522;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=101114;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_35000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=63361;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=795987;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_35000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=677321;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=660693;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_35000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=616676;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=878068;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_35000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_35000 +[09:55:35 01/24] INFO  | >> Saving current state to ]8;id=169230;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=273529;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.c   +  osin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000   +[09:55:36 01/24] INFO  | >> Model weights saved in ]8;id=908584;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=941677;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/model.safetensors   +  INFO  | >> Optimizer state saved in ]8;id=16179;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=60095;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=128996;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=98429;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=899091;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=367323;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=656123;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=686520;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=423824;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=331416;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_40000 +[10:10:32 01/24] INFO  | >> Saving current state to ]8;id=446823;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=708752;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.c   +  osin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999   +[10:10:33 01/24] INFO  | >> Model weights saved in ]8;id=903139;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=525611;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/model.safetensors   +  INFO  | >> Optimizer state saved in ]8;id=306059;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=993068;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=749912;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=457956;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=346604;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=303621;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=128256;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=311366;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=946822;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=21254;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/random_states_0.pkl   +Saved state to .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999 +Training has reached maximum steps. +[10:10:45 01/24] INFO  | >> Saving current state to ]8;id=58872;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py\accelerator.py]8;;\:]8;id=756734;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/accelerator.py#3644\3644]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.c   +  osin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999   +[10:10:47 01/24] INFO  | >> Model weights saved in ]8;id=124028;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=39131;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#114\114]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/model.safetensors   +[10:10:48 01/24] INFO  | >> Optimizer state saved in ]8;id=679365;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=687655;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#121\121]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/optimizer.bin   +  INFO  | >> Scheduler state saved in ]8;id=132432;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=373263;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#128\128]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/scheduler.bin   +  INFO  | >> Sampler state for dataloader 0 saved in ]8;id=814323;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=441435;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/sampler.bin   +  INFO  | >> Sampler state for dataloader 1 saved in ]8;id=192379;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=569300;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#145\145]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/sampler_1.bin   +  INFO  | >> Random states saved in ]8;id=567626;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py\checkpointing.py]8;;\:]8;id=716966;file:///home/jliu/we_learn/.venv-pusht/lib/python3.10/site-packages/accelerate/checkpointing.py#176\176]8;;\ +  .runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.   +  cosin.lr1.0e-04.b64.gpus2.2601240339/checkpoints/ckpt_41999/random_states_0.pkl   +  INFO  | >> [*] Finalized ACT Trainer. Epoch losses: [0.0, 0.3774549961090088, 0.16167297959327698, ]8;id=154857;file:///home/jliu/we_learn/src/we/learn/trainers/act_g1_real.py\act_g1_real.py]8;;\:]8;id=239343;file:///home/jliu/we_learn/src/we/learn/trainers/act_g1_real.py#331\331]8;;\ +  0.10211339592933655, 0.08448188006877899]   +  INFO  | >> [*] Happy Ending! ]8;id=438178;file:///home/jliu/we_learn/scripts/train.py\train.py]8;;\:]8;id=276631;file:///home/jliu/we_learn/scripts/train.py#314\314]8;;\ diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/files/requirements.txt b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..20111d90fafa72be7424aebca4417875b5f6e178 --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/files/requirements.txt @@ -0,0 +1,237 @@ +markdown-it-py==4.0.0 +tensorflow-datasets==4.9.9 +pynput==1.8.1 +nvidia-cufile-cu12==1.13.1.3 +nvidia-nccl-cu12==2.27.3 +pandas==2.3.3 +sniffio==1.3.1 +keras==3.12.0 +expandvars==1.1.2 +isort==6.0.1 +accelerate==1.12.0 +diffusers==0.30.2 +lazy_loader==0.4 +safetensors==0.6.2 +iniconfig==2.1.0 +Jinja2==3.1.6 +pymunk==7.2.0 +datasets==4.4.1 +libclang==18.1.1 +fasteners==0.20 +httpcore==1.0.9 +immutabledict==4.2.2 +protobuf==6.33.2 +einx==0.3.0 +requests==2.32.5 +cffi==2.0.0 +optree==0.18.0 +astunparse==1.6.3 +transforms3d==0.4.2 +xxhash==3.5.0 +wcwidth==0.2.14 +click==8.3.0 +sympy==1.14.0 +annotated-types==0.7.0 +nvidia-nvjitlink-cu12==12.8.93 +numpydantic==1.6.7 +Pygments==2.19.2 +mergedeep==1.3.4 +black==25.9.0 +albumentations==1.4.18 +gymnasium==0.29.1 +multiprocess==0.70.16 +nvidia-cusparse-cu12==12.5.8.93 +certifi==2025.8.3 +array_record==0.8.1 +pyyaml-include==1.4.1 +tzdata==2025.2 +dm-tree==0.1.8 +cycler==0.12.1 +fastapi==0.128.0 +wrapt==2.0.1 +pydantic_core==2.27.2 +tqdm==4.67.1 +starlette==0.50.0 +OpenEXR==3.4.4 +av==16.0.1 +argparse==1.4.0 +pyparsing==3.3.2 +yarl==1.20.1 +dotenv==0.9.9 +omegaconf==2.3.0 +opt_einsum==3.4.0 +trimesh==4.11.1 +imageio==2.34.2 +soupsieve==2.8 +gitdb==4.0.12 +beautifulsoup4==4.14.2 +argcomplete==3.6.2 +tabulate==0.9.0 +mypy_extensions==1.1.0 +propcache==0.3.2 +nvidia-cusparselt-cu12==0.7.1 +annotated-doc==0.0.4 +PySocks==1.7.1 +simple-parsing==0.1.8 +yacs==0.1.8 +pycparser==2.23 +nvidia-cudnn-cu12==9.10.2.21 +tensorboard==2.20.0 +transformers==4.44.2 +opencv-python-headless==4.11.0.86 +websockets==15.0.1 +ruamel.yaml==0.18.15 +torchvision==0.23.0 +matplotlib==3.10.8 +rerun-sdk==0.22.1 +evdev==1.9.2 +blinker==1.9.0 +h11==0.16.0 +pyzmq==27.1.0 +nvidia-curand-cu12==10.3.9.90 +idna==3.10 +plotly==6.2.0 +aiohappyeyeballs==2.6.1 +antlr4-python3-runtime==4.9.3 +platformdirs==4.4.0 +filelock==3.19.1 +pytokens==0.1.10 +hf-xet==1.1.10 +aiohttp-cors==0.8.1 +decord==0.6.0 +docker-pycreds==0.4.0 +tifffile==2025.5.10 +pytz==2025.2 +portalocker==3.2.0 +nvidia-cuda-cupti-cu12==12.8.90 +scikit-image==0.25.2 +ruamel.yaml.clib==0.2.14 +python-xlib==0.33 +narwhals==2.6.0 +shellingham==1.5.4 +python-dotenv==1.2.1 +triton==3.4.0 +draccus==0.10.0 +ml_dtypes==0.5.4 +wheel==0.46.3 +kiwisolver==1.4.9 +tomli==2.2.1 +termcolor==3.1.0 +we.learn==0.1.0 +typing-inspect==0.9.0 +exceptiongroup==1.3.0 +Farama-Notifications==0.0.4 +setproctitle==1.3.7 +hf_transfer==0.1.9 +albucore==0.0.17 +nvidia-cusolver-cu12==11.7.3.90 +shtab==1.7.2 +imageio-ffmpeg==0.6.0 +typeguard==4.4.4 +contourpy==1.3.2 +nvidia-nvtx-cu12==12.8.90 +zipp==3.23.0 +inquirerpy==0.3.4 +torchcodec==0.6.0 +tensorboard-data-server==0.7.2 +importlib_metadata==8.7.0 +fvcore==0.1.5.post20221221 +llvmlite==0.46.0 +lerobot==0.1.0 +setuptools==80.9.0 +pluggy==1.6.0 +asciitree==0.3.3 +MarkupSafe==3.0.3 +wandb==0.23.1 +Flask==3.1.2 +pydantic==2.10.6 +httpx==0.28.1 +pfzy==0.3.4 +frozendict==2.4.6 +numpy==1.26.4 +gast==0.7.0 +pathspec==0.12.1 +ruff==0.13.2 +packaging==25.0 +promise==2.3 +tensorflow-addons==0.19.0 +importlib_resources==6.5.2 +torchaudio==2.8.0 +iopath==0.1.10 +google-pasta==0.2.0 +mypy==1.18.2 +orderly-set==5.5.0 +pyarrow==21.0.0 +vuer==0.0.68 +toml==0.10.2 +PyYAML==6.0.3 +torch==2.8.0 +numcodecs==0.13.1 +pydantic-yaml==1.6.0 +numba==0.63.1 +eval_type_backport==0.2.2 +psutil==7.1.0 +fsspec==2025.3.0 +gdown==5.2.0 +GitPython==3.1.45 +huggingface-hub==0.35.3 +fonttools==4.61.1 +frozenlist==1.7.0 +Markdown==3.10.1 +typing_extensions==4.15.0 +tensorflow==2.20.0 +h5py==3.14.0 +tyro==0.9.32 +cloudpickle==3.1.1 +cmake==4.1.0 +mdurl==0.1.2 +typer==0.19.2 +anyio==4.11.0 +namex==0.1.0 +pipablepytorch3d==0.7.6 +nvidia-cuda-nvrtc-cu12==12.8.93 +docstring_parser==0.17.0 +killport==1.2.0 +nvidia-cuda-runtime-cu12==12.8.90 +aiosignal==1.4.0 +dill==0.3.8 +nvidia-cublas-cu12==12.8.4.1 +zarr==2.18.3 +msgpack==1.1.1 +flatbuffers==25.12.19 +regex==2025.9.18 +python-dateutil==2.9.0.post0 +tokenizers==0.19.1 +pytest==8.4.2 +charset-normalizer==3.4.3 +sentry-sdk==2.39.0 +rich==14.1.0 +itsdangerous==2.2.0 +nvidia-cufft-cu12==11.3.3.83 +scipy==1.15.3 +simplejpeg==1.9.0 +uvicorn==0.40.0 +grpcio==1.76.0 +six==1.17.0 +absl-py==2.3.1 +jsonlines==4.0.0 +urllib3==2.5.0 +opencv-python==4.11.0.86 +etils==1.13.0 +mpmath==1.3.0 +einops==0.8.1 +smmap==5.0.2 +tensorflow-graphics==2021.12.3 +tensorflow-metadata==1.17.3 +multidict==6.6.4 +Werkzeug==3.1.3 +pillow==11.3.0 +prompt_toolkit==3.0.52 +waterbear==2.6.8 +async-timeout==4.0.3 +networkx==3.4.2 +attrs==25.3.0 +params_proto==2.13.2 +deepdiff==8.6.1 +aiohttp==3.10.5 +we.learn==0.1.0 diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/files/wandb-metadata.json b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3236c97cc31b8de664308c4e2bfd8716f39c6806 --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/files/wandb-metadata.json @@ -0,0 +1,136 @@ +{ + "os": "Linux-6.8.0-88-generic-x86_64-with-glibc2.39", + "python": "CPython 3.10.19", + "startedAt": "2026-01-24T03:39:37.709638Z", + "args": [ + "data:gr00t", + "data.transform.repack:gr00t-we", + "data.transform.action-state:maxmin", + "data.transform.model:act", + "model:act", + "--seed=2026", + "--exp=act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water", + "--train.name=act-g1-real", + "--log.report-to=wandb", + "--wandb.entity=jliu530-soochow-university", + "--train.data_parallel=ddp", + "--train.mixed_precision=bf16", + "--train.train-batch-size=32", + "--train.gradient_accumulation_steps=1", + "--train.validation_steps=500", + "--train.val_num_batches=20", + "--train.max-training-steps=42000", + "--train.learning-rate=1e-4", + "--train.max-grad-norm=1.0", + "--train.lr_scheduler_kwargs.weight_decay=1e-6", + "--train.lr_scheduler_kwargs.betas", + "0.95", + "0.999", + "--train.lr_scheduler_type=cosine", + "--train.warmup-steps=1000", + "--train.warmup-ratio=None", + "--train.checkpointing-steps=5000", + "--data.root_dir=/hfm/data/real_teleop_g1/lerobot", + "--data.dataset_paths=Remove_the_cap_turn_on_the_faucet_and_fill_the_bottle_with_water", + "--data.data_config=we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig", + "--data.transform.repack.pad-action-dim=36", + "--data.transform.repack.pad-state-dim=32", + "--data.transform.action-state.stat-path=dataset_statistics/g1-stats-Remove_the_cap_turn_on_the_faucet_and_fill_the_bottle_with_water.json", + "--data.transform.action-state.stat-action-key=action", + "--data.transform.action-state.stat-state-key=states", + "--data.transform.action-state.normalize-state", + "--data.transform.action-state.action-norm-type=bounds", + "--data.transform.model.img-aug", + "--data.action-chunk-size=100", + "--model.chunk-size=100", + "--model.n-action-steps=100", + "--model.action-dim=36", + "--model.state-dim=32", + "--model.use-vae", + "--model.kl-weight=10.0" + ], + "program": "/home/jliu/we_learn/scripts/train.py", + "codePath": "scripts/train.py", + "codePathLocal": "scripts/train.py", + "git": { + "remote": "https://github.com/songlin/we_learn.git", + "commit": "df0aa6074e8b16d0e2be002a9a9c1971edf209ec" + }, + "email": "jliu530@163.com", + "root": "/home/jliu/we_learn/.runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339", + "host": "nebula99", + "executable": "/home/jliu/we_learn/.venv-pusht/bin/python3", + "cpu_count": 128, + "cpu_count_logical": 128, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 8, + "disk": { + "/": { + "total": "105089261568", + "used": "95562919936" + } + }, + "memory": { + "total": "1623177732096" + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-37370f00-90f5-4e03-f763-7a35649e6783" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-d4b40383-188b-cc16-9180-20c1a71a777f" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-0f3d827c-66cf-04da-b182-4ed0414a2549" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-164a49bb-43d3-2250-59e2-1b40eee0757f" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-c0471f40-ae19-5371-a1af-da406ca30f83" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-d9040f59-13d3-6c37-4ea0-c6c751424527" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-f834d0e8-a118-24c7-5461-30fce111bea6" + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere", + "uuid": "GPU-6e687915-3247-735c-6641-544540804e79" + } + ], + "cudaVersion": "12.9", + "writerId": "hslktvgyjla3ec98tcf0kvcf80yf0nx5" +} \ No newline at end of file diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/files/wandb-summary.json b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..325c777a5c67493f559fc4d9deb582a44187ac7d --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb":{"runtime":23469},"val/denorm_err_l1_arm_joints":0.03352993354201317,"train/epoch":31,"val/denorm_err_l1_torso_vy":0,"_runtime":23469.602011138,"val/denorm_err_l1_torso_vyaw":0,"_step":42000,"train/kld_loss":0.0018478408455848694,"val/bc_loss":0.025573616847395897,"train/l1_loss":0.024116333574056625,"val/denorm_err_l1_height":0.0009528946247883141,"val/denorm_err_l1_hand_joints":0.07726205140352249,"train/loss":0.04259474202990532,"val/kld_loss":0.0005318400450050831,"val/denorm_err_l1_torso_target_yaw":0,"_timestamp":1.769249445309259e+09,"val/denorm_err_l1_rpy":0.003510949667543173,"val/denorm_err_l1_torso_vx":0} \ No newline at end of file diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/logs/debug-internal.log b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7f61b5cb10168457e9983324d9b2cafdd15c5301 --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/logs/debug-internal.log @@ -0,0 +1,12 @@ +{"time":"2026-01-24T03:39:37.969947817Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"} +{"time":"2026-01-24T03:39:38.23205191Z","level":"INFO","msg":"stream: created new stream","id":"1i9wg620"} +{"time":"2026-01-24T03:39:38.232211289Z","level":"INFO","msg":"handler: started","stream_id":"1i9wg620"} +{"time":"2026-01-24T03:39:38.232363264Z","level":"INFO","msg":"stream: started","id":"1i9wg620"} +{"time":"2026-01-24T03:39:38.232403912Z","level":"INFO","msg":"writer: started","stream_id":"1i9wg620"} +{"time":"2026-01-24T03:39:38.232444141Z","level":"INFO","msg":"sender: started","stream_id":"1i9wg620"} +{"time":"2026-01-24T10:10:48.870596221Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2026-01-24T10:10:49.038115851Z","level":"INFO","msg":"handler: operation stats","stats":{}} +{"time":"2026-01-24T10:10:49.054407773Z","level":"INFO","msg":"stream: closing","id":"1i9wg620"} +{"time":"2026-01-24T10:10:49.054472656Z","level":"INFO","msg":"handler: closed","stream_id":"1i9wg620"} +{"time":"2026-01-24T10:10:49.054719547Z","level":"INFO","msg":"sender: closed","stream_id":"1i9wg620"} +{"time":"2026-01-24T10:10:49.054847288Z","level":"INFO","msg":"stream: closed","id":"1i9wg620"} diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/logs/debug.log b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2d5f52376ff701938848731734b305a8afd4c150 --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/logs/debug.log @@ -0,0 +1,27 @@ +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1 +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_setup.py:_flush():80] Configure stats pid to 2641700 +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_setup.py:_flush():80] Loading settings from /home/jliu/.config/wandb/settings +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_setup.py:_flush():80] Loading settings from /home/jliu/we_learn/wandb/settings +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_setup.py:_flush():80] Loading settings from environment variables +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_init.py:setup_run_log_directory():714] Logging user logs to /home/jliu/we_learn/.runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/logs/debug.log +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to /home/jliu/we_learn/.runs/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/logs/debug-internal.log +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_init.py:init():841] calling init triggers +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_init.py:init():846] wandb.init called with sweep_config: {} +config: {'_wandb': {}} +2026-01-24 03:39:37,711 INFO MainThread:2641700 [wandb_init.py:init():889] starting backend +2026-01-24 03:39:37,961 INFO MainThread:2641700 [wandb_init.py:init():892] sending inform_init request +2026-01-24 03:39:37,967 INFO MainThread:2641700 [wandb_init.py:init():900] backend started and connected +2026-01-24 03:39:37,968 INFO MainThread:2641700 [wandb_init.py:init():970] updated telemetry +2026-01-24 03:39:37,975 INFO MainThread:2641700 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout +2026-01-24 03:39:38,699 INFO MainThread:2641700 [wandb_init.py:init():1041] starting run threads in backend +2026-01-24 03:39:38,909 INFO MainThread:2641700 [wandb_run.py:_console_start():2521] atexit reg +2026-01-24 03:39:38,909 INFO MainThread:2641700 [wandb_run.py:_redirect():2369] redirect: wrap_raw +2026-01-24 03:39:38,909 INFO MainThread:2641700 [wandb_run.py:_redirect():2438] Wrapping output streams. +2026-01-24 03:39:38,909 INFO MainThread:2641700 [wandb_run.py:_redirect():2461] Redirects installed. +2026-01-24 03:39:38,914 INFO MainThread:2641700 [wandb_init.py:init():1081] run started, returning control to user process +2026-01-24 03:39:38,915 INFO MainThread:2641700 [wandb_run.py:_config_callback():1396] config_cb None None {'exp': 'act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water', 'seed': 2026, 'auto_tag_run': False, 'eval': False, 'debug': False, 'timestamp': '2601240339', 'log': {'logging_dir': 'logs', 'report_to': 'wandb', 'log_freq': 100}, 'wandb': {'project': 'hfm', 'entity': 'jliu530-soochow-university', 'group': None, 'id': None, 'name': None, 'resume': 'allow'}, 'train': {'num_workers': 8, 'overfit_single_batch': False, 'name': 'act-g1-real', 'resume_from_checkpoint': None, 'skip_resumed_steps': False, 'hf_token': '.hf_token', 'lora': False, 'output_dir': '.runs', 'gradient_accumulation_steps': 1, 'mixed_precision': 'bf16', 'max_grad_norm': 1.0, 'train_batch_size': 32, 'val_batch_size': 16, 'val_num_batches': 20, 'checkpointing_steps': 5000, 'max_checkpoints_to_keep': None, 'validation_steps': 500, 'learning_rate': 0.0001, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {'betas': [0.95, 0.999], 'weight_decay': 1e-06, 'eps': 1e-08}, 'scheduler_specific_kwargs': {}, 'data_parallel': 'ddp', 'sharding_strategy': 'full-shard', 'deepspeed_config': '/home/jliu/we_learn/src/InternVLA/config/deepseeds/zero3.json', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': True, 'max_training_steps': 42000, 'num_train_epochs': None, 'warmup_steps': 1000, 'warmup_ratio': None}, 'data': {'transform': {'repack': {'dataset_name': 'we-g1-teleop', 'conditions': [], 'image_keys': ['video'], 'state_key': 'state', 'action_key': 'action', 'instruction_key': 'annotation.human.task_description', 'pad_action_dim': 36, 'pad_state_dim': 32}, 'action_state': {'stat_path': 'dataset_statistics/g1-stats-Remove_the_cap_turn_on_the_faucet_and_fill_the_bottle_with_water.json', 'action_norm_type': 'bounds', 'stat_action_key': 'action', 'stat_state_key': 'states', 'use_norm_mask': False, 'action_norm_masks': [True, True, True, True, True, True, False], 'action_min': [-1.0481975078582764, -0.27768605947494507, 0.3844982385635376, -1.2728569507598877, -1.7463293075561523, -1.1422605514526367, -1.6586812734603882, -0.9686063528060913, -0.7862647771835327, -1.034379243850708, -0.0010000000474974513, -0.0010000000474974513, -0.0010000000474974513, -0.0010000000474974513, -2.0079522132873535, 0.04465832561254501, -1.0082837343215942, -0.8037082552909851, -1.1698691844940186, -1.4439691305160522, -1.2980564832687378, -1.3225891590118408, -0.8259854316711426, -0.2885793149471283, -0.8615938425064087, -0.444199800491333, -1.2984219789505005, -0.8555077910423279, -0.1804429143667221, -0.13963909447193146, -0.6072064638137817, 0.6901783347129822, 0.0, 0.0, 0.0, 0.0], 'action_max': [0.4100879728794098, 0.9210000038146973, 1.254960298538208, 0.0010000000474974513, 0.0010000000474974513, 0.0010000000474974513, 0.0010000000474974513, 0.11807795614004135, -0.13724753260612488, -0.4963247776031494, 1.2912133932113647, 1.7463293075561523, 0.9923550486564636, 1.6118189096450806, 0.6930765509605408, 1.1467127799987793, 0.887980043888092, 1.3561404943466187, 1.6023852825164795, 0.7071925401687622, 0.8807530403137207, 0.9672761559486389, 0.357116162776947, 0.4602624475955963, 1.3487075567245483, 0.729375422000885, 0.6513803601264954, 1.052833914756775, 0.08730942010879517, 0.28669121861457825, 0.3420403003692627, 0.839439868927002, 0.0, 0.0, 0.0, 0.0], 'state_min': [-1.0061851739883423, -0.08936958014965057, 0.22508153319358826, -1.2385730743408203, -1.7122353315353394, -1.1028423309326172, -1.5751651525497437, -0.9382277131080627, -0.7008255124092102, -0.9635780453681946, -0.16482605040073395, 0.022227419540286064, 0.011465233750641346, 0.019190151244401932, -1.9539560079574585, 0.014932344667613506, -0.9010579586029053, -0.6989918947219849, -0.8808764815330505, -1.3907837867736816, -1.2593215703964233, -1.301702618598938, -0.6721951961517334, -0.2562706768512726, -0.6034896373748779, -0.4228993356227875, -1.0893871784210205, -0.7712242007255554, -0.1804429143667221, -0.13963909447193146, -0.6072064638137817, 0.6901783347129822], 'state_max': [0.31211021542549133, 0.8900995850563049, 1.2004348039627075, 0.05436473712325096, -0.02029556967318058, -0.02401699870824814, -0.03348304331302643, 0.07547632604837418, -0.22075261175632477, -0.2170424908399582, 1.2214250564575195, 1.6621747016906738, 0.9483049511909485, 1.2975637912750244, 0.5761176347732544, 1.100870966911316, 0.7628798484802246, 1.3479257822036743, 1.4265302419662476, 0.6714234352111816, 0.6750589609146118, 0.7493496537208557, 0.11452125757932663, 0.42554786801338196, 1.298634648323059, 0.48713478446006775, 0.6239774227142334, 1.0436362028121948, 0.08730942010879517, 0.28669121861457825, 0.3420403003692627, 0.839439868927002], 'normalize_state': True, 'pad_action_dim': None, 'pad_state_dim': None}, 'model': {'resize': {'size': [256, 480]}, 'center_crop': {'size': [224, 224]}, 'color_jitter': {'brightness': 0.2, 'contrast': [0.8, 1.2], 'saturation': [0.8, 1.2], 'hue': 0.05}, 'normalize': {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}, 'img_aug': True}}, 'action_chunk_size': 100, 'root_dir': '/hfm/data/real_teleop_g1/lerobot', 'dataset_paths': ['Remove_the_cap_turn_on_the_faucet_and_fill_the_bottle_with_water'], 'data_config': 'we.data.gr00t.data_config:WeUnitreeG1LocoDataConfig', 'embodiment_tag': 'new_embodiment', 'video_backend': 'decord', 'balance_dataset_weights': True, 'balance_trajectory_weights': True, 'preload_all': False}, 'model': {'n_obs_steps': 1, 'chunk_size': 100, 'n_action_steps': 100, 'action_dim': 36, 'state_dim': 32, 'dim_model': 512, 'n_heads': 8, 'dim_feedforward': 3200, 'feedforward_activation': 'relu', 'n_encoder_layers': 4, 'n_decoder_layers': 1, 'pre_norm': False, 'dropout': 0.1, 'use_vae': True, 'latent_dim': 32, 'n_vae_encoder_layers': 4, 'kl_weight': 10.0, 'temporal_ensemble_coeff': None}, 'environment_variables': {'OMP_NUM_THREADS': '32', 'HF_HOME': '/hfm/cache', 'TORCH_HOME': '/hfm/cache', 'HF_TOKEN': 'hf_...adRb', 'HF_LEROBOT_HOME': 'Not Set', 'WE_HOME': 'Not Set', 'DATA_HOME': 'Not Set', 'UV_CACHE_DIR': 'Not Set', 'WANDB_API_KEY': 'Not Set', 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION': 'python', 'CUDA_VISIBLE_DEVICES': '6,7', 'WORLD_SIZE': '2', 'LOCAL_WORLD_SIZE': '2', 'RANK': '0', 'LOCAL_RANK': '0', 'MASTER_ADDR': 'nebula99', 'MASTER_PORT': '44401'}} +2026-01-24 10:10:48,299 INFO MainThread:2641700 [wandb_run.py:_finish():2287] finishing run jliu530-soochow-university/hfm/1i9wg620 +2026-01-24 10:10:48,300 INFO MainThread:2641700 [wandb_run.py:_atexit_cleanup():2486] got exitcode: 0 +2026-01-24 10:10:48,301 INFO MainThread:2641700 [wandb_run.py:_restore():2468] restore +2026-01-24 10:10:48,301 INFO MainThread:2641700 [wandb_run.py:_restore():2474] restore done +2026-01-24 10:10:49,049 INFO MainThread:2641700 [wandb_run.py:_footer_sync_info():3862] logging synced files diff --git a/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/run-1i9wg620.wandb b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/run-1i9wg620.wandb new file mode 100644 index 0000000000000000000000000000000000000000..149f2133670bfbb1e512d8fc992730ca68c9f1b1 --- /dev/null +++ b/act/act-g1-real/act-remove-the-cap-turn-on-the-faucet-and-fill-the-bottle-with-water.g1-real.cosin.lr1.0e-04.b64.gpus2.2601240339/wandb/run-20260124_033937-1i9wg620/run-1i9wg620.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:453c3184c757c4d1ea140d87573e4e64f71e29114063fcc3645c422e13971959 +size 48545143