| { |
| "auto_tag_run": false, |
| "data": { |
| "decision_labels_path": "/training/EmbodiedDyna-runs/sg2t37-v3varlen-20261002/sam3v2/decision_labels_sg2t37.json", |
| "decision_labels_sha256": "82beae2fc8f1609da06e9605652fc3a22b2adc9804a751770cbcf13e6ec2ba79", |
| "framesamp_dataset_revision": "799b00f3a1e08a0328d54bff13f1b8d8a7489205", |
| "framesamp_feature_manifest_sha256": "1d809e0cbf5d1018551ca01f46ff197ff32f654a9f79179b95c4623190764c23", |
| "framesamp_feature_root": "/training/EmbodiedDyna-psi0-framesamp-data/features-sg2t37v3varlen/qwen-vision-merger-raw1/4efc80fed43af5e3503f7fe3d803fc32a880c6db/SimFactoryShellGame2T-v0", |
| "framesamp_feature_scope": "full", |
| "native_keyframe_manifest_sha256": "67f7023b19d47bea3461aa97ec0fd983cc33acce374798ac8f17ce1f1c42a545", |
| "native_keyframe_root": "/training/keyframe-native7x77-sg2t37v3-20261002", |
| "native_keyframe_selector": "sam3", |
| "object_memory_manifest_sha256": "875398dcb3e9e2e21d7c60689c5584fc0b3825a03fd26c39d434be51e12a2eee", |
| "object_memory_root": "/training/EmbodiedDyna-runs/sg2t37-v3varlen-20261002/sam3v2/objects-flexle7c3", |
| "root_dir": "/training/EmbodiedDyna-psi0-framesamp-data/psi-data-sg2t37v3varlen/799b00f3a1e08a0328d54bff13f1b8d8a7489205/extracted", |
| "symbolic_memory_mode": "disabled", |
| "symbolic_memory_rows_path": null, |
| "symbolic_memory_seed": 292285, |
| "symbolic_use_phase_action_mask": true, |
| "train_repo_ids": [ |
| "SimFactoryShellGame2T-v0" |
| ], |
| "transform": { |
| "field": { |
| "action_max": [ |
| 0.20344078540802002, |
| 0.17177540063858032, |
| 0.5726423859596252, |
| 0.060586556792259216, |
| 0.030544286593794823, |
| 0.03791704773902893, |
| 0.03118515945971012, |
| 1.0442190170288086, |
| 0.3619808554649353, |
| 4.94015648655477e-06, |
| 1.567848563194275, |
| 1.737970232963562, |
| 1.5620214939117432, |
| 1.5995635986328125, |
| 0.026498660445213318, |
| 0.43013057112693787, |
| 0.04635608568787575, |
| 0.06244596838951111, |
| 0.04676114767789841, |
| 0.012783452868461609, |
| 0.41602256894111633, |
| 0.003717988496646285, |
| 0.2647508382797241, |
| 0.5406146049499512, |
| 0.9597413539886475, |
| 1.1283698081970215, |
| 1.4887322187423706, |
| 0.7542539238929749, |
| 0.14426511526107788, |
| 0.111517533659935, |
| 0.0343802385032177, |
| 0.75, |
| 0.25, |
| 0.20000000298023224, |
| 0.0, |
| 0.08828353881835938 |
| ], |
| "action_min": [ |
| -0.2876609265804291, |
| -0.14852772653102875, |
| -0.04031404107809067, |
| -0.2837502062320709, |
| -0.3911275565624237, |
| -0.4982183575630188, |
| -0.4393625855445862, |
| -1.0449014902114868, |
| -0.25999966263771057, |
| -0.9059439301490784, |
| -7.455036393366754e-05, |
| -1.7042308172676712e-05, |
| 0.0, |
| 0.0, |
| -0.05483657866716385, |
| -0.04589128494262695, |
| -0.4559757709503174, |
| 0.0, |
| -0.4417652189731598, |
| -0.12400467693805695, |
| -0.04442235454916954, |
| -1.1999903917312622, |
| -0.7976195812225342, |
| -0.3771253526210785, |
| -0.704856812953949, |
| -0.27635878324508667, |
| 0.0, |
| -1.557715654373169, |
| -0.05882887542247772, |
| -0.16772495210170746, |
| -0.14286065101623535, |
| 0.75, |
| -0.18000000715255737, |
| -0.20000000298023224, |
| 0.0, |
| -0.053018126636743546 |
| ], |
| "action_norm_masks": [ |
| true, |
| true, |
| true, |
| true, |
| true, |
| true, |
| false |
| ], |
| "action_norm_type": "bounds", |
| "normalize_state": true, |
| "pad_action_dim": 36, |
| "pad_state_dim": 36, |
| "stat_action_key": "action", |
| "stat_path": "meta/stats_psi0.json", |
| "stat_state_key": "states", |
| "state_max": [ |
| 0.20360559225082397, |
| 0.1720379739999771, |
| 0.5726733803749084, |
| 0.0044308616779744625, |
| 0.0022593322210013866, |
| 0.004013810306787491, |
| 0.002306972397491336, |
| 1.0320099592208862, |
| 0.7252905964851379, |
| 0.017051663249731064, |
| 1.1951128244400024, |
| 1.696426272392273, |
| 1.0122944116592407, |
| 1.5571244955062866, |
| 0.03242148831486702, |
| 0.42924612760543823, |
| 0.0469878651201725, |
| 0.06706680357456207, |
| 0.04684685170650482, |
| 0.0148002402856946, |
| 0.41609084606170654, |
| 0.010240839794278145, |
| 0.264823317527771, |
| 0.542106568813324, |
| 0.9625779986381531, |
| 1.1291245222091675, |
| 1.4903392791748047, |
| 0.7545167207717896, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.75, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0 |
| ], |
| "state_min": [ |
| -0.2878865599632263, |
| -0.1484350711107254, |
| -0.0029822050128132105, |
| -0.2838318347930908, |
| -0.39114177227020264, |
| -0.4984891712665558, |
| -0.43937596678733826, |
| -1.0394816398620605, |
| -0.051454901695251465, |
| -0.8826939463615417, |
| -0.024508588016033173, |
| -0.019203325733542442, |
| -0.017816338688135147, |
| -0.019350627437233925, |
| -0.05183826759457588, |
| -0.047514408826828, |
| -0.45884960889816284, |
| 0.00024363555712625384, |
| -0.4422004818916321, |
| -0.12305071204900742, |
| -0.04441813379526138, |
| -1.1937716007232666, |
| -0.7942893505096436, |
| -0.37647077441215515, |
| -0.7034225463867188, |
| -0.2765703499317169, |
| -6.341539119603112e-05, |
| -1.5577093362808228, |
| 0.0, |
| -0.15000000596046448, |
| 0.0, |
| 0.75, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0 |
| ], |
| "use_norm_mask": false |
| }, |
| "model": { |
| "adaptive_resize": false, |
| "center_crop": { |
| "size": [ |
| 180, |
| 320 |
| ] |
| }, |
| "color_jitter": { |
| "brightness": 0.2, |
| "contrast": [ |
| 0.8, |
| 1.2 |
| ], |
| "hue": 0.05, |
| "saturation": [ |
| 0.8, |
| 1.2 |
| ] |
| }, |
| "gaussian_noise": { |
| "mean": 0.0, |
| "prob_skip": 0.1, |
| "std": 3.0 |
| }, |
| "img_aug": true, |
| "img_sizes": { |
| "egodex": [ |
| 270, |
| 480 |
| ], |
| "he": [ |
| 240, |
| 320 |
| ] |
| }, |
| "resize": { |
| "size": [ |
| 180, |
| 320 |
| ] |
| } |
| }, |
| "repack": { |
| "action_chunk_size": 30, |
| "dataset_name": "simple", |
| "num_past_frames": 0, |
| "pad_action_dim": 36, |
| "pad_state_dim": 36 |
| } |
| }, |
| "val_repo_ids": [ |
| "SimFactoryShellGame2T-v0" |
| ], |
| "video_backend": "decord" |
| }, |
| "debug": false, |
| "eval": false, |
| "exp": "r-50e6d5fadaaa", |
| "log": { |
| "log_freq": 100, |
| "logging_dir": "logs", |
| "report_to": "wandb" |
| }, |
| "model": { |
| "action_chunk_size": 30, |
| "action_dim": 36, |
| "action_exec_horizon": 30, |
| "activation": "gelu", |
| "combined_temb": false, |
| "data_flatten": true, |
| "data_packing": true, |
| "decision_head_enabled": true, |
| "decision_head_inputs": "both", |
| "decision_loss_weight": 0.5, |
| "decision_num_classes": 3, |
| "decision_teacher_forcing": 0.7, |
| "dim_feedforward": 2048, |
| "dropout": 0.1, |
| "early_fusion": false, |
| "eval_diffusion_steps": 10, |
| "framesamp_dataset_revision": "799b00f3a1e08a0328d54bff13f1b8d8a7489205", |
| "framesamp_enabled": true, |
| "framesamp_entry_probe_only": false, |
| "framesamp_feature_dim": 2048, |
| "framesamp_feature_manifest": "/training/EmbodiedDyna-psi0-framesamp-data/features-sg2t37v3varlen/qwen-vision-merger-raw1/4efc80fed43af5e3503f7fe3d803fc32a880c6db/SimFactoryShellGame2T-v0/manifest.json", |
| "framesamp_feature_manifest_sha256": "1d809e0cbf5d1018551ca01f46ff197ff32f654a9f79179b95c4623190764c23", |
| "framesamp_feature_scope": "full", |
| "framesamp_include_current": true, |
| "framesamp_memory_budget": 539, |
| "framesamp_memory_clock": "execution_frame", |
| "framesamp_memory_head_dim": 256, |
| "framesamp_memory_heads": 6, |
| "framesamp_memory_kv_heads": 1, |
| "framesamp_modulation_init": "robomme_normal_std_0_002", |
| "framesamp_off_clock_mapping": "all_raw_frames_le_current", |
| "framesamp_online_memory_view": "rgb_head_stereo_left", |
| "framesamp_padding_side": "right", |
| "framesamp_pool_type": "none", |
| "framesamp_position_dim": 768, |
| "framesamp_query_stride": 1, |
| "framesamp_temporal_position_source": "raw_frame_index", |
| "framesamp_tokens_per_frame": 77, |
| "framesamp_training_origin": "common_psi0_warm_start_step0", |
| "gradient_checkpointing": true, |
| "hidden_dim": 1536, |
| "img_chunk": 1, |
| "lang_backbone_lr": 1e-05, |
| "loss_w": [ |
| 0.1, |
| 0.2, |
| 0.1 |
| ], |
| "max_delay": 8, |
| "max_pixels": 451584, |
| "max_world_tokens": 256, |
| "min_pixels": 12544, |
| "mm_projector_lr": 1e-05, |
| "model_max_length": 4096, |
| "model_name_or_path": "/training/EmbodiedDyna/.psi/cache/checkpoints/psi0/pre.fast.1by1.2601091803.ckpt.ego200k.he30k", |
| "n_cams": 1, |
| "n_conditions": 0, |
| "native_keyframe_contract_version": 1, |
| "native_keyframe_enabled": true, |
| "native_keyframe_grid_hw": [ |
| 7, |
| 11 |
| ], |
| "native_keyframe_manifest_sha256": "67f7023b19d47bea3461aa97ec0fd983cc33acce374798ac8f17ce1f1c42a545", |
| "native_keyframe_root": "/training/keyframe-native7x77-sg2t37v3-20261002", |
| "native_keyframe_selector": "sam3", |
| "native_keyframe_spatial_coordinates": "cell_centers_normalized_to_16", |
| "nhead": 24, |
| "noise_scheduler": "flow", |
| "num_blocks": 6, |
| "object_memory_contract_version": 1, |
| "object_memory_enabled": true, |
| "object_memory_gate_init": "zero_out_proj", |
| "object_memory_geometry_dim": 8, |
| "object_memory_manifest_sha256": "875398dcb3e9e2e21d7c60689c5584fc0b3825a03fd26c39d434be51e12a2eee", |
| "object_memory_root": "/training/EmbodiedDyna-runs/sg2t37-v3varlen-20261002/sam3v2/objects-flexle7c3", |
| "object_memory_slots": 2, |
| "observation_horizon": 1, |
| "odim": 36, |
| "optim": "adamw_torch", |
| "pretrained_action_header_path": "/training/EmbodiedDyna/.psi/cache/checkpoints/psi0/postpre.1by1.pad36.2601131206.ckpt.he30k", |
| "resnet_store_path": null, |
| "rtc": true, |
| "share_cam_features": false, |
| "time_dim": 256, |
| "token_fusion": "concat", |
| "train_diffusion_steps": 1000, |
| "tune_mm_llm": false, |
| "tune_mm_mlp": false, |
| "tune_mm_vision": false, |
| "tune_vlm": false, |
| "use_dit": false, |
| "use_film": false, |
| "use_obs": "add_token", |
| "video_height": 160, |
| "video_latent_channels": 48, |
| "video_latent_frames": 3, |
| "video_num_frames": 9, |
| "video_vae_filename": "Wan2.2_VAE.pth", |
| "video_vae_path": null, |
| "video_vae_repo_id": "Wan-AI/Wan2.2-TI2V-5B", |
| "video_vae_tiled": false, |
| "video_width": 320, |
| "view_feature_dim": 2048, |
| "vision_tower_lr": 1e-06, |
| "vlm_ckpt_step": null, |
| "warm_start_checkpoint": "/training/EmbodiedDyna-runs/sg2t37-v3varlen-20261002/sam3v2/training-40k-dectf-bind/runs/finetune_keyframe_native_object/r-985fae5fc791/checkpoints/ckpt_40000", |
| "weight_decay": 0.01, |
| "world_learning_rate": null, |
| "world_loss_weight": 1.0, |
| "world_patch_size": 2 |
| }, |
| "seed": 292285, |
| "timestamp": "50e6d5fadaaa", |
| "train": { |
| "checkpointing_steps": 2500, |
| "data_parallel": "ddp", |
| "deepspeed_config": "/training/EmbodiedDyna-litlamp-fs-20260926/src/InternVLA/config/deepseeds/zero3.json", |
| "enable_gradient_checkpointing": true, |
| "enable_mixed_precision_training": true, |
| "gradient_accumulation_steps": 2, |
| "hf_token": ".hf_token", |
| "learning_rate": 5e-05, |
| "lora": false, |
| "lr_scheduler_kwargs": { |
| "betas": [ |
| 0.95, |
| 0.999 |
| ], |
| "eps": 1e-08, |
| "weight_decay": 1e-06 |
| }, |
| "lr_scheduler_type": "cosine", |
| "max_checkpoints_to_keep": 4, |
| "max_grad_norm": 1.0, |
| "max_training_steps": 10000, |
| "mixed_precision": "bf16", |
| "name": "finetune_keyframe_native_object", |
| "num_train_epochs": null, |
| "num_workers": 8, |
| "optimizer_foreach": null, |
| "output_dir": "/training/EmbodiedDyna-runs/sg2t37-v3varlen-20261002/sam3v2/training-10k-flexkf/runs", |
| "overfit_single_batch": false, |
| "reduce_in_full_precision": true, |
| "resume_from_checkpoint": null, |
| "scheduler_specific_kwargs": {}, |
| "sharding_strategy": "full-shard", |
| "skip_resumed_steps": false, |
| "train_batch_size": 8, |
| "val_batch_size": 8, |
| "val_num_batches": 20, |
| "validation_steps": 2000, |
| "warmup_ratio": null, |
| "warmup_steps": 200 |
| }, |
| "wandb": { |
| "entity": "wholebodymem", |
| "group": "r-522a7df20da1", |
| "id": "r-50e6d5fadaaa", |
| "name": "r-50e6d5fadaaa", |
| "project": "wholebodymem", |
| "resume": "never", |
| "tags": [ |
| "Psi0", |
| "native-keyframe", |
| "7x77", |
| "sam3", |
| "sam3-object-memory", |
| "gated-objects", |
| "gate-zero_out_proj", |
| "10k", |
| "ShellGame2tV3cf_flexkf10k", |
| "shellgame2tv3cf-flexkf10k", |
| "full-dataset", |
| "training-set-diagnostic", |
| "cpu-bf16-bank", |
| "decision-head-both", |
| "decision-teacher-forcing", |
| "warm-start" |
| ] |
| } |
| } |
|
|
