Add immutable checkpoint backup wan22_5b_lora_video_depth_camera_dino_5k resolved_run_config

stage1_4d_ablations/wan22_5b_lora_video_depth_camera_dino/step_005000/config.yaml

@@ -0,0 +1,536 @@
+output_dir: /scratch/shared/engs2907/logs/vgg_forecast_4d/train/flexpi_four_d_pretraining_wan22_5b_separate_streams_lora_modality_ladder_video_depth_camera_and_dino_20261101/run
+gpu_model: B200
+batch_size: 8
+num_workers: 8
+learning_rate: 5.0e-05
+num_epochs: 20
+max_steps: 10000
+log_every: 10
+save_every: 1000
+eval_every: 0
+eval_num_inference_steps: 25
+keep_last_n_states: 1
+keep_last_n_weights: 5
+eval_video: false
+share_pca_basis: true
+gradient_accumulation_steps: 4
+mixed_precision: bf16
+seed: 42
+max_grad_norm: 1.0
+weight_decay: 0.01
+resume: null
+resume_reanchor_lr_schedule: false
+resume_config_source: current
+pretrained_ckpt: null
+pretrained_ckpt_strict_shape: false
+wandb:
+  enabled: true
+  workspace: null
+  project: omni4d-robotwin
+  name: flexpi_four_d_pretraining_wan22_5b_separate_streams_lora_modality_ladder_video_depth_camera_and_dino
+  group: flexpi_four_d_pretraining_wan22_5b_separate_streams_lora_modality_ladder
+  mode: offline
+data:
+  train:
+    _target_: four_d_windows.FourDPretrainingWindows
+    sources:
+    - name: four_d_pretraining_relaunch_syn4d_sources
+      kind: track4d_source_config
+      config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_rendered_depth_sources_21_frames_512
+      weight: 15.2
+      left_out_sources:
+      - omniworld_game
+      - tartanair
+      - mvs_synth
+    - name: four_d_pretraining_relaunch_omniworld_game_source
+      kind: track4d_source_config
+      config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_rendered_depth_sources_21_frames_512
+      weight: 8.75
+      left_out_sources:
+      - syn4d_v1
+      - syn4d_v2_mixed
+      - syn4d_v2_mixed_no_sim
+      - tartanair
+      - mvs_synth
+    - name: four_d_pretraining_relaunch_synthverse_source
+      kind: track4d_source_config
+      config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_synthverse_dynamic_replica_vkitti_midair_spring_sources_without_midair_quality_masks_21_frames_512
+      weight: 11.05
+      left_out_sources:
+      - dynamic_replica
+      - vkitti
+      - midair
+      - spring
+    - name: four_d_pretraining_relaunch_vkitti_source
+      kind: track4d_source_config
+      config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_synthverse_dynamic_replica_vkitti_midair_spring_sources_without_midair_quality_masks_21_frames_512
+      weight: 6.0
+      left_out_sources:
+      - synthverse_depth
+      - dynamic_replica
+      - midair
+      - spring
+    - name: four_d_pretraining_relaunch_tartanair_sky_then_far_percentile_source
+      kind: track4d_source_config
+      config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+      weight: 6.0
+      left_out_sources:
+      - dynamic_replica
+      - spring
+      - midair
+      - mvs_synth
+      - tartanground
+      - paralleldomain4d
+    - name: four_d_pretraining_relaunch_dynamic_replica_stored_depth_source
+      kind: track4d_source_config
+      config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+      weight: 5.0
+      left_out_sources:
+      - tartanair
+      - spring
+      - midair
+      - mvs_synth
+      - tartanground
+      - paralleldomain4d
+    - name: four_d_pretraining_relaunch_spring_stored_depth_source
+      kind: track4d_source_config
+      config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+      weight: 8.0
+      left_out_sources:
+      - tartanair
+      - dynamic_replica
+      - midair
+      - mvs_synth
+      - tartanground
+      - paralleldomain4d
+    - name: four_d_pretraining_relaunch_midair_within_200_meters_source
+      kind: track4d_source_config
+      config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+      weight: 3.0
+      left_out_sources:
+      - tartanair
+      - dynamic_replica
+      - spring
+      - mvs_synth
+      - tartanground
+      - paralleldomain4d
+    - name: four_d_pretraining_relaunch_mvs_synth_1000_then_far_percentile_source
+      kind: track4d_source_config
+      config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+      weight: 3.0
+      left_out_sources:
+      - tartanair
+      - dynamic_replica
+      - spring
+      - midair
+      - tartanground
+      - paralleldomain4d
+    - name: four_d_pretraining_relaunch_tartanground_sky_then_far_percentile_source
+      kind: track4d_source_config
+      config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+      weight: 3.0
+      left_out_sources:
+      - tartanair
+      - dynamic_replica
+      - spring
+      - midair
+      - mvs_synth
+      - paralleldomain4d
+    - name: four_d_pretraining_relaunch_paralleldomain4d_sky_then_far_percentile_source
+      kind: track4d_source_config
+      config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+      weight: 10.0
+      left_out_sources:
+      - tartanair
+      - dynamic_replica
+      - spring
+      - midair
+      - mvs_synth
+      - tartanground
+    frames: 9
+    raster_hw:
+    - 288
+    - 512
+    raymap_camera:
+      origin_fraction: 0.3333333333333333
+      origin_range: 1.0
+      decode_margin: 16
+      origin_box_grid: 1
+    pointmap_log_factor: 0.4
+    pointmap_hole_fill:
+      synthverse_depth: far_clip
+      syn4d: far_clip
+      omniworld_game: far_clip
+      dynamic_replica: nearest_labelled_depth_of_3x3_neighbours
+      tartanair: far_clip
+      dl3dv: far_clip
+      scannet: telea_log_depth
+      virtual_kitti2: far_clip
+      midair: far_clip
+      pointodyssey: far_clip
+      mvs_synth: far_clip
+      wildrgbd: telea_log_depth
+      hypersim: far_clip
+      spring: far_clip
+      kubric49: far_clip
+      kubric81: far_clip
+      droid_stereo_depth: telea_log_depth
+      tartanground: far_clip
+      paralleldomain4d: far_clip
+    pointmap_gauge: window_labelled_median_depth
+    geometry_image: genception_log_depth
+    context_len: 512
+    batch_size: 8
+    stream_seed: 42
+    epoch_windows: 32000000
+    val_set_proportion: 0.0
+    resume_state: null
+    gradient_accumulation_steps: 4
+    loader_workers: 8
+    loader_opencv_threads: 1
+    window_captions:
+      caption_dropout: 0.1
+      missing_caption: empty_prompt
+      caches:
+        syn4d_v1_gemini_holo4d:
+          embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/syn4d_v1_gemini_holo4d_local_and_global_with_repaired_globals_20261031/captions_t5_emb.safetensors
+          meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/syn4d_v1_gemini_holo4d_local_and_global_with_repaired_globals_20261031/captions_t5_emb_meta.json
+        syn4d_v2_gemini_holo4d:
+          embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/syn4d_v2_gemini_holo4d_20261030/captions_t5_emb.safetensors
+          meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/syn4d_v2_gemini_holo4d_20261030/captions_t5_emb_meta.json
+        tartanair_gemini_holo4d:
+          embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/tartanair_gemini_holo4d/captions_t5_emb.safetensors
+          meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/tartanair_gemini_holo4d/captions_t5_emb_meta.json
+        mvs_synth_gemini_holo4d:
+          embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/mvs_synth_gemini_holo4d/captions_t5_emb.safetensors
+          meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/mvs_synth_gemini_holo4d/captions_t5_emb_meta.json
+        omniworld_game_gemini_local_official_text_chunks_and_gemini_global:
+          embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/omniworld_game_gemini_local_official_text_chunks_and_gemini_global_20261030b/captions_t5_emb.safetensors
+          meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/omniworld_game_gemini_local_official_text_chunks_and_gemini_global_20261030b/captions_t5_emb_meta.json
+        synthverse_gemini_holo4d:
+          embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/synthverse_gemini_holo4d_local96_and_global_20261031/captions_t5_emb.safetensors
+          meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/synthverse_gemini_holo4d_local96_and_global_20261031/captions_t5_emb_meta.json
+        dynamic_replica_gemini_holo4d:
+          embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/dynamic_replica_gemini_holo4d/captions_t5_emb.safetensors
+          meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/dynamic_replica_gemini_holo4d/captions_t5_emb_meta.json
+        vkitti_gemini_holo4d:
+          embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/vkitti_gemini_holo4d/captions_t5_emb.safetensors
+          meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/vkitti_gemini_holo4d/captions_t5_emb_meta.json
+        midair_gemini_holo4d:
+          embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/midair_gemini_holo4d_20261030/captions_t5_emb.safetensors
+          meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/midair_gemini_holo4d_20261030/captions_t5_emb_meta.json
+        spring_gemini_holo4d:
+          embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/spring_gemini_holo4d/captions_t5_emb.safetensors
+          meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/spring_gemini_holo4d/captions_t5_emb_meta.json
+        tartanground_gemini_holo4d:
+          embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/tartanground_gemini_holo4d_20261030/captions_t5_emb.safetensors
+          meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/tartanground_gemini_holo4d_20261030/captions_t5_emb_meta.json
+        paralleldomain4d_gemini_holo4d:
+          embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/paralleldomain4d_gemini_holo4d_20261030/captions_t5_emb.safetensors
+          meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/paralleldomain4d_gemini_holo4d_20261030/captions_t5_emb_meta.json
+      stanzas:
+        syn4d_v1:
+          product: syn4d_v1_gemini_holo4d
+          join: local_covering_then_global
+        syn4d_v2_mixed:
+          product: syn4d_v2_gemini_holo4d
+          join: local_covering_then_global
+        syn4d_v2_mixed_no_sim:
+          product: syn4d_v2_gemini_holo4d
+          join: local_covering_then_global
+        tartanair:
+          product: tartanair_gemini_holo4d
+          join: local_covering
+        mvs_synth:
+          product: mvs_synth_gemini_holo4d
+          join: local_covering
+        omniworld_game:
+          product: omniworld_game_gemini_local_official_text_chunks_and_gemini_global
+          join: local_covering_then_second_local_covering_then_global
+        pointodyssey_test: none
+        kubric49_validation: none
+        wildrgbd_test: none
+        vkitti_test:
+          product: vkitti_gemini_holo4d
+          join: local_covering_then_global
+        droid_held_out: none
+        synthverse_depth:
+          product: synthverse_gemini_holo4d
+          join: local_covering_then_global
+        dynamic_replica:
+          product: dynamic_replica_gemini_holo4d
+          join: global
+        vkitti:
+          product: vkitti_gemini_holo4d
+          join: local_covering_then_global
+        midair:
+          product: midair_gemini_holo4d
+          join: local_covering_then_global
+        spring:
+          product: spring_gemini_holo4d
+          join: global
+        tartanground:
+          product: tartanground_gemini_holo4d
+          join: local_covering_then_global
+        paralleldomain4d:
+          product: paralleldomain4d_gemini_holo4d
+          join: global
+        paralleldomain4d_validation:
+          product: paralleldomain4d_gemini_holo4d
+          join: global
+    training_scene_exclusions:
+      reason: rendered depth passes through glass (raw EXR depth through glass panes
+        at 190-260 m), owner 10-29; evidence /scratch/shared/engs2907/logs/vgg_forecast_4d/tools/syn4d_glass_depth_check_20261029;
+        scifiroom_bald added 10-30 from /scratch/shared/engs2907/logs/vgg_forecast_4d/tools/depth_clipping_audit_20261030
+      stanzas:
+        syn4d_v1:
+        - bigoffice_v1
+        - lab_bald
+        - office_bald
+        - scifiroom_bald
+        syn4d_v2_mixed:
+        - citynight
+        - mall
+        - metro
+        syn4d_v2_mixed_no_sim:
+        - citynight
+        - mall
+        - metro
+model:
+  _target_: flexpi_track_video.create_flexpi_track_video
+  model_id: Wan-AI/Wan2.2-TI2V-5B
+  tokenizer_model_id: Wan-AI/Wan2.1-T2V-1.3B
+  tokenizer_max_len: 128
+  load_text_encoder: false
+  proprio_dim: null
+  redirect_common_files: true
+  mot_checkpoint_mixed_attn: true
+  action_dit_pretrained_path: null
+  skip_dit_load_from_pretrain: false
+  dino_dim: 768
+  dino_model_name: vit_base_patch16_dinov3.lvd1689m
+  composite_layout: tshape_robotwin_384x320_uniform_dino_on_video_grid
+  dino_cam_patches: null
+  dino_cam_regions: null
+  dino_temporal_stride: 1
+  dino_stride_keep_far: true
+  dino_pred_x0: true
+  dino_pool_mode: avg
+  freeze_dino_encoder: true
+  dino_pixel_unshuffle: 2
+  enable_pointmap: true
+  pointmap_max_depth_m: 2.0
+  pointmap_norm_bounds:
+    x:
+    - -0.5
+    - 0.5
+    'y':
+    - -0.5
+    - 0.5
+    z:
+    - 0.0
+    - 1.5
+  pointmap_depth_vis_mode: turbo
+  joint_video: true
+  joint_dino: true
+  joint_pointmap: true
+  flex_joint:
+    enabled: true
+    p_present_video: 0.5
+    p_present_dino: 0.5
+    p_present_pointmap: 0.5
+    p_jv: 0.5
+    p_jd: 0.5
+    p_jp: 0.5
+    cross_modal_predict_video: true
+    cross_modal_predict_dino: true
+    cross_modal_predict_pointmap: true
+  video_dit_config:
+    has_image_input: false
+    patch_size:
+    - 1
+    - 2
+    - 2
+    in_dim: 48
+    hidden_dim: 3072
+    ffn_dim: 14336
+    freq_dim: 256
+    text_dim: 4096
+    out_dim: 48
+    num_heads: 24
+    attn_head_dim: 128
+    num_layers: 30
+    eps: 1.0e-06
+    seperated_timestep: true
+    require_clip_embedding: false
+    require_vae_embedding: false
+    fuse_vae_embedding_in_latents: true
+    use_gradient_checkpointing: true
+    video_attention_mask_mode: first_frame_causal
+    action_conditioned: false
+    action_dim: 14
+    action_group_causal_mask_mode: group_diagonal
+  action_dit_config: null
+  video_scheduler:
+    train_shift: 6.0
+    infer_shift: 6.0
+    num_train_timesteps: 1000
+  action_scheduler:
+    train_shift: 1.0
+    infer_shift: 1.0
+    num_train_timesteps: 1000
+  dino_scheduler:
+    train_shift: 6.0
+    infer_shift: 6.0
+    num_train_timesteps: 1000
+  pointmap_scheduler:
+    train_shift: 6.0
+    infer_shift: 6.0
+    num_train_timesteps: 1000
+  loss:
+    lambda_action: 1.0
+    lambda_dino: 1.0
+    lambda_pointmap: 1.0
+    lambda_video: 1.0
+  hbridge:
+    enabled: true
+    bottom_ratio: 0.25
+    top_ratio: 0.25
+  track_video:
+    attention: visual_stream
+    sparse_action: true
+    train_shift: 6.0
+    infer_shift: 6.0
+    loss_lambda: 1.0
+    action_edge: two_way
+    flex:
+      p_present: 1.0
+      p_joint: 0.5
+      cross_modal_predict: true
+  noise_level_coupling: shared_uniform
+  raymap_camera:
+    origin_fraction: 0.3333333333333333
+    origin_range: 1.0
+    decode_margin: 16
+    origin_box_grid: 1
+    train_shift: 6.0
+    infer_shift: 6.0
+    loss_lambda: 1.0
+  four_d_pretraining:
+    geometry_streams: generated
+    dino_stream: generated
+    video_loss_lambda: 1.0
+    modality_forcing:
+      video:
+        p_present: 0.5
+        p_joint: 0.5
+        cross_modal: true
+      dino:
+        p_present: 0.5
+        p_joint: 0.5
+        cross_modal: true
+      pointmap:
+        p_present: 0.5
+        p_joint: 0.5
+        cross_modal: true
+      camera:
+        p_present: 0.5
+        p_joint: 0.5
+        cross_modal: true
+      training_mode_shares:
+        flexible_joint: 1.0
+        v2d: 0.0
+        v2c: 0.0
+        c2v_from_first_frame: 0.0
+        c2vd_from_first_frame: 0.0
+        c2v: 0.0
+        c2vd: 0.0
+        vd2c: 0.0
+        vc2d: 0.0
+        v2dc: 0.0
+        dc2v_from_first_frame: 0.0
+        d2vc_from_first_frame: 0.0
+        d2v_from_first_frame: 0.0
+        dc2v: 0.0
+        d2vc: 0.0
+        d2v: 0.0
+        d2c: 0.0
+        c2d: 0.0
+        v2d_dino_generated: 0.0
+        v2c_dino_generated: 0.0
+        v2dc_dino_generated: 0.0
+      draw_rules:
+        first_frame_anchor: false
+        absent:
+          video: 0.0
+          dino: 0.0
+          pointmap: 0.0
+          camera: 0.0
+        whole_given:
+          video: 0.3333333333333333
+          dino: 0.3333333333333333
+          pointmap: 0.3333333333333333
+          camera: 0.3333333333333333
+    pointmap_log_factor: 0.4
+  channel_concat: null
+  stage_one_without_action_expert: true
+  stream_only_end_blocks:
+    bottom: 7
+    top: 7
+minimum_learning_rate: 1.0e-06
+training_scope: the 5B stage-1 modality ladder's arm with every stream (video, depth
+  image, RayMap camera, DINO) as separate token streams, LoRA rank 128, one shared
+  generated noise level, global batch 32 on one B200, 10,000 updates
+native_control_fps: 50
+action_dimension: 14
+initialization: native_wan_actiondit
+track_video_encoding:
+  record_root: /scratch/shared/engs2907/logs/vgg_forecast_4d/data/robotwin_robot_records_all_tasks_7bf528a2_20261023
+  embodiment_root: /scratch/shared/engs2907/checkouts/robotwin_objects_b5230d8b_266f3aad/RoboTwin/assets/embodiments/aloha-agilex
+  surface_points: 8192
+  min_surface_points_per_body: 16
+  query_count:
+  - 64
+  - 256
+  camera_query_quota:
+  - 224
+  - 16
+  - 16
+  min_visible_queries: 16
+  query_draw: per_read
+  query_seed: 0
+  visibility_margin_m: 0.01
+  pixel_center: 0.0
+  colouring: frame0_head_xyz
+  xyz_normalization: quantile_0.02_0.98
+  hues: null
+  neighbour_px: null
+  colour_floor: 0.2
+  min_gap_px:
+  - 8
+  - 6
+  - 6
+  wrist_dense_links:
+  - fl_link7
+  - fl_link8
+  - fr_link7
+  - fr_link8
+  wrist_dense_points: 16384
+  disc_radius_px: 2
+  occluded_style: ring
+  ring_radius_px: 3
+  ring_width_px: 1
+deepspeed_gradient_clipping: true
+warmup_steps: 500
+loader_prefetch_factor: 2
+loader_opencv_threads: 1
+weights_milestone_every: 10000
+fp32_milestone_every: null
+train:
+  trainable_parameters:
+    mode: lora
+    lora:
+      rank: 128
+      alpha: 128
+      target_modules:
+      - all_block_linears