Add immutable checkpoint backup cosmos3_nano_stage_one_lora_r256_replicate_only_500k_step_006000_20261104 data_config_provenance

cosmos3_nano_stage_one_lora_r256_replicate_only_500k_20261103/attempt01/step_006000/source_windows.yaml

@@ -0,0 +1,387 @@
+vgg_repository: /vgg/yushi/Repo/vgg_forecast_4d_cosmos_pinned_source_20261102
+vgg_commit: f5a59cd511f20c20cf09e2edd21781ca6907d4c8
+recipe_source_config: /scratch/shared/engs2907/logs/vgg_forecast_4d/train/flexpi_four_d_pretraining_wan22_5b_separate_streams_full_parameters_video_depth_camera_dino_and_sparse_tracking_video_from_the_15000_update_four_d_run_20261101/attempt02/run/config.yaml
+recipe_source_config_sha256: 72590f4598cae1b329c31b3cb45d059bdaeca23642faa10ca3f31372f9ebc0aa
+window_block:
+  _target_: four_d_windows.FourDPretrainingWindows
+  sources:
+  - name: four_d_pretraining_relaunch_syn4d_sources
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_rendered_depth_sources_21_frames_512
+    weight: 15.2
+    left_out_sources:
+    - omniworld_game
+    - tartanair
+    - mvs_synth
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_omniworld_game_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_rendered_depth_sources_21_frames_512
+    weight: 8.75
+    left_out_sources:
+    - syn4d_v1
+    - syn4d_v2_mixed
+    - syn4d_v2_mixed_no_sim
+    - tartanair
+    - mvs_synth
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_synthverse_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_synthverse_dynamic_replica_vkitti_midair_spring_sources_without_midair_quality_masks_21_frames_512
+    weight: 11.05
+    left_out_sources:
+    - dynamic_replica
+    - vkitti
+    - midair
+    - spring
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_vkitti_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_synthverse_dynamic_replica_vkitti_midair_spring_sources_without_midair_quality_masks_21_frames_512
+    weight: 6.0
+    left_out_sources:
+    - synthverse_depth
+    - dynamic_replica
+    - midair
+    - spring
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_tartanair_sky_then_far_percentile_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 6.0
+    left_out_sources:
+    - dynamic_replica
+    - spring
+    - midair
+    - mvs_synth
+    - tartanground
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_dynamic_replica_stored_depth_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 5.0
+    left_out_sources:
+    - tartanair
+    - spring
+    - midair
+    - mvs_synth
+    - tartanground
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_spring_stored_depth_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 8.0
+    left_out_sources:
+    - tartanair
+    - dynamic_replica
+    - midair
+    - mvs_synth
+    - tartanground
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_midair_within_200_meters_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 3.0
+    left_out_sources:
+    - tartanair
+    - dynamic_replica
+    - spring
+    - mvs_synth
+    - tartanground
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_mvs_synth_1000_then_far_percentile_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 3.0
+    left_out_sources:
+    - tartanair
+    - dynamic_replica
+    - spring
+    - midair
+    - tartanground
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_tartanground_sky_then_far_percentile_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 3.0
+    left_out_sources:
+    - tartanair
+    - dynamic_replica
+    - spring
+    - midair
+    - mvs_synth
+    - paralleldomain4d
+    condition_only_streams: []
+  - name: four_d_pretraining_relaunch_paralleldomain4d_sky_then_far_percentile_source
+    kind: track4d_source_config
+    config: train/mot/wan_video_depth_vae_from_video_four_d_pretraining_relaunch_sky_and_far_percentile_depth_validity_rules_21_frames_512
+    weight: 10.0
+    left_out_sources:
+    - tartanair
+    - dynamic_replica
+    - spring
+    - midair
+    - mvs_synth
+    - tartanground
+    condition_only_streams: []
+  frames: 9
+  raster_hw:
+  - 288
+  - 512
+  raymap_camera:
+    origin_fraction: 0.3333333333333333
+    origin_range: 1.0
+    decode_margin: 16
+    origin_box_grid: 1
+  pointmap_log_factor: 0.4
+  pointmap_hole_fill:
+    synthverse_depth: far_clip
+    syn4d: far_clip
+    omniworld_game: far_clip
+    dynamic_replica: nearest_labelled_depth_of_3x3_neighbours
+    tartanair: far_clip
+    dl3dv: far_clip
+    scannet: telea_log_depth
+    virtual_kitti2: far_clip
+    midair: far_clip
+    pointodyssey: far_clip
+    mvs_synth: far_clip
+    wildrgbd: telea_log_depth
+    hypersim: far_clip
+    spring: far_clip
+    kubric49: far_clip
+    kubric81: far_clip
+    droid_stereo_depth: telea_log_depth
+    tartanground: far_clip
+    paralleldomain4d: far_clip
+    syn4d_v2: far_clip
+  pointmap_gauge: window_labelled_median_depth
+  geometry_image: genception_log_depth
+  context_len: 512
+  batch_size: 16
+  stream_seed: 42
+  epoch_windows: 32000000
+  val_set_proportion: 0.0
+  resume_state: null
+  gradient_accumulation_steps: 1
+  loader_workers: 12
+  loader_opencv_threads: 1
+  window_captions:
+    caption_dropout: 0.1
+    missing_caption: empty_prompt
+    caches:
+      syn4d_v1_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/syn4d_v1_gemini_holo4d_local_and_global_with_repaired_globals_20261031/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/syn4d_v1_gemini_holo4d_local_and_global_with_repaired_globals_20261031/captions_t5_emb_meta.json
+      syn4d_v2_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/syn4d_v2_gemini_holo4d_20261030/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/syn4d_v2_gemini_holo4d_20261030/captions_t5_emb_meta.json
+      tartanair_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/tartanair_gemini_holo4d/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/tartanair_gemini_holo4d/captions_t5_emb_meta.json
+      mvs_synth_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/mvs_synth_gemini_holo4d/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/mvs_synth_gemini_holo4d/captions_t5_emb_meta.json
+      omniworld_game_gemini_local_official_text_chunks_and_gemini_global:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/omniworld_game_gemini_local_official_text_chunks_and_gemini_global_20261030b/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/omniworld_game_gemini_local_official_text_chunks_and_gemini_global_20261030b/captions_t5_emb_meta.json
+      synthverse_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/synthverse_gemini_holo4d_local96_and_global_20261031/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/synthverse_gemini_holo4d_local96_and_global_20261031/captions_t5_emb_meta.json
+      dynamic_replica_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/dynamic_replica_gemini_holo4d/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/dynamic_replica_gemini_holo4d/captions_t5_emb_meta.json
+      vkitti_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/vkitti_gemini_holo4d/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/vkitti_gemini_holo4d/captions_t5_emb_meta.json
+      midair_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/midair_gemini_holo4d_20261030/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/midair_gemini_holo4d_20261030/captions_t5_emb_meta.json
+      spring_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/spring_gemini_holo4d/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/spring_gemini_holo4d/captions_t5_emb_meta.json
+      tartanground_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/tartanground_gemini_holo4d_20261030/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/tartanground_gemini_holo4d_20261030/captions_t5_emb_meta.json
+      paralleldomain4d_gemini_holo4d:
+        embeddings: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/paralleldomain4d_gemini_holo4d_20261030/captions_t5_emb.safetensors
+        meta: /scratch/shared/engs2907/logs/vgg_forecast_4d/cache/wan_umt5_captions/four_d_window_captions_512/paralleldomain4d_gemini_holo4d_20261030/captions_t5_emb_meta.json
+    stanzas:
+      syn4d_v1:
+        product: syn4d_v1_gemini_holo4d
+        join: local_covering_then_global
+      syn4d_v2_mixed:
+        product: syn4d_v2_gemini_holo4d
+        join: local_covering_then_global
+      syn4d_v2_mixed_no_sim:
+        product: syn4d_v2_gemini_holo4d
+        join: local_covering_then_global
+      tartanair:
+        product: tartanair_gemini_holo4d
+        join: local_covering
+      mvs_synth:
+        product: mvs_synth_gemini_holo4d
+        join: local_covering
+      omniworld_game:
+        product: omniworld_game_gemini_local_official_text_chunks_and_gemini_global
+        join: local_covering_then_second_local_covering_then_global
+      pointodyssey_test: none
+      kubric49_validation: none
+      wildrgbd_test: none
+      vkitti_test:
+        product: vkitti_gemini_holo4d
+        join: local_covering_then_global
+      droid_held_out: none
+      synthverse_depth:
+        product: synthverse_gemini_holo4d
+        join: local_covering_then_global
+      dynamic_replica:
+        product: dynamic_replica_gemini_holo4d
+        join: global
+      vkitti:
+        product: vkitti_gemini_holo4d
+        join: local_covering_then_global
+      midair:
+        product: midair_gemini_holo4d
+        join: local_covering_then_global
+      spring:
+        product: spring_gemini_holo4d
+        join: global
+      tartanground:
+        product: tartanground_gemini_holo4d
+        join: local_covering_then_global
+      paralleldomain4d:
+        product: paralleldomain4d_gemini_holo4d
+        join: global
+      paralleldomain4d_validation:
+        product: paralleldomain4d_gemini_holo4d
+        join: global
+  training_scene_exclusions:
+    reason: rendered depth passes through glass (raw EXR depth through glass panes
+      at 190-260 m), owner 10-29; evidence /scratch/shared/engs2907/logs/vgg_forecast_4d/tools/syn4d_glass_depth_check_20261029;
+      scifiroom_bald added 10-30 from /scratch/shared/engs2907/logs/vgg_forecast_4d/tools/depth_clipping_audit_20261030
+    stanzas:
+      syn4d_v1:
+      - bigoffice_v1
+      - lab_bald
+      - office_bald
+      - scifiroom_bald
+      syn4d_v2_mixed:
+      - citynight
+      - mall
+      - metro
+      syn4d_v2_mixed_no_sim:
+      - citynight
+      - mall
+      - metro
+  tracking_video:
+    query_grid_px: 10
+    disc_radius_px: 2
+    occluded_style: ring
+    ring_radius_px: 3
+    ring_width_px: 1
+    xyz_normalization: quantile_0.02_0.98
+    colour_floor: 0.2
+    min_frame_zero_points: 16
+    surface_test:
+      relative_tolerance: 0.1
+      absolute_tolerance: 0.05
+    stanza_rules:
+      tartanair: static_scene_points
+      midair: static_scene_points
+      mvs_synth: static_scene_points
+      tartanground: static_scene_points
+      syn4d_v1: tracked_points
+      syn4d_v2_mixed: tracked_points
+      syn4d_v2_mixed_no_sim: tracked_points
+      omniworld_game: no_tracking_video
+      synthverse_depth: no_tracking_video
+      dynamic_replica: tracked_points
+      vkitti: no_tracking_video
+      spring: no_tracking_video
+      paralleldomain4d: no_tracking_video
+      scannet: static_scene_points
+      wildrgbd_test: static_scene_points
+      pointodyssey_test: tracked_points
+      kubric49_validation: no_tracking_video
+      vkitti_test: no_tracking_video
+      droid_held_out: no_tracking_video
+      paralleldomain4d_validation: no_tracking_video
+      dynamic_replica_test: tracked_points
+    stanza_track_reading:
+      syn4d_v1:
+        stanza:
+          capability: dynamic_cross_time
+          supervision:
+          - pointmap
+          - source_motion
+          annotation_provenance: syn4d_mesh_surface_forward_track_from_frame0
+          allow_missing_tracks: false
+        kwargs:
+          return_tracks: true
+          depth_only: false
+          metadata_root: /vgg/chenzhao/syn4d_metadata
+          wrapped_face_distance_thresh: 0.1
+          query_inconsistent_distance_thresh: 0.1
+      pointodyssey_test:
+        stanza:
+          supervision:
+          - pointmap
+          - source_motion
+        kwargs:
+          return_tracks: true
+      dynamic_replica_test:
+        stanza:
+          supervision:
+          - pointmap
+          - source_motion
+        kwargs:
+          return_tracks: true
+      syn4d_v2_mixed:
+        loader: syn4d_v2
+        stanza:
+          capability: dynamic_cross_time
+          supervision:
+          - pointmap
+          - source_motion
+          annotation_provenance: syn4d_v2_mesh_surface_forward_track_from_frame0
+          allow_missing_tracks: false
+        kwargs:
+          return_tracks: true
+          depth_only: false
+          metadata_root: /vgg/chenzhao/syn4d_metadata
+          tracking_format: safetensor
+          mask_bg: false
+          trusted_background_depth_max: 80.0
+          wrapped_face_distance_thresh: 0.1
+          query_inconsistent_distance_thresh: 0.1
+      syn4d_v2_mixed_no_sim:
+        loader: syn4d_v2
+        stanza:
+          capability: dynamic_cross_time
+          supervision:
+          - pointmap
+          - source_motion
+          annotation_provenance: syn4d_v2_mesh_surface_forward_track_from_frame0
+          allow_missing_tracks: false
+        kwargs:
+          return_tracks: true
+          depth_only: false
+          metadata_root: /vgg/chenzhao/syn4d_metadata
+          tracking_format: safetensor
+          mask_bg: false
+          trusted_background_depth_max: 80.0
+          wrapped_face_distance_thresh: 0.1
+          query_inconsistent_distance_thresh: 0.1
+      dynamic_replica:
+        stanza:
+          supervision:
+          - pointmap
+          - source_motion
+        kwargs:
+          return_tracks: true