| { |
| "format": "oneworld-rae-inference-v2", |
| "step": 30000, |
| "contract": { |
| "version": 19, |
| "parent_checkpoint": "", |
| "resolution_hw": null, |
| "resolutions_hw": [ |
| [ |
| 224, |
| 448 |
| ], |
| [ |
| 448, |
| 448 |
| ], |
| [ |
| 252, |
| 448 |
| ], |
| [ |
| 336, |
| 448 |
| ] |
| ], |
| "patch_grids_hw": [ |
| [ |
| 16, |
| 32 |
| ], |
| [ |
| 32, |
| 32 |
| ], |
| [ |
| 18, |
| 32 |
| ], |
| [ |
| 24, |
| 32 |
| ] |
| ], |
| "context_views": 8, |
| "heldout_target_views": 8, |
| "boundary_name": "independent_layernorm_stock_pi3_z18_and_rgb32", |
| "boundary_block_zero_based": 17, |
| "boundary_attention": "global", |
| "pre_boundary": "frozen_stock_pi3_blocks_0_to_17_no_rgb_injection", |
| "continuation_blocks": [ |
| 18, |
| 35 |
| ], |
| "register_tokens": "not_generated_shared_five_learned_parameters", |
| "pose": "all_16_opencv_c2w_ref0_pi3_native_mean_baseline_scale_aug_1", |
| "view_condition": "fixed_context_view0_identity_pose_plus_intrinsic_rays;remaining_context_views_permutation_equivariant", |
| "decoders": "independent_full_resolution_rgb_dpt_and_gaussian_dpt;pi3_point_head_depth_on_intrinsic_pixel_rays", |
| "level_fusion": "independent_four_scale_softmax_gates", |
| "level_drop_probability": 0.0, |
| "level_drop_start_step": 30000, |
| "decoder_noise": { |
| "distribution": "post_fusion_layernorm_per_scene_uniform_sigma_iid_gaussian", |
| "tau": 0.0, |
| "probability": 1.0, |
| "start_step": 0, |
| "shared_strength_across_views": true |
| }, |
| "nvs": "eight_endpoint_bracketing_context_to_eight_heldout_interior_targets", |
| "view_split": "eight_evenly_spaced_context_including_both_endpoints_plus_eight_interior_targets", |
| "losses": "balanced_rgb_nvs_mse_plus_lpips_plus_stock_pi3_depth_and_rendered_depth_no_gan", |
| "loss_weights": { |
| "rgb": 1.0, |
| "nvs": 1.0, |
| "merged_context_rgb": 1.0, |
| "lpips": 0.05, |
| "lpips_start": 0, |
| "teacher_depth": 0.01, |
| "render_depth": 0.0 |
| }, |
| "gan": { |
| "discriminator": "dino_s8_ks9_bn_spectral_norm", |
| "disc_loss": "hinge", |
| "generator_loss": "negative_mean_fake_logits", |
| "weight": 0.0, |
| "adaptive_max": 10000.0, |
| "disc_start_step": 3750, |
| "generator_start_step": 5000 |
| }, |
| "latent_channels": { |
| "stock_pi3_z18": 1024, |
| "svg_rgb": 32 |
| }, |
| "latent_concat_dim": 1056, |
| "dit_latent": "concat(layernorm_stock_pi3_z18,layernorm_rgb32);reversible_per_resolution_channel_standardization_deferred_to_dit", |
| "decoder_adapter": "layernorm_z18_plus_zero_init_mlp_layernorm_rgb32_to_1024_then_layernorm_then_fixed_channel_denorm", |
| "decoder_taps": [ |
| 17, |
| 35 |
| ], |
| "rgb_alignment": "independent_per_token_layernorm_no_affine", |
| "post_concat_norm": null, |
| "rgb_direct_skip": "scene_shared_drop_p0_layernorm_rgb32_exact_patch14_full_resolution_detail_merger_before_rgb_output_conv_not_rgb_dpt_concat", |
| "rgb_trunk_condition": "zero_init_32_to_256_to_1024_fixed_gate_0.1", |
| "rgb_branch_mask": "before_alignment_shared_per_multiview_scene", |
| "rgb_branch_drop": 0.0, |
| "rgb_branch_drop_scope": "scene_shared_synchronized_full_resolution_rgb_and_gaussian_input_mergers;block19_condition_keeps_clean_rgb", |
| "rgb_transformer_dropout": 0.0, |
| "training_scope": "full", |
| "overfit_scene_pool_size": 0, |
| "dataset": "mixed_re10k_nvs_refined", |
| "data_mix": "exact_global_sample_parity_50pct_nvs_refined_static_50pct_legacy_re10k", |
| "nvs_refined_subsets": [ |
| "ACID", |
| "DL3DV", |
| "Re10K" |
| ], |
| "source_view_sampling": { |
| "legacy_re10k": "ordered_16_views_existing_loader", |
| "nvs_refined_acid": "uniform_16_view_window_span_48_160_endpoint_bracketed_8_to_8", |
| "nvs_refined_re10k": "uniform_16_view_window_span_48_160_endpoint_bracketed_8_to_8", |
| "nvs_refined_dl3dv": "depthsplat_adapted_span_warmup_15_30_to_20_50_over_10k;8_context=temporal_endpoints_plus_camera_center_fps;8_distinct_targets_inside_context_window" |
| }, |
| "resolution_schedule": "global_step_synchronized_shuffled_uniform_cycle", |
| "normalization": { |
| "z18": "fixed_layernorm_per_token_no_affine", |
| "rgb32": "fixed_layernorm_per_token_no_affine", |
| "fused_block18_input": "fixed_layernorm_per_token_no_affine", |
| "z36_decoder_tap": "trainable_layernorm_per_token" |
| }, |
| "gaussians": { |
| "centres": "pi3_depth_on_intrinsic_pixel_rays_no_xy_offset", |
| "ray_mode": "z_depth", |
| "sh_degree": 1, |
| "attribute_channels": 20, |
| "rgb_skip": true, |
| "merged_context_rgb_weight": 1.0, |
| "single_source_diagonal_rgb_weight": 0.0, |
| "render_cloud": "all_context_views_merged_for_every_camera", |
| "random_background_during_training": false, |
| "source_views_kept_per_training_render": 8, |
| "source_cloud_dropout_probability": 0.0, |
| "training_geometry_oracle": false |
| }, |
| "geometry_teacher": { |
| "source": "frozen_stock_pi3_blocks18_to35_and_point_head", |
| "direct_depth_weight": 0.01, |
| "rendered_context_depth_weight": 0.0, |
| "anchor_end_step_exclusive": 0, |
| "direct_depth_mode": "scale_invariant_pixel_log_l1" |
| }, |
| "cross_view_depth_consistency": { |
| "weight": 0.0, |
| "source_pairing": "directed_nearest_camera", |
| "pixel_stride": 4, |
| "robust_retained_quantile": 0.7 |
| }, |
| "wan_alignment": { |
| "status": "active", |
| "teacher": "frozen_Wan2.1_VAE_16d_deterministic_mean", |
| "frame_contract": "each_view_independent_T1_first_frame", |
| "teacher_grid": "H_over_8_W_over_8", |
| "teacher_to_latent_grid": "fixed_parameter_free_adaptive_area_aggregation_to_H_over_14", |
| "latent_projection": "final_layernorm_RGB32_1x1_conv_32_to_16", |
| "loss": "VA-VAE_direct_cosine_plus_token_relation_matrix", |
| "weight": 0.1, |
| "adaptive_gradient_balance": true, |
| "cosine_margin": 0.0, |
| "distance_margin": 0.0, |
| "relation_token_samples": 256, |
| "teacher_encode_batch_size": 1 |
| }, |
| "rgb_decoder": { |
| "dpt_input": "geometry_z18_and_z36_only_no_rgb32_concat", |
| "dpt_prefinal_scale": "8x_patch_grid_then_bilinear_to_image", |
| "detail_path": "rgb32_hidden0_pixelshuffle_exact_patch14_to_full_resolution_feature32_plus_3x3_refine", |
| "fusion": "feature" |
| }, |
| "decoder_denorm": { |
| "format": "pi3-v19-ln-to-raw-channel-affine-v1", |
| "statistics": "embedded_in_model.safetensors", |
| "position": "after_fusion_layernorm_before_pi3_block18", |
| "fixed_during_rae_training": true, |
| "scale_mean": 0.09068994224071503, |
| "scale_std": 0.0018938040593639016, |
| "bias_mean": 0.025266744196414948, |
| "bias_std": 0.001551253255456686 |
| }, |
| "appearance_posterior": "identity" |
| }, |
| "args": { |
| "appearance_dim": 32, |
| "appearance_hidden_dim": 384, |
| "appearance_depth": 6, |
| "appearance_heads": 8, |
| "appearance_transformer_dropout": 0.0, |
| "appearance_drop": 0.0, |
| "features": 256, |
| "level_drop": 0.0, |
| "sh_degree": 1, |
| "gs_ray_mode": "z_depth", |
| "rgb_detail_hidden_dim": 0, |
| "rgb_detail_merge": "feature", |
| "rgb_coarse_scale": 1.0, |
| "rgb_detail_scale": 1.0 |
| }, |
| "tensor_source": { |
| "format": "safetensors", |
| "file": "model.safetensors", |
| "prefix": "rae." |
| }, |
| "decoder_ema_merged": true |
| } |
|
|
