File size: 7,387 Bytes
27431ea | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 | {
"format": "oneworld-rae-inference-v2",
"step": 30000,
"contract": {
"version": 19,
"parent_checkpoint": "",
"resolution_hw": null,
"resolutions_hw": [
[
224,
448
],
[
448,
448
],
[
252,
448
],
[
336,
448
]
],
"patch_grids_hw": [
[
16,
32
],
[
32,
32
],
[
18,
32
],
[
24,
32
]
],
"context_views": 8,
"heldout_target_views": 8,
"boundary_name": "independent_layernorm_stock_pi3_z18_and_rgb32",
"boundary_block_zero_based": 17,
"boundary_attention": "global",
"pre_boundary": "frozen_stock_pi3_blocks_0_to_17_no_rgb_injection",
"continuation_blocks": [
18,
35
],
"register_tokens": "not_generated_shared_five_learned_parameters",
"pose": "all_16_opencv_c2w_ref0_pi3_native_mean_baseline_scale_aug_1",
"view_condition": "fixed_context_view0_identity_pose_plus_intrinsic_rays;remaining_context_views_permutation_equivariant",
"decoders": "independent_full_resolution_rgb_dpt_and_gaussian_dpt;pi3_point_head_depth_on_intrinsic_pixel_rays",
"level_fusion": "independent_four_scale_softmax_gates",
"level_drop_probability": 0.0,
"level_drop_start_step": 30000,
"decoder_noise": {
"distribution": "post_fusion_layernorm_per_scene_uniform_sigma_iid_gaussian",
"tau": 0.0,
"probability": 1.0,
"start_step": 0,
"shared_strength_across_views": true
},
"nvs": "eight_endpoint_bracketing_context_to_eight_heldout_interior_targets",
"view_split": "eight_evenly_spaced_context_including_both_endpoints_plus_eight_interior_targets",
"losses": "balanced_rgb_nvs_mse_plus_lpips_plus_stock_pi3_depth_and_rendered_depth_no_gan",
"loss_weights": {
"rgb": 1.0,
"nvs": 1.0,
"merged_context_rgb": 1.0,
"lpips": 0.05,
"lpips_start": 0,
"teacher_depth": 0.01,
"render_depth": 0.0
},
"gan": {
"discriminator": "dino_s8_ks9_bn_spectral_norm",
"disc_loss": "hinge",
"generator_loss": "negative_mean_fake_logits",
"weight": 0.0,
"adaptive_max": 10000.0,
"disc_start_step": 3750,
"generator_start_step": 5000
},
"latent_channels": {
"stock_pi3_z18": 1024,
"svg_rgb": 32
},
"latent_concat_dim": 1056,
"dit_latent": "concat(layernorm_stock_pi3_z18,layernorm_rgb32);reversible_per_resolution_channel_standardization_deferred_to_dit",
"decoder_adapter": "layernorm_z18_plus_zero_init_mlp_layernorm_rgb32_to_1024_then_layernorm_then_fixed_channel_denorm",
"decoder_taps": [
17,
35
],
"rgb_alignment": "independent_per_token_layernorm_no_affine",
"post_concat_norm": null,
"rgb_direct_skip": "scene_shared_drop_p0_layernorm_rgb32_exact_patch14_full_resolution_detail_merger_before_rgb_output_conv_not_rgb_dpt_concat",
"rgb_trunk_condition": "zero_init_32_to_256_to_1024_fixed_gate_0.1",
"rgb_branch_mask": "before_alignment_shared_per_multiview_scene",
"rgb_branch_drop": 0.0,
"rgb_branch_drop_scope": "scene_shared_synchronized_full_resolution_rgb_and_gaussian_input_mergers;block19_condition_keeps_clean_rgb",
"rgb_transformer_dropout": 0.0,
"training_scope": "full",
"overfit_scene_pool_size": 0,
"dataset": "mixed_re10k_nvs_refined",
"data_mix": "exact_global_sample_parity_50pct_nvs_refined_static_50pct_legacy_re10k",
"nvs_refined_subsets": [
"ACID",
"DL3DV",
"Re10K"
],
"source_view_sampling": {
"legacy_re10k": "ordered_16_views_existing_loader",
"nvs_refined_acid": "uniform_16_view_window_span_48_160_endpoint_bracketed_8_to_8",
"nvs_refined_re10k": "uniform_16_view_window_span_48_160_endpoint_bracketed_8_to_8",
"nvs_refined_dl3dv": "depthsplat_adapted_span_warmup_15_30_to_20_50_over_10k;8_context=temporal_endpoints_plus_camera_center_fps;8_distinct_targets_inside_context_window"
},
"resolution_schedule": "global_step_synchronized_shuffled_uniform_cycle",
"normalization": {
"z18": "fixed_layernorm_per_token_no_affine",
"rgb32": "fixed_layernorm_per_token_no_affine",
"fused_block18_input": "fixed_layernorm_per_token_no_affine",
"z36_decoder_tap": "trainable_layernorm_per_token"
},
"gaussians": {
"centres": "pi3_depth_on_intrinsic_pixel_rays_no_xy_offset",
"ray_mode": "z_depth",
"sh_degree": 1,
"attribute_channels": 20,
"rgb_skip": true,
"merged_context_rgb_weight": 1.0,
"single_source_diagonal_rgb_weight": 0.0,
"render_cloud": "all_context_views_merged_for_every_camera",
"random_background_during_training": false,
"source_views_kept_per_training_render": 8,
"source_cloud_dropout_probability": 0.0,
"training_geometry_oracle": false
},
"geometry_teacher": {
"source": "frozen_stock_pi3_blocks18_to35_and_point_head",
"direct_depth_weight": 0.01,
"rendered_context_depth_weight": 0.0,
"anchor_end_step_exclusive": 0,
"direct_depth_mode": "scale_invariant_pixel_log_l1"
},
"cross_view_depth_consistency": {
"weight": 0.0,
"source_pairing": "directed_nearest_camera",
"pixel_stride": 4,
"robust_retained_quantile": 0.7
},
"wan_alignment": {
"status": "active",
"teacher": "frozen_Wan2.1_VAE_16d_deterministic_mean",
"frame_contract": "each_view_independent_T1_first_frame",
"teacher_grid": "H_over_8_W_over_8",
"teacher_to_latent_grid": "fixed_parameter_free_adaptive_area_aggregation_to_H_over_14",
"latent_projection": "final_layernorm_RGB32_1x1_conv_32_to_16",
"loss": "VA-VAE_direct_cosine_plus_token_relation_matrix",
"weight": 0.1,
"adaptive_gradient_balance": true,
"cosine_margin": 0.0,
"distance_margin": 0.0,
"relation_token_samples": 256,
"teacher_encode_batch_size": 1
},
"rgb_decoder": {
"dpt_input": "geometry_z18_and_z36_only_no_rgb32_concat",
"dpt_prefinal_scale": "8x_patch_grid_then_bilinear_to_image",
"detail_path": "rgb32_hidden0_pixelshuffle_exact_patch14_to_full_resolution_feature32_plus_3x3_refine",
"fusion": "feature"
},
"decoder_denorm": {
"format": "pi3-v19-ln-to-raw-channel-affine-v1",
"statistics": "embedded_in_model.safetensors",
"position": "after_fusion_layernorm_before_pi3_block18",
"fixed_during_rae_training": true,
"scale_mean": 0.09068994224071503,
"scale_std": 0.0018938040593639016,
"bias_mean": 0.025266744196414948,
"bias_std": 0.001551253255456686
},
"appearance_posterior": "identity"
},
"args": {
"appearance_dim": 32,
"appearance_hidden_dim": 384,
"appearance_depth": 6,
"appearance_heads": 8,
"appearance_transformer_dropout": 0.0,
"appearance_drop": 0.0,
"features": 256,
"level_drop": 0.0,
"sh_degree": 1,
"gs_ray_mode": "z_depth",
"rgb_detail_hidden_dim": 0,
"rgb_detail_merge": "feature",
"rgb_coarse_scale": 1.0,
"rgb_detail_scale": 1.0
},
"tensor_source": {
"format": "safetensors",
"file": "model.safetensors",
"prefix": "rae."
},
"decoder_ema_merged": true
}
|