Adding NeRAF weights

RAF/EmptyRoom_NeRAF/NeRAF/2024-10-10_041356/config.yml

@@ -0,0 +1,295 @@
+!!python/object:nerfstudio.engine.trainer.TrainerConfig
+_target: !!python/name:nerfstudio.engine.trainer.Trainer ''
+data: &id004 !!python/object/apply:pathlib.PosixPath
+- ../
+- data
+- RAF
+- EmptyRoom
+experiment_name: EmptyRoom_NeRAF
+gradient_accumulation_steps: {}
+load_checkpoint: null
+load_config: null
+load_dir: null
+load_scheduler: true
+load_step: null
+log_gradients: false
+logging: !!python/object:nerfstudio.configs.base_config.LoggingConfig
+  local_writer: !!python/object:nerfstudio.configs.base_config.LocalWriterConfig
+    _target: !!python/name:nerfstudio.utils.writer.LocalWriter ''
+    enable: true
+    max_log_size: 10
+    stats_to_track: !!python/tuple
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Train Iter (time)
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Train Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Test PSNR
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Vis Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Test Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - ETA (time)
+  max_buffer_size: 20
+  profiler: basic
+  relative_log_dir: !!python/object/apply:pathlib.PosixPath []
+  steps_per_log: 10
+machine: !!python/object:nerfstudio.configs.base_config.MachineConfig
+  device_type: cuda
+  dist_url: auto
+  machine_rank: 0
+  num_devices: 1
+  num_machines: 1
+  seed: 42
+max_num_iterations: 300001
+method_name: NeRAF
+mixed_precision: true
+optimizers:
+  audio_fields:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: &id001 !!python/name:torch.optim.adam.Adam ''
+      eps: 1.0e-15
+      lr: 0.0001
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: &id002 !!python/name:nerfstudio.engine.schedulers.ExponentialDecayScheduler ''
+      lr_final: 1.0e-08
+      lr_pre_warmup: 1.0e-08
+      max_steps: 1002000
+      ramp: cosine
+      warmup_steps: 2000
+  camera_opt:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.001
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 5000
+      ramp: cosine
+      warmup_steps: 0
+  fields:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.01
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 200000
+      ramp: cosine
+      warmup_steps: 0
+  proposal_networks:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.01
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 200000
+      ramp: cosine
+      warmup_steps: 0
+output_dir: !!python/object/apply:pathlib.PosixPath
+- ./
+- outputs
+pipeline: !!python/object:NeRAF.NeRAF_pipeline.NeRAFPipelineConfig
+  _target: !!python/name:NeRAF.NeRAF_pipeline.NeRAFPipeline ''
+  audio_datamanager: !!python/object:NeRAF.NeRAF_datamanager.RAFDataManagerConfig
+    _target: !!python/name:NeRAF.NeRAF_datamanager.RAFDataManager ''
+    collate_fn: &id003 !!python/name:nerfstudio.data.utils.nerfstudio_collate.nerfstudio_collate ''
+    data: null
+    dataparser: !!python/object:NeRAF.NeRAF_dataparser.RAFDataParserConfig
+      _target: !!python/name:NeRAF.NeRAF_dataparser.RAFDataParser ''
+      data: !!python/object/apply:pathlib.PosixPath []
+    eval_num_rays_per_batch: 2048
+    fs: 48000
+    hop_len: 256
+    images_on_gpu: false
+    masks_on_gpu: false
+    max_len: 0.32
+    test_mode: test
+    train_num_rays_per_batch: 2048
+  audio_model: !!python/object:NeRAF.NeRAF_model.NeRAFAudioModelConfig
+    N_features: 1024
+    N_freq_stft: 257
+    W_field: 512
+    _target: !!python/name:NeRAF.NeRAF_model.NeRAFAudioModel ''
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    criterion: SC+SLMSE
+    dataset: RAF
+    enable_collider: true
+    eval_num_rays_per_chunk: 4096
+    fs: 48000
+    grid_step: 0.0078125
+    hop_len: 128
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    loss_factor: 0.001
+    max_len: 0.32
+    prompt: null
+    use_grid: true
+    use_multiple_viewing_directions: true
+    win_len: 512
+  datamanager: !!python/object:nerfstudio.data.datamanagers.parallel_datamanager.ParallelDataManagerConfig
+    _target: !!python/name:nerfstudio.data.datamanagers.parallel_datamanager.ParallelDataManager ''
+    camera_optimizer: null
+    camera_res_scale_factor: 1.0
+    collate_fn: *id003
+    data: *id004
+    dataparser: !!python/object:nerfstudio.data.dataparsers.nerfstudio_dataparser.NerfstudioDataParserConfig
+      _target: !!python/name:nerfstudio.data.dataparsers.nerfstudio_dataparser.Nerfstudio ''
+      auto_scale_poses: true
+      center_method: poses
+      data: !!python/object/apply:pathlib.PosixPath []
+      depth_unit_scale_factor: 0.001
+      downscale_factor: null
+      eval_interval: 8
+      eval_mode: fraction
+      load_3D_points: false
+      mask_color: null
+      orientation_method: up
+      scale_factor: 1.0
+      scene_scale: 1.0
+      train_split_fraction: 0.9
+    eval_image_indices: !!python/tuple
+    - 0
+    eval_num_images_to_sample_from: -1
+    eval_num_rays_per_batch: 4096
+    eval_num_times_to_repeat_images: -1
+    images_on_gpu: true
+    masks_on_gpu: true
+    max_thread_workers: null
+    num_processes: 1
+    patch_size: 1
+    pixel_sampler: !!python/object:nerfstudio.data.pixel_samplers.PixelSamplerConfig
+      _target: !!python/name:nerfstudio.data.pixel_samplers.PixelSampler ''
+      fisheye_crop_radius: null
+      ignore_mask: false
+      is_equirectangular: false
+      keep_full_image: false
+      max_num_iterations: 100
+      num_rays_per_batch: 4096
+      rejection_sample_mask: true
+    queue_size: 2
+    train_num_images_to_sample_from: -1
+    train_num_rays_per_batch: 4096
+    train_num_times_to_repeat_images: -1
+  model: !!python/object:nerfstudio.models.base_model.ModelConfig
+    _target: !!python/name:nerfstudio.models.base_model.Model ''
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    enable_collider: true
+    eval_num_rays_per_chunk: 4096
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    prompt: null
+  save_eval_audio_path: null
+  start_step_audio: 2000
+  vision_model: !!python/object:NeRAF.NeRAF_model.NeRAFVisionModelConfig
+    _target: !!python/name:NeRAF.NeRAF_model.NeRAFVisionModel ''
+    appearance_embed_dim: 32
+    average_init_density: 0.01
+    background_color: last_sample
+    base_res: 16
+    camera_optimizer: !!python/object:nerfstudio.cameras.camera_optimizers.CameraOptimizerConfig
+      _target: !!python/name:nerfstudio.cameras.camera_optimizers.CameraOptimizer ''
+      mode: SO3xR3
+      optimizer: null
+      rot_l2_penalty: 0.001
+      scheduler: null
+      trans_l2_penalty: 0.01
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    disable_scene_contraction: false
+    distortion_loss_mult: 0.002
+    enable_collider: true
+    eval_num_rays_per_chunk: 32768
+    far_plane: 1000.0
+    features_per_level: 2
+    hidden_dim: 64
+    hidden_dim_color: 64
+    hidden_dim_transient: 64
+    implementation: tcnn
+    interlevel_loss_mult: 1.0
+    log2_hashmap_size: 19
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    max_res: 2048
+    near_plane: 0.05
+    num_levels: 16
+    num_nerf_samples_per_ray: 48
+    num_proposal_iterations: 2
+    num_proposal_samples_per_ray: !!python/tuple
+    - 256
+    - 96
+    orientation_loss_mult: 0.0001
+    pred_normal_loss_mult: 0.001
+    predict_normals: false
+    prompt: null
+    proposal_initial_sampler: piecewise
+    proposal_net_args_list:
+    - hidden_dim: 16
+      log2_hashmap_size: 17
+      max_res: 128
+      num_levels: 5
+      use_linear: false
+    - hidden_dim: 16
+      log2_hashmap_size: 17
+      max_res: 256
+      num_levels: 5
+      use_linear: false
+    proposal_update_every: 5
+    proposal_warmup: 5000
+    proposal_weights_anneal_max_num_iters: 1000
+    proposal_weights_anneal_slope: 10.0
+    use_appearance_embedding: true
+    use_average_appearance_embedding: true
+    use_gradient_scaling: false
+    use_proposal_weight_anneal: true
+    use_same_proposal_network: false
+    use_single_jitter: true
+project_name: nerfstudio-project
+prompt: null
+relative_model_dir: !!python/object/apply:pathlib.PosixPath
+- nerfstudio_models
+save_only_latest_checkpoint: false
+steps_per_eval_all_images: 10000
+steps_per_eval_batch: 10000
+steps_per_eval_image: 10000
+steps_per_save: 10000
+timestamp: 2024-10-10_041356
+use_grad_scaler: false
+viewer: !!python/object:nerfstudio.configs.base_config.ViewerConfig
+  camera_frustum_scale: 0.1
+  default_composite_depth: true
+  image_format: jpeg
+  jpeg_quality: 75
+  make_share_url: false
+  max_num_display_images: 512
+  num_rays_per_chunk: 32768
+  quit_on_train_completion: false
+  relative_log_filename: viewer_log_filename.txt
+  websocket_host: 0.0.0.0
+  websocket_port: null
+  websocket_port_default: 7007
+vis: tensorboard

RAF/EmptyRoom_NeRAF/NeRAF/2024-10-10_041356/dataparser_transforms.json

@@ -0,0 +1,23 @@
+{
+    "transform": [
+        [
+            0.9952307343482971,
+            -0.002114448929205537,
+            0.09752625972032547,
+            10.1349458694458
+        ],
+        [
+            -0.002114448929205537,
+            0.9990625381469727,
+            0.04323790222406387,
+            3.0788350105285645
+        ],
+        [
+            -0.09752625972032547,
+            -0.04323790222406387,
+            0.9942932724952698,
+            -0.6873995065689087
+        ]
+    ],
+    "scale": 0.05830662043845625
+}

RAF/EmptyRoom_NeRAF/NeRAF/2024-10-10_041356/nerfstudio_models/step-000090000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:990aedc1d456a4f3e6ffecaafe47c5cfd85a2bbd1d9f259ef84fc126c1cb9fb5
+size 783247931

RAF/FurnishedRoom_NeRAF/NeRAF/2024-10-08_091734/config.yml

@@ -0,0 +1,295 @@
+!!python/object:nerfstudio.engine.trainer.TrainerConfig
+_target: !!python/name:nerfstudio.engine.trainer.Trainer ''
+data: &id004 !!python/object/apply:pathlib.PosixPath
+- ../
+- data
+- RAF
+- FurnishedRoom
+experiment_name: FurnishedRoom_NeRAF
+gradient_accumulation_steps: {}
+load_checkpoint: null
+load_config: null
+load_dir: null
+load_scheduler: true
+load_step: null
+log_gradients: false
+logging: !!python/object:nerfstudio.configs.base_config.LoggingConfig
+  local_writer: !!python/object:nerfstudio.configs.base_config.LocalWriterConfig
+    _target: !!python/name:nerfstudio.utils.writer.LocalWriter ''
+    enable: true
+    max_log_size: 10
+    stats_to_track: !!python/tuple
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Train Iter (time)
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Train Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Test PSNR
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Vis Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Test Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - ETA (time)
+  max_buffer_size: 20
+  profiler: basic
+  relative_log_dir: !!python/object/apply:pathlib.PosixPath []
+  steps_per_log: 10
+machine: !!python/object:nerfstudio.configs.base_config.MachineConfig
+  device_type: cuda
+  dist_url: auto
+  machine_rank: 0
+  num_devices: 1
+  num_machines: 1
+  seed: 42
+max_num_iterations: 300001
+method_name: NeRAF
+mixed_precision: true
+optimizers:
+  audio_fields:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: &id001 !!python/name:torch.optim.adam.Adam ''
+      eps: 1.0e-15
+      lr: 0.0001
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: &id002 !!python/name:nerfstudio.engine.schedulers.ExponentialDecayScheduler ''
+      lr_final: 1.0e-08
+      lr_pre_warmup: 1.0e-08
+      max_steps: 1002000
+      ramp: cosine
+      warmup_steps: 2000
+  camera_opt:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.001
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 5000
+      ramp: cosine
+      warmup_steps: 0
+  fields:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.01
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 200000
+      ramp: cosine
+      warmup_steps: 0
+  proposal_networks:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.01
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 200000
+      ramp: cosine
+      warmup_steps: 0
+output_dir: !!python/object/apply:pathlib.PosixPath
+- ./
+- outputs
+pipeline: !!python/object:NeRAF.NeRAF_pipeline.NeRAFPipelineConfig
+  _target: !!python/name:NeRAF.NeRAF_pipeline.NeRAFPipeline ''
+  audio_datamanager: !!python/object:NeRAF.NeRAF_datamanager.RAFDataManagerConfig
+    _target: !!python/name:NeRAF.NeRAF_datamanager.RAFDataManager ''
+    collate_fn: &id003 !!python/name:nerfstudio.data.utils.nerfstudio_collate.nerfstudio_collate ''
+    data: null
+    dataparser: !!python/object:NeRAF.NeRAF_dataparser.RAFDataParserConfig
+      _target: !!python/name:NeRAF.NeRAF_dataparser.RAFDataParser ''
+      data: !!python/object/apply:pathlib.PosixPath []
+    eval_num_rays_per_batch: 2048
+    fs: 48000
+    hop_len: 256
+    images_on_gpu: false
+    masks_on_gpu: false
+    max_len: 0.32
+    test_mode: test
+    train_num_rays_per_batch: 2048
+  audio_model: !!python/object:NeRAF.NeRAF_model.NeRAFAudioModelConfig
+    N_features: 1024
+    N_freq_stft: 257
+    W_field: 512
+    _target: !!python/name:NeRAF.NeRAF_model.NeRAFAudioModel ''
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    criterion: SC+SLMSE
+    dataset: RAF
+    enable_collider: true
+    eval_num_rays_per_chunk: 4096
+    fs: 48000
+    grid_step: 0.0078125
+    hop_len: 128
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    loss_factor: 0.001
+    max_len: 0.32
+    prompt: null
+    use_grid: true
+    use_multiple_viewing_directions: true
+    win_len: 512
+  datamanager: !!python/object:nerfstudio.data.datamanagers.parallel_datamanager.ParallelDataManagerConfig
+    _target: !!python/name:nerfstudio.data.datamanagers.parallel_datamanager.ParallelDataManager ''
+    camera_optimizer: null
+    camera_res_scale_factor: 1.0
+    collate_fn: *id003
+    data: *id004
+    dataparser: !!python/object:nerfstudio.data.dataparsers.nerfstudio_dataparser.NerfstudioDataParserConfig
+      _target: !!python/name:nerfstudio.data.dataparsers.nerfstudio_dataparser.Nerfstudio ''
+      auto_scale_poses: true
+      center_method: poses
+      data: !!python/object/apply:pathlib.PosixPath []
+      depth_unit_scale_factor: 0.001
+      downscale_factor: null
+      eval_interval: 8
+      eval_mode: fraction
+      load_3D_points: false
+      mask_color: null
+      orientation_method: up
+      scale_factor: 1.0
+      scene_scale: 1.0
+      train_split_fraction: 0.9
+    eval_image_indices: !!python/tuple
+    - 0
+    eval_num_images_to_sample_from: -1
+    eval_num_rays_per_batch: 4096
+    eval_num_times_to_repeat_images: -1
+    images_on_gpu: true
+    masks_on_gpu: true
+    max_thread_workers: null
+    num_processes: 1
+    patch_size: 1
+    pixel_sampler: !!python/object:nerfstudio.data.pixel_samplers.PixelSamplerConfig
+      _target: !!python/name:nerfstudio.data.pixel_samplers.PixelSampler ''
+      fisheye_crop_radius: null
+      ignore_mask: false
+      is_equirectangular: false
+      keep_full_image: false
+      max_num_iterations: 100
+      num_rays_per_batch: 4096
+      rejection_sample_mask: true
+    queue_size: 2
+    train_num_images_to_sample_from: -1
+    train_num_rays_per_batch: 4096
+    train_num_times_to_repeat_images: -1
+  model: !!python/object:nerfstudio.models.base_model.ModelConfig
+    _target: !!python/name:nerfstudio.models.base_model.Model ''
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    enable_collider: true
+    eval_num_rays_per_chunk: 4096
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    prompt: null
+  save_eval_audio_path: null
+  start_step_audio: 2000
+  vision_model: !!python/object:NeRAF.NeRAF_model.NeRAFVisionModelConfig
+    _target: !!python/name:NeRAF.NeRAF_model.NeRAFVisionModel ''
+    appearance_embed_dim: 32
+    average_init_density: 0.01
+    background_color: last_sample
+    base_res: 16
+    camera_optimizer: !!python/object:nerfstudio.cameras.camera_optimizers.CameraOptimizerConfig
+      _target: !!python/name:nerfstudio.cameras.camera_optimizers.CameraOptimizer ''
+      mode: SO3xR3
+      optimizer: null
+      rot_l2_penalty: 0.001
+      scheduler: null
+      trans_l2_penalty: 0.01
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    disable_scene_contraction: false
+    distortion_loss_mult: 0.002
+    enable_collider: true
+    eval_num_rays_per_chunk: 32768
+    far_plane: 1000.0
+    features_per_level: 2
+    hidden_dim: 64
+    hidden_dim_color: 64
+    hidden_dim_transient: 64
+    implementation: tcnn
+    interlevel_loss_mult: 1.0
+    log2_hashmap_size: 19
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    max_res: 2048
+    near_plane: 0.05
+    num_levels: 16
+    num_nerf_samples_per_ray: 48
+    num_proposal_iterations: 2
+    num_proposal_samples_per_ray: !!python/tuple
+    - 256
+    - 96
+    orientation_loss_mult: 0.0001
+    pred_normal_loss_mult: 0.001
+    predict_normals: false
+    prompt: null
+    proposal_initial_sampler: piecewise
+    proposal_net_args_list:
+    - hidden_dim: 16
+      log2_hashmap_size: 17
+      max_res: 128
+      num_levels: 5
+      use_linear: false
+    - hidden_dim: 16
+      log2_hashmap_size: 17
+      max_res: 256
+      num_levels: 5
+      use_linear: false
+    proposal_update_every: 5
+    proposal_warmup: 5000
+    proposal_weights_anneal_max_num_iters: 1000
+    proposal_weights_anneal_slope: 10.0
+    use_appearance_embedding: true
+    use_average_appearance_embedding: true
+    use_gradient_scaling: false
+    use_proposal_weight_anneal: true
+    use_same_proposal_network: false
+    use_single_jitter: true
+project_name: nerfstudio-project
+prompt: null
+relative_model_dir: !!python/object/apply:pathlib.PosixPath
+- nerfstudio_models
+save_only_latest_checkpoint: false
+steps_per_eval_all_images: 10000
+steps_per_eval_batch: 10000
+steps_per_eval_image: 10000
+steps_per_save: 10000
+timestamp: 2024-10-08_091734
+use_grad_scaler: false
+viewer: !!python/object:nerfstudio.configs.base_config.ViewerConfig
+  camera_frustum_scale: 0.1
+  default_composite_depth: true
+  image_format: jpeg
+  jpeg_quality: 75
+  make_share_url: false
+  max_num_display_images: 512
+  num_rays_per_chunk: 32768
+  quit_on_train_completion: false
+  relative_log_filename: viewer_log_filename.txt
+  websocket_host: 0.0.0.0
+  websocket_port: null
+  websocket_port_default: 7007
+vis: tensorboard

RAF/FurnishedRoom_NeRAF/NeRAF/2024-10-08_091734/dataparser_transforms.json

@@ -0,0 +1,23 @@
+{
+    "transform": [
+        [
+            0.9999954104423523,
+            -2.209847116318997e-05,
+            0.0030350869055837393,
+            0.9352688193321228
+        ],
+        [
+            -2.209847116318997e-05,
+            0.9998939633369446,
+            0.01456119678914547,
+            4.980982780456543
+        ],
+        [
+            -0.0030350869055837393,
+            -0.01456119678914547,
+            0.9998893737792969,
+            0.0385591983795166
+        ]
+    ],
+    "scale": 0.1115622692578588
+}

RAF/FurnishedRoom_NeRAF/NeRAF/2024-10-08_091734/nerfstudio_models/step-000070000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93c9a728c4051e826799d2ee156f9ec53d69f4288427c31882b4b61a602be227
+size 782844283

README.md

@@ -1,3 +1,40 @@
 ---
 license: cc-by-nc-sa-4.0
+tags:
+- audio-visual-learning
+- neural-radiance-fields
+- neural-acoustic-fields
+- spatial-audio
+- nerfstudio
+- pytorch
+datasets:
+- RAF
+- SoundSpaces
 ---
+
+Official pretrained model weights for NeRAF from the paper: **"NeRAF: 3D Scene Infused Neural Radiance and Acoustic Fields"** (ICLR 2025).
+
+This repository contains pretrained checkpoints for RAF dataset and 6 SoundSpaces scenes.
+
+For training, evaluation, and usage instructions please see the official codebase:
+👉 https://github.com/AmandineBtto/NeRAF
+
+
+You can download all weights using:
+
+```bash
+huggingface-cli download AmandineBtto/NeRAF --local-dir weights/NeRAF
+```
+
+If you use these weights, please cite:
+
+```
+@inproceedings{
+    brunetto2025neraf,
+    title={Ne{RAF}: 3D Scene Infused Neural Radiance and Acoustic Fields},
+    author={Amandine Brunetto and Sascha Hornauer and Fabien Moutarde},
+    booktitle={The Thirteenth International Conference on Learning Representations},
+    year={2025},
+    url={https://openreview.net/forum?id=njvSBvtiwp}
+}
+```

SoundSpaces/apartment_1_NeRAF/NeRAF/2024-10-18_093105/config.yml

@@ -0,0 +1,294 @@
+!!python/object:nerfstudio.engine.trainer.TrainerConfig
+_target: !!python/name:nerfstudio.engine.trainer.Trainer ''
+data: &id004 !!python/object/apply:pathlib.PosixPath
+- ../
+- data
+- SoundSpaces
+- apartment_1
+experiment_name: apartment_1_NeRAF
+gradient_accumulation_steps: {}
+load_checkpoint: null
+load_config: null
+load_dir: null
+load_scheduler: true
+load_step: null
+log_gradients: false
+logging: !!python/object:nerfstudio.configs.base_config.LoggingConfig
+  local_writer: !!python/object:nerfstudio.configs.base_config.LocalWriterConfig
+    _target: !!python/name:nerfstudio.utils.writer.LocalWriter ''
+    enable: true
+    max_log_size: 10
+    stats_to_track: !!python/tuple
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Train Iter (time)
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Train Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Test PSNR
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Vis Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Test Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - ETA (time)
+  max_buffer_size: 20
+  profiler: basic
+  relative_log_dir: !!python/object/apply:pathlib.PosixPath []
+  steps_per_log: 10
+machine: !!python/object:nerfstudio.configs.base_config.MachineConfig
+  device_type: cuda
+  dist_url: auto
+  machine_rank: 0
+  num_devices: 1
+  num_machines: 1
+  seed: 42
+max_num_iterations: 500001
+method_name: NeRAF
+mixed_precision: true
+optimizers:
+  audio_fields:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: &id001 !!python/name:torch.optim.adam.Adam ''
+      eps: 1.0e-15
+      lr: 0.0001
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: &id002 !!python/name:nerfstudio.engine.schedulers.ExponentialDecayScheduler ''
+      lr_final: 1.0e-08
+      lr_pre_warmup: 1.0e-08
+      max_steps: 1002000
+      ramp: cosine
+      warmup_steps: 2000
+  camera_opt:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.001
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 5000
+      ramp: cosine
+      warmup_steps: 0
+  fields:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.01
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 200000
+      ramp: cosine
+      warmup_steps: 0
+  proposal_networks:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.01
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 200000
+      ramp: cosine
+      warmup_steps: 0
+output_dir: !!python/object/apply:pathlib.PosixPath
+- ../
+- outputs
+pipeline: !!python/object:NeRAF.NeRAF_pipeline.NeRAFPipelineConfig
+  _target: !!python/name:NeRAF.NeRAF_pipeline.NeRAFPipeline ''
+  audio_datamanager: !!python/object:NeRAF.NeRAF_datamanager.SoundSpacesDataManagerConfig
+    _target: !!python/name:NeRAF.NeRAF_datamanager.SoundSpacesDataManager ''
+    collate_fn: &id003 !!python/name:nerfstudio.data.utils.nerfstudio_collate.nerfstudio_collate ''
+    data: null
+    dataparser: !!python/object:NeRAF.NeRAF_dataparser.SoundSpacesDataParserConfig
+      _target: !!python/name:NeRAF.NeRAF_dataparser.SoundSpacesDataParser ''
+      data: !!python/object/apply:pathlib.PosixPath []
+    eval_num_rays_per_batch: 2048
+    fs: 22050
+    hop_len: 128
+    images_on_gpu: false
+    masks_on_gpu: false
+    max_len: 101
+    train_num_rays_per_batch: 2048
+  audio_model: !!python/object:NeRAF.NeRAF_model.NeRAFAudioModelConfig
+    N_features: 2048
+    N_freq_stft: 257
+    W_field: 512
+    _target: !!python/name:NeRAF.NeRAF_model.NeRAFAudioModel ''
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    criterion: SC+SLMSE
+    dataset: SoundSpaces
+    enable_collider: true
+    eval_num_rays_per_chunk: 4096
+    fs: 22050
+    grid_step: 0.0078125
+    hop_len: 128
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    loss_factor: 0.001
+    max_len: 101
+    prompt: null
+    use_grid: true
+    use_multiple_viewing_directions: true
+    win_len: 512
+  datamanager: !!python/object:nerfstudio.data.datamanagers.parallel_datamanager.ParallelDataManagerConfig
+    _target: !!python/name:nerfstudio.data.datamanagers.parallel_datamanager.ParallelDataManager ''
+    camera_optimizer: null
+    camera_res_scale_factor: 1.0
+    collate_fn: *id003
+    data: *id004
+    dataparser: !!python/object:nerfstudio.data.dataparsers.nerfstudio_dataparser.NerfstudioDataParserConfig
+      _target: !!python/name:nerfstudio.data.dataparsers.nerfstudio_dataparser.Nerfstudio ''
+      auto_scale_poses: true
+      center_method: poses
+      data: !!python/object/apply:pathlib.PosixPath []
+      depth_unit_scale_factor: 0.001
+      downscale_factor: null
+      eval_interval: 8
+      eval_mode: filename
+      load_3D_points: false
+      mask_color: null
+      orientation_method: up
+      scale_factor: 1.0
+      scene_scale: 1.0
+      train_split_fraction: 0.9
+    eval_image_indices: !!python/tuple
+    - 0
+    eval_num_images_to_sample_from: -1
+    eval_num_rays_per_batch: 4096
+    eval_num_times_to_repeat_images: -1
+    images_on_gpu: true
+    masks_on_gpu: true
+    max_thread_workers: null
+    num_processes: 1
+    patch_size: 1
+    pixel_sampler: !!python/object:nerfstudio.data.pixel_samplers.PixelSamplerConfig
+      _target: !!python/name:nerfstudio.data.pixel_samplers.PixelSampler ''
+      fisheye_crop_radius: null
+      ignore_mask: false
+      is_equirectangular: false
+      keep_full_image: false
+      max_num_iterations: 100
+      num_rays_per_batch: 4096
+      rejection_sample_mask: true
+    queue_size: 2
+    train_num_images_to_sample_from: -1
+    train_num_rays_per_batch: 4096
+    train_num_times_to_repeat_images: -1
+  model: !!python/object:nerfstudio.models.base_model.ModelConfig
+    _target: !!python/name:nerfstudio.models.base_model.Model ''
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    enable_collider: true
+    eval_num_rays_per_chunk: 4096
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    prompt: null
+  save_eval_audio_path: null
+  start_step_audio: 2000
+  vision_model: !!python/object:NeRAF.NeRAF_model.NeRAFVisionModelConfig
+    _target: !!python/name:NeRAF.NeRAF_model.NeRAFVisionModel ''
+    appearance_embed_dim: 32
+    average_init_density: 0.01
+    background_color: last_sample
+    base_res: 16
+    camera_optimizer: !!python/object:nerfstudio.cameras.camera_optimizers.CameraOptimizerConfig
+      _target: !!python/name:nerfstudio.cameras.camera_optimizers.CameraOptimizer ''
+      mode: SO3xR3
+      optimizer: null
+      rot_l2_penalty: 0.001
+      scheduler: null
+      trans_l2_penalty: 0.01
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    disable_scene_contraction: false
+    distortion_loss_mult: 0.002
+    enable_collider: true
+    eval_num_rays_per_chunk: 32768
+    far_plane: 1000.0
+    features_per_level: 2
+    hidden_dim: 64
+    hidden_dim_color: 64
+    hidden_dim_transient: 64
+    implementation: tcnn
+    interlevel_loss_mult: 1.0
+    log2_hashmap_size: 19
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    max_res: 2048
+    near_plane: 0.05
+    num_levels: 16
+    num_nerf_samples_per_ray: 48
+    num_proposal_iterations: 2
+    num_proposal_samples_per_ray: !!python/tuple
+    - 256
+    - 96
+    orientation_loss_mult: 0.0001
+    pred_normal_loss_mult: 0.001
+    predict_normals: false
+    prompt: null
+    proposal_initial_sampler: piecewise
+    proposal_net_args_list:
+    - hidden_dim: 16
+      log2_hashmap_size: 17
+      max_res: 128
+      num_levels: 5
+      use_linear: false
+    - hidden_dim: 16
+      log2_hashmap_size: 17
+      max_res: 256
+      num_levels: 5
+      use_linear: false
+    proposal_update_every: 5
+    proposal_warmup: 5000
+    proposal_weights_anneal_max_num_iters: 1000
+    proposal_weights_anneal_slope: 10.0
+    use_appearance_embedding: true
+    use_average_appearance_embedding: true
+    use_gradient_scaling: false
+    use_proposal_weight_anneal: true
+    use_same_proposal_network: false
+    use_single_jitter: true
+project_name: nerfstudio-project
+prompt: null
+relative_model_dir: !!python/object/apply:pathlib.PosixPath
+- nerfstudio_models
+save_only_latest_checkpoint: false
+steps_per_eval_all_images: 10000
+steps_per_eval_batch: 10000
+steps_per_eval_image: 10000
+steps_per_save: 10000
+timestamp: 2024-10-18_093105
+use_grad_scaler: false
+viewer: !!python/object:nerfstudio.configs.base_config.ViewerConfig
+  camera_frustum_scale: 0.1
+  default_composite_depth: true
+  image_format: jpeg
+  jpeg_quality: 75
+  make_share_url: false
+  max_num_display_images: 512
+  num_rays_per_chunk: 32768
+  quit_on_train_completion: false
+  relative_log_filename: viewer_log_filename.txt
+  websocket_host: 0.0.0.0
+  websocket_port: null
+  websocket_port_default: 7007
+vis: tensorboard

SoundSpaces/apartment_1_NeRAF/NeRAF/2024-10-18_093105/dataparser_transforms.json

@@ -0,0 +1,23 @@
+{
+    "transform": [
+        [
+            1.0,
+            0.0,
+            0.0,
+            3.3542232513427734
+        ],
+        [
+            0.0,
+            1.0,
+            0.0,
+            3.233105182647705
+        ],
+        [
+            0.0,
+            0.0,
+            1.0,
+            1.7497400045394897
+        ]
+    ],
+    "scale": 0.1816968860606569
+}

SoundSpaces/apartment_1_NeRAF/NeRAF/2024-10-18_093105/nerfstudio_models/step-000480000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a79809688f6e10992404e5174090d0165cc0107c369a05dd3693f7e1de9a0dd3
+size 1195042065

SoundSpaces/apartment_2_NeRAF/NeRAF/2024-10-13_131052/config.yml

@@ -0,0 +1,294 @@
+!!python/object:nerfstudio.engine.trainer.TrainerConfig
+_target: !!python/name:nerfstudio.engine.trainer.Trainer ''
+data: &id004 !!python/object/apply:pathlib.PosixPath
+- ../
+- data
+- SoundSpaces
+- apartment_2
+experiment_name: apartment_2_NeRAF
+gradient_accumulation_steps: {}
+load_checkpoint: null
+load_config: null
+load_dir: null
+load_scheduler: true
+load_step: null
+log_gradients: false
+logging: !!python/object:nerfstudio.configs.base_config.LoggingConfig
+  local_writer: !!python/object:nerfstudio.configs.base_config.LocalWriterConfig
+    _target: !!python/name:nerfstudio.utils.writer.LocalWriter ''
+    enable: true
+    max_log_size: 10
+    stats_to_track: !!python/tuple
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Train Iter (time)
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Train Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Test PSNR
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Vis Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Test Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - ETA (time)
+  max_buffer_size: 20
+  profiler: basic
+  relative_log_dir: !!python/object/apply:pathlib.PosixPath []
+  steps_per_log: 10
+machine: !!python/object:nerfstudio.configs.base_config.MachineConfig
+  device_type: cuda
+  dist_url: auto
+  machine_rank: 0
+  num_devices: 1
+  num_machines: 1
+  seed: 42
+max_num_iterations: 500001
+method_name: NeRAF
+mixed_precision: true
+optimizers:
+  audio_fields:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: &id001 !!python/name:torch.optim.adam.Adam ''
+      eps: 1.0e-15
+      lr: 0.0001
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: &id002 !!python/name:nerfstudio.engine.schedulers.ExponentialDecayScheduler ''
+      lr_final: 1.0e-08
+      lr_pre_warmup: 1.0e-08
+      max_steps: 1002000
+      ramp: cosine
+      warmup_steps: 2000
+  camera_opt:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.001
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 5000
+      ramp: cosine
+      warmup_steps: 0
+  fields:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.01
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 200000
+      ramp: cosine
+      warmup_steps: 0
+  proposal_networks:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.01
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 200000
+      ramp: cosine
+      warmup_steps: 0
+output_dir: !!python/object/apply:pathlib.PosixPath
+- ../
+- outputs
+pipeline: !!python/object:NeRAF.NeRAF_pipeline.NeRAFPipelineConfig
+  _target: !!python/name:NeRAF.NeRAF_pipeline.NeRAFPipeline ''
+  audio_datamanager: !!python/object:NeRAF.NeRAF_datamanager.SoundSpacesDataManagerConfig
+    _target: !!python/name:NeRAF.NeRAF_datamanager.SoundSpacesDataManager ''
+    collate_fn: &id003 !!python/name:nerfstudio.data.utils.nerfstudio_collate.nerfstudio_collate ''
+    data: null
+    dataparser: !!python/object:NeRAF.NeRAF_dataparser.SoundSpacesDataParserConfig
+      _target: !!python/name:NeRAF.NeRAF_dataparser.SoundSpacesDataParser ''
+      data: !!python/object/apply:pathlib.PosixPath []
+    eval_num_rays_per_batch: 2048
+    fs: 22050
+    hop_len: 128
+    images_on_gpu: false
+    masks_on_gpu: false
+    max_len: 86
+    train_num_rays_per_batch: 2048
+  audio_model: !!python/object:NeRAF.NeRAF_model.NeRAFAudioModelConfig
+    N_features: 1024
+    N_freq_stft: 257
+    W_field: 512
+    _target: !!python/name:NeRAF.NeRAF_model.NeRAFAudioModel ''
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    criterion: SC+SLMSE
+    dataset: SoundSpaces
+    enable_collider: true
+    eval_num_rays_per_chunk: 4096
+    fs: 22050
+    grid_step: 0.0078125
+    hop_len: 128
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    loss_factor: 0.001
+    max_len: 86
+    prompt: null
+    use_grid: true
+    use_multiple_viewing_directions: true
+    win_len: 512
+  datamanager: !!python/object:nerfstudio.data.datamanagers.parallel_datamanager.ParallelDataManagerConfig
+    _target: !!python/name:nerfstudio.data.datamanagers.parallel_datamanager.ParallelDataManager ''
+    camera_optimizer: null
+    camera_res_scale_factor: 1.0
+    collate_fn: *id003
+    data: *id004
+    dataparser: !!python/object:nerfstudio.data.dataparsers.nerfstudio_dataparser.NerfstudioDataParserConfig
+      _target: !!python/name:nerfstudio.data.dataparsers.nerfstudio_dataparser.Nerfstudio ''
+      auto_scale_poses: true
+      center_method: poses
+      data: !!python/object/apply:pathlib.PosixPath []
+      depth_unit_scale_factor: 0.001
+      downscale_factor: null
+      eval_interval: 8
+      eval_mode: filename
+      load_3D_points: false
+      mask_color: null
+      orientation_method: up
+      scale_factor: 1.0
+      scene_scale: 1.0
+      train_split_fraction: 0.9
+    eval_image_indices: !!python/tuple
+    - 0
+    eval_num_images_to_sample_from: -1
+    eval_num_rays_per_batch: 4096
+    eval_num_times_to_repeat_images: -1
+    images_on_gpu: true
+    masks_on_gpu: true
+    max_thread_workers: null
+    num_processes: 1
+    patch_size: 1
+    pixel_sampler: !!python/object:nerfstudio.data.pixel_samplers.PixelSamplerConfig
+      _target: !!python/name:nerfstudio.data.pixel_samplers.PixelSampler ''
+      fisheye_crop_radius: null
+      ignore_mask: false
+      is_equirectangular: false
+      keep_full_image: false
+      max_num_iterations: 100
+      num_rays_per_batch: 4096
+      rejection_sample_mask: true
+    queue_size: 2
+    train_num_images_to_sample_from: -1
+    train_num_rays_per_batch: 4096
+    train_num_times_to_repeat_images: -1
+  model: !!python/object:nerfstudio.models.base_model.ModelConfig
+    _target: !!python/name:nerfstudio.models.base_model.Model ''
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    enable_collider: true
+    eval_num_rays_per_chunk: 4096
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    prompt: null
+  save_eval_audio_path: null
+  start_step_audio: 2000
+  vision_model: !!python/object:NeRAF.NeRAF_model.NeRAFVisionModelConfig
+    _target: !!python/name:NeRAF.NeRAF_model.NeRAFVisionModel ''
+    appearance_embed_dim: 32
+    average_init_density: 0.01
+    background_color: last_sample
+    base_res: 16
+    camera_optimizer: !!python/object:nerfstudio.cameras.camera_optimizers.CameraOptimizerConfig
+      _target: !!python/name:nerfstudio.cameras.camera_optimizers.CameraOptimizer ''
+      mode: SO3xR3
+      optimizer: null
+      rot_l2_penalty: 0.001
+      scheduler: null
+      trans_l2_penalty: 0.01
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    disable_scene_contraction: false
+    distortion_loss_mult: 0.002
+    enable_collider: true
+    eval_num_rays_per_chunk: 32768
+    far_plane: 1000.0
+    features_per_level: 2
+    hidden_dim: 64
+    hidden_dim_color: 64
+    hidden_dim_transient: 64
+    implementation: tcnn
+    interlevel_loss_mult: 1.0
+    log2_hashmap_size: 19
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    max_res: 2048
+    near_plane: 0.05
+    num_levels: 16
+    num_nerf_samples_per_ray: 48
+    num_proposal_iterations: 2
+    num_proposal_samples_per_ray: !!python/tuple
+    - 256
+    - 96
+    orientation_loss_mult: 0.0001
+    pred_normal_loss_mult: 0.001
+    predict_normals: false
+    prompt: null
+    proposal_initial_sampler: piecewise
+    proposal_net_args_list:
+    - hidden_dim: 16
+      log2_hashmap_size: 17
+      max_res: 128
+      num_levels: 5
+      use_linear: false
+    - hidden_dim: 16
+      log2_hashmap_size: 17
+      max_res: 256
+      num_levels: 5
+      use_linear: false
+    proposal_update_every: 5
+    proposal_warmup: 5000
+    proposal_weights_anneal_max_num_iters: 1000
+    proposal_weights_anneal_slope: 10.0
+    use_appearance_embedding: true
+    use_average_appearance_embedding: true
+    use_gradient_scaling: false
+    use_proposal_weight_anneal: true
+    use_same_proposal_network: false
+    use_single_jitter: true
+project_name: nerfstudio-project
+prompt: null
+relative_model_dir: !!python/object/apply:pathlib.PosixPath
+- nerfstudio_models
+save_only_latest_checkpoint: false
+steps_per_eval_all_images: 10000
+steps_per_eval_batch: 10000
+steps_per_eval_image: 10000
+steps_per_save: 10000
+timestamp: 2024-10-13_131052
+use_grad_scaler: false
+viewer: !!python/object:nerfstudio.configs.base_config.ViewerConfig
+  camera_frustum_scale: 0.1
+  default_composite_depth: true
+  image_format: jpeg
+  jpeg_quality: 75
+  make_share_url: false
+  max_num_display_images: 512
+  num_rays_per_chunk: 32768
+  quit_on_train_completion: false
+  relative_log_filename: viewer_log_filename.txt
+  websocket_host: 0.0.0.0
+  websocket_port: null
+  websocket_port_default: 7007
+vis: tensorboard

SoundSpaces/apartment_2_NeRAF/NeRAF/2024-10-13_131052/dataparser_transforms.json

@@ -0,0 +1,23 @@
+{
+    "transform": [
+        [
+            1.0,
+            0.0,
+            0.0,
+            2.539200782775879
+        ],
+        [
+            0.0,
+            1.0,
+            0.0,
+            3.0175981521606445
+        ],
+        [
+            0.0,
+            0.0,
+            1.0,
+            1.6110721826553345
+        ]
+    ],
+    "scale": 0.1954609374883496
+}

SoundSpaces/apartment_2_NeRAF/NeRAF/2024-10-13_131052/nerfstudio_models/step-000150000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af1e65265dfba186c644b4054f6faf664d79badd6e4405356ed5f555362a0c99
+size 782825379

SoundSpaces/frl_apartment_2_NeRAF/NeRAF/2024-10-18_133400/config.yml

@@ -0,0 +1,294 @@
+!!python/object:nerfstudio.engine.trainer.TrainerConfig
+_target: !!python/name:nerfstudio.engine.trainer.Trainer ''
+data: &id004 !!python/object/apply:pathlib.PosixPath
+- ../
+- data
+- SoundSpaces
+- frl_apartment_2
+experiment_name: frl_apartment_2_NeRAF
+gradient_accumulation_steps: {}
+load_checkpoint: null
+load_config: null
+load_dir: null
+load_scheduler: true
+load_step: null
+log_gradients: false
+logging: !!python/object:nerfstudio.configs.base_config.LoggingConfig
+  local_writer: !!python/object:nerfstudio.configs.base_config.LocalWriterConfig
+    _target: !!python/name:nerfstudio.utils.writer.LocalWriter ''
+    enable: true
+    max_log_size: 10
+    stats_to_track: !!python/tuple
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Train Iter (time)
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Train Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Test PSNR
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Vis Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Test Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - ETA (time)
+  max_buffer_size: 20
+  profiler: basic
+  relative_log_dir: !!python/object/apply:pathlib.PosixPath []
+  steps_per_log: 10
+machine: !!python/object:nerfstudio.configs.base_config.MachineConfig
+  device_type: cuda
+  dist_url: auto
+  machine_rank: 0
+  num_devices: 1
+  num_machines: 1
+  seed: 42
+max_num_iterations: 500001
+method_name: NeRAF
+mixed_precision: true
+optimizers:
+  audio_fields:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: &id001 !!python/name:torch.optim.adam.Adam ''
+      eps: 1.0e-15
+      lr: 0.0001
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: &id002 !!python/name:nerfstudio.engine.schedulers.ExponentialDecayScheduler ''
+      lr_final: 1.0e-08
+      lr_pre_warmup: 1.0e-08
+      max_steps: 1002000
+      ramp: cosine
+      warmup_steps: 2000
+  camera_opt:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.001
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 5000
+      ramp: cosine
+      warmup_steps: 0
+  fields:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.01
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 200000
+      ramp: cosine
+      warmup_steps: 0
+  proposal_networks:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.01
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 200000
+      ramp: cosine
+      warmup_steps: 0
+output_dir: !!python/object/apply:pathlib.PosixPath
+- ../
+- outputs
+pipeline: !!python/object:NeRAF.NeRAF_pipeline.NeRAFPipelineConfig
+  _target: !!python/name:NeRAF.NeRAF_pipeline.NeRAFPipeline ''
+  audio_datamanager: !!python/object:NeRAF.NeRAF_datamanager.SoundSpacesDataManagerConfig
+    _target: !!python/name:NeRAF.NeRAF_datamanager.SoundSpacesDataManager ''
+    collate_fn: &id003 !!python/name:nerfstudio.data.utils.nerfstudio_collate.nerfstudio_collate ''
+    data: null
+    dataparser: !!python/object:NeRAF.NeRAF_dataparser.SoundSpacesDataParserConfig
+      _target: !!python/name:NeRAF.NeRAF_dataparser.SoundSpacesDataParser ''
+      data: !!python/object/apply:pathlib.PosixPath []
+    eval_num_rays_per_batch: 2048
+    fs: 22050
+    hop_len: 128
+    images_on_gpu: false
+    masks_on_gpu: false
+    max_len: 107
+    train_num_rays_per_batch: 2048
+  audio_model: !!python/object:NeRAF.NeRAF_model.NeRAFAudioModelConfig
+    N_features: 2048
+    N_freq_stft: 257
+    W_field: 512
+    _target: !!python/name:NeRAF.NeRAF_model.NeRAFAudioModel ''
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    criterion: SC+SLMSE
+    dataset: SoundSpaces
+    enable_collider: true
+    eval_num_rays_per_chunk: 4096
+    fs: 22050
+    grid_step: 0.0078125
+    hop_len: 128
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    loss_factor: 0.001
+    max_len: 107
+    prompt: null
+    use_grid: true
+    use_multiple_viewing_directions: true
+    win_len: 512
+  datamanager: !!python/object:nerfstudio.data.datamanagers.parallel_datamanager.ParallelDataManagerConfig
+    _target: !!python/name:nerfstudio.data.datamanagers.parallel_datamanager.ParallelDataManager ''
+    camera_optimizer: null
+    camera_res_scale_factor: 1.0
+    collate_fn: *id003
+    data: *id004
+    dataparser: !!python/object:nerfstudio.data.dataparsers.nerfstudio_dataparser.NerfstudioDataParserConfig
+      _target: !!python/name:nerfstudio.data.dataparsers.nerfstudio_dataparser.Nerfstudio ''
+      auto_scale_poses: true
+      center_method: poses
+      data: !!python/object/apply:pathlib.PosixPath []
+      depth_unit_scale_factor: 0.001
+      downscale_factor: null
+      eval_interval: 8
+      eval_mode: filename
+      load_3D_points: false
+      mask_color: null
+      orientation_method: up
+      scale_factor: 1.0
+      scene_scale: 1.0
+      train_split_fraction: 0.9
+    eval_image_indices: !!python/tuple
+    - 0
+    eval_num_images_to_sample_from: -1
+    eval_num_rays_per_batch: 4096
+    eval_num_times_to_repeat_images: -1
+    images_on_gpu: true
+    masks_on_gpu: true
+    max_thread_workers: null
+    num_processes: 1
+    patch_size: 1
+    pixel_sampler: !!python/object:nerfstudio.data.pixel_samplers.PixelSamplerConfig
+      _target: !!python/name:nerfstudio.data.pixel_samplers.PixelSampler ''
+      fisheye_crop_radius: null
+      ignore_mask: false
+      is_equirectangular: false
+      keep_full_image: false
+      max_num_iterations: 100
+      num_rays_per_batch: 4096
+      rejection_sample_mask: true
+    queue_size: 2
+    train_num_images_to_sample_from: -1
+    train_num_rays_per_batch: 4096
+    train_num_times_to_repeat_images: -1
+  model: !!python/object:nerfstudio.models.base_model.ModelConfig
+    _target: !!python/name:nerfstudio.models.base_model.Model ''
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    enable_collider: true
+    eval_num_rays_per_chunk: 4096
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    prompt: null
+  save_eval_audio_path: null
+  start_step_audio: 2000
+  vision_model: !!python/object:NeRAF.NeRAF_model.NeRAFVisionModelConfig
+    _target: !!python/name:NeRAF.NeRAF_model.NeRAFVisionModel ''
+    appearance_embed_dim: 32
+    average_init_density: 0.01
+    background_color: last_sample
+    base_res: 16
+    camera_optimizer: !!python/object:nerfstudio.cameras.camera_optimizers.CameraOptimizerConfig
+      _target: !!python/name:nerfstudio.cameras.camera_optimizers.CameraOptimizer ''
+      mode: SO3xR3
+      optimizer: null
+      rot_l2_penalty: 0.001
+      scheduler: null
+      trans_l2_penalty: 0.01
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    disable_scene_contraction: false
+    distortion_loss_mult: 0.002
+    enable_collider: true
+    eval_num_rays_per_chunk: 32768
+    far_plane: 1000.0
+    features_per_level: 2
+    hidden_dim: 64
+    hidden_dim_color: 64
+    hidden_dim_transient: 64
+    implementation: tcnn
+    interlevel_loss_mult: 1.0
+    log2_hashmap_size: 19
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    max_res: 2048
+    near_plane: 0.05
+    num_levels: 16
+    num_nerf_samples_per_ray: 48
+    num_proposal_iterations: 2
+    num_proposal_samples_per_ray: !!python/tuple
+    - 256
+    - 96
+    orientation_loss_mult: 0.0001
+    pred_normal_loss_mult: 0.001
+    predict_normals: false
+    prompt: null
+    proposal_initial_sampler: piecewise
+    proposal_net_args_list:
+    - hidden_dim: 16
+      log2_hashmap_size: 17
+      max_res: 128
+      num_levels: 5
+      use_linear: false
+    - hidden_dim: 16
+      log2_hashmap_size: 17
+      max_res: 256
+      num_levels: 5
+      use_linear: false
+    proposal_update_every: 5
+    proposal_warmup: 5000
+    proposal_weights_anneal_max_num_iters: 1000
+    proposal_weights_anneal_slope: 10.0
+    use_appearance_embedding: true
+    use_average_appearance_embedding: true
+    use_gradient_scaling: false
+    use_proposal_weight_anneal: true
+    use_same_proposal_network: false
+    use_single_jitter: true
+project_name: nerfstudio-project
+prompt: null
+relative_model_dir: !!python/object/apply:pathlib.PosixPath
+- nerfstudio_models
+save_only_latest_checkpoint: false
+steps_per_eval_all_images: 10000
+steps_per_eval_batch: 10000
+steps_per_eval_image: 10000
+steps_per_save: 10000
+timestamp: 2024-10-18_133400
+use_grad_scaler: false
+viewer: !!python/object:nerfstudio.configs.base_config.ViewerConfig
+  camera_frustum_scale: 0.1
+  default_composite_depth: true
+  image_format: jpeg
+  jpeg_quality: 75
+  make_share_url: false
+  max_num_display_images: 512
+  num_rays_per_chunk: 32768
+  quit_on_train_completion: false
+  relative_log_filename: viewer_log_filename.txt
+  websocket_host: 0.0.0.0
+  websocket_port: null
+  websocket_port_default: 7007
+vis: tensorboard

SoundSpaces/frl_apartment_2_NeRAF/NeRAF/2024-10-18_133400/dataparser_transforms.json

@@ -0,0 +1,23 @@
+{
+    "transform": [
+        [
+            1.0,
+            0.0,
+            0.0,
+            2.636273145675659
+        ],
+        [
+            0.0,
+            1.0,
+            0.0,
+            1.7466418743133545
+        ],
+        [
+            0.0,
+            0.0,
+            1.0,
+            1.4013906717300415
+        ]
+    ],
+    "scale": 0.18044055968484302
+}

SoundSpaces/frl_apartment_2_NeRAF/NeRAF/2024-10-18_133400/nerfstudio_models/step-000450000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6724db0d8f9e72b48c43f2fa575840eba9c9b44eb6b357df3745dedf3eea4b96
+size 1194988689

SoundSpaces/frl_apartment_4_NeRAF/NeRAF/2024-10-20_105001/config.yml

@@ -0,0 +1,294 @@
+!!python/object:nerfstudio.engine.trainer.TrainerConfig
+_target: !!python/name:nerfstudio.engine.trainer.Trainer ''
+data: &id004 !!python/object/apply:pathlib.PosixPath
+- ../
+- data
+- SoundSpaces
+- frl_apartment_4
+experiment_name: frl_apartment_4_NeRAF
+gradient_accumulation_steps: {}
+load_checkpoint: null
+load_config: null
+load_dir: null
+load_scheduler: true
+load_step: null
+log_gradients: false
+logging: !!python/object:nerfstudio.configs.base_config.LoggingConfig
+  local_writer: !!python/object:nerfstudio.configs.base_config.LocalWriterConfig
+    _target: !!python/name:nerfstudio.utils.writer.LocalWriter ''
+    enable: true
+    max_log_size: 10
+    stats_to_track: !!python/tuple
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Train Iter (time)
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Train Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Test PSNR
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Vis Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Test Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - ETA (time)
+  max_buffer_size: 20
+  profiler: basic
+  relative_log_dir: !!python/object/apply:pathlib.PosixPath []
+  steps_per_log: 10
+machine: !!python/object:nerfstudio.configs.base_config.MachineConfig
+  device_type: cuda
+  dist_url: auto
+  machine_rank: 0
+  num_devices: 1
+  num_machines: 1
+  seed: 42
+max_num_iterations: 500001
+method_name: NeRAF
+mixed_precision: true
+optimizers:
+  audio_fields:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: &id001 !!python/name:torch.optim.adam.Adam ''
+      eps: 1.0e-15
+      lr: 0.0001
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: &id002 !!python/name:nerfstudio.engine.schedulers.ExponentialDecayScheduler ''
+      lr_final: 1.0e-08
+      lr_pre_warmup: 1.0e-08
+      max_steps: 1002000
+      ramp: cosine
+      warmup_steps: 2000
+  camera_opt:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.001
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 5000
+      ramp: cosine
+      warmup_steps: 0
+  fields:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.01
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 200000
+      ramp: cosine
+      warmup_steps: 0
+  proposal_networks:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.01
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 200000
+      ramp: cosine
+      warmup_steps: 0
+output_dir: !!python/object/apply:pathlib.PosixPath
+- ../
+- outputs
+pipeline: !!python/object:NeRAF.NeRAF_pipeline.NeRAFPipelineConfig
+  _target: !!python/name:NeRAF.NeRAF_pipeline.NeRAFPipeline ''
+  audio_datamanager: !!python/object:NeRAF.NeRAF_datamanager.SoundSpacesDataManagerConfig
+    _target: !!python/name:NeRAF.NeRAF_datamanager.SoundSpacesDataManager ''
+    collate_fn: &id003 !!python/name:nerfstudio.data.utils.nerfstudio_collate.nerfstudio_collate ''
+    data: null
+    dataparser: !!python/object:NeRAF.NeRAF_dataparser.SoundSpacesDataParserConfig
+      _target: !!python/name:NeRAF.NeRAF_dataparser.SoundSpacesDataParser ''
+      data: !!python/object/apply:pathlib.PosixPath []
+    eval_num_rays_per_batch: 2048
+    fs: 22050
+    hop_len: 128
+    images_on_gpu: false
+    masks_on_gpu: false
+    max_len: 103
+    train_num_rays_per_batch: 2048
+  audio_model: !!python/object:NeRAF.NeRAF_model.NeRAFAudioModelConfig
+    N_features: 2048
+    N_freq_stft: 257
+    W_field: 512
+    _target: !!python/name:NeRAF.NeRAF_model.NeRAFAudioModel ''
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    criterion: SC+SLMSE
+    dataset: SoundSpaces
+    enable_collider: true
+    eval_num_rays_per_chunk: 4096
+    fs: 22050
+    grid_step: 0.0078125
+    hop_len: 128
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    loss_factor: 0.001
+    max_len: 103
+    prompt: null
+    use_grid: true
+    use_multiple_viewing_directions: true
+    win_len: 512
+  datamanager: !!python/object:nerfstudio.data.datamanagers.parallel_datamanager.ParallelDataManagerConfig
+    _target: !!python/name:nerfstudio.data.datamanagers.parallel_datamanager.ParallelDataManager ''
+    camera_optimizer: null
+    camera_res_scale_factor: 1.0
+    collate_fn: *id003
+    data: *id004
+    dataparser: !!python/object:nerfstudio.data.dataparsers.nerfstudio_dataparser.NerfstudioDataParserConfig
+      _target: !!python/name:nerfstudio.data.dataparsers.nerfstudio_dataparser.Nerfstudio ''
+      auto_scale_poses: true
+      center_method: poses
+      data: !!python/object/apply:pathlib.PosixPath []
+      depth_unit_scale_factor: 0.001
+      downscale_factor: null
+      eval_interval: 8
+      eval_mode: filename
+      load_3D_points: false
+      mask_color: null
+      orientation_method: up
+      scale_factor: 1.0
+      scene_scale: 1.0
+      train_split_fraction: 0.9
+    eval_image_indices: !!python/tuple
+    - 0
+    eval_num_images_to_sample_from: -1
+    eval_num_rays_per_batch: 4096
+    eval_num_times_to_repeat_images: -1
+    images_on_gpu: true
+    masks_on_gpu: true
+    max_thread_workers: null
+    num_processes: 1
+    patch_size: 1
+    pixel_sampler: !!python/object:nerfstudio.data.pixel_samplers.PixelSamplerConfig
+      _target: !!python/name:nerfstudio.data.pixel_samplers.PixelSampler ''
+      fisheye_crop_radius: null
+      ignore_mask: false
+      is_equirectangular: false
+      keep_full_image: false
+      max_num_iterations: 100
+      num_rays_per_batch: 4096
+      rejection_sample_mask: true
+    queue_size: 2
+    train_num_images_to_sample_from: -1
+    train_num_rays_per_batch: 4096
+    train_num_times_to_repeat_images: -1
+  model: !!python/object:nerfstudio.models.base_model.ModelConfig
+    _target: !!python/name:nerfstudio.models.base_model.Model ''
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    enable_collider: true
+    eval_num_rays_per_chunk: 4096
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    prompt: null
+  save_eval_audio_path: null
+  start_step_audio: 2000
+  vision_model: !!python/object:NeRAF.NeRAF_model.NeRAFVisionModelConfig
+    _target: !!python/name:NeRAF.NeRAF_model.NeRAFVisionModel ''
+    appearance_embed_dim: 32
+    average_init_density: 0.01
+    background_color: last_sample
+    base_res: 16
+    camera_optimizer: !!python/object:nerfstudio.cameras.camera_optimizers.CameraOptimizerConfig
+      _target: !!python/name:nerfstudio.cameras.camera_optimizers.CameraOptimizer ''
+      mode: SO3xR3
+      optimizer: null
+      rot_l2_penalty: 0.001
+      scheduler: null
+      trans_l2_penalty: 0.01
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    disable_scene_contraction: false
+    distortion_loss_mult: 0.002
+    enable_collider: true
+    eval_num_rays_per_chunk: 32768
+    far_plane: 1000.0
+    features_per_level: 2
+    hidden_dim: 64
+    hidden_dim_color: 64
+    hidden_dim_transient: 64
+    implementation: tcnn
+    interlevel_loss_mult: 1.0
+    log2_hashmap_size: 19
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    max_res: 2048
+    near_plane: 0.05
+    num_levels: 16
+    num_nerf_samples_per_ray: 48
+    num_proposal_iterations: 2
+    num_proposal_samples_per_ray: !!python/tuple
+    - 256
+    - 96
+    orientation_loss_mult: 0.0001
+    pred_normal_loss_mult: 0.001
+    predict_normals: false
+    prompt: null
+    proposal_initial_sampler: piecewise
+    proposal_net_args_list:
+    - hidden_dim: 16
+      log2_hashmap_size: 17
+      max_res: 128
+      num_levels: 5
+      use_linear: false
+    - hidden_dim: 16
+      log2_hashmap_size: 17
+      max_res: 256
+      num_levels: 5
+      use_linear: false
+    proposal_update_every: 5
+    proposal_warmup: 5000
+    proposal_weights_anneal_max_num_iters: 1000
+    proposal_weights_anneal_slope: 10.0
+    use_appearance_embedding: true
+    use_average_appearance_embedding: true
+    use_gradient_scaling: false
+    use_proposal_weight_anneal: true
+    use_same_proposal_network: false
+    use_single_jitter: true
+project_name: nerfstudio-project
+prompt: null
+relative_model_dir: !!python/object/apply:pathlib.PosixPath
+- nerfstudio_models
+save_only_latest_checkpoint: false
+steps_per_eval_all_images: 10000
+steps_per_eval_batch: 10000
+steps_per_eval_image: 10000
+steps_per_save: 10000
+timestamp: 2024-10-20_105001
+use_grad_scaler: false
+viewer: !!python/object:nerfstudio.configs.base_config.ViewerConfig
+  camera_frustum_scale: 0.1
+  default_composite_depth: true
+  image_format: jpeg
+  jpeg_quality: 75
+  make_share_url: false
+  max_num_display_images: 512
+  num_rays_per_chunk: 32768
+  quit_on_train_completion: false
+  relative_log_filename: viewer_log_filename.txt
+  websocket_host: 0.0.0.0
+  websocket_port: null
+  websocket_port_default: 7007
+vis: tensorboard

SoundSpaces/frl_apartment_4_NeRAF/NeRAF/2024-10-20_105001/dataparser_transforms.json

@@ -0,0 +1,23 @@
+{
+    "transform": [
+        [
+            1.0,
+            0.0,
+            0.0,
+            2.6369545459747314
+        ],
+        [
+            0.0,
+            1.0,
+            0.0,
+            -2.6249184608459473
+        ],
+        [
+            0.0,
+            0.0,
+            1.0,
+            1.451151728630066
+        ]
+    ],
+    "scale": 0.18359158603929643
+}

SoundSpaces/frl_apartment_4_NeRAF/NeRAF/2024-10-20_105001/nerfstudio_models/step-000170000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:040c2a17899a01ba911ee70eb37b644c43bcbec08a993376e3b5c92fee422223
+size 1194988689

SoundSpaces/office_4_NeRAF/NeRAF/2024-10-11_101120/config.yml

@@ -0,0 +1,294 @@
+!!python/object:nerfstudio.engine.trainer.TrainerConfig
+_target: !!python/name:nerfstudio.engine.trainer.Trainer ''
+data: &id004 !!python/object/apply:pathlib.PosixPath
+- ../
+- data
+- SoundSpaces
+- office_4
+experiment_name: office_4_NeRAF
+gradient_accumulation_steps: {}
+load_checkpoint: null
+load_config: null
+load_dir: null
+load_scheduler: true
+load_step: null
+log_gradients: false
+logging: !!python/object:nerfstudio.configs.base_config.LoggingConfig
+  local_writer: !!python/object:nerfstudio.configs.base_config.LocalWriterConfig
+    _target: !!python/name:nerfstudio.utils.writer.LocalWriter ''
+    enable: true
+    max_log_size: 10
+    stats_to_track: !!python/tuple
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Train Iter (time)
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Train Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Test PSNR
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Vis Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Test Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - ETA (time)
+  max_buffer_size: 20
+  profiler: basic
+  relative_log_dir: !!python/object/apply:pathlib.PosixPath []
+  steps_per_log: 10
+machine: !!python/object:nerfstudio.configs.base_config.MachineConfig
+  device_type: cuda
+  dist_url: auto
+  machine_rank: 0
+  num_devices: 1
+  num_machines: 1
+  seed: 42
+max_num_iterations: 500001
+method_name: NeRAF
+mixed_precision: true
+optimizers:
+  audio_fields:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: &id001 !!python/name:torch.optim.adam.Adam ''
+      eps: 1.0e-15
+      lr: 0.0001
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: &id002 !!python/name:nerfstudio.engine.schedulers.ExponentialDecayScheduler ''
+      lr_final: 1.0e-08
+      lr_pre_warmup: 1.0e-08
+      max_steps: 1002000
+      ramp: cosine
+      warmup_steps: 2000
+  camera_opt:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.001
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 5000
+      ramp: cosine
+      warmup_steps: 0
+  fields:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.01
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 200000
+      ramp: cosine
+      warmup_steps: 0
+  proposal_networks:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.01
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 200000
+      ramp: cosine
+      warmup_steps: 0
+output_dir: !!python/object/apply:pathlib.PosixPath
+- ../
+- outputs
+pipeline: !!python/object:NeRAF.NeRAF_pipeline.NeRAFPipelineConfig
+  _target: !!python/name:NeRAF.NeRAF_pipeline.NeRAFPipeline ''
+  audio_datamanager: !!python/object:NeRAF.NeRAF_datamanager.SoundSpacesDataManagerConfig
+    _target: !!python/name:NeRAF.NeRAF_datamanager.SoundSpacesDataManager ''
+    collate_fn: &id003 !!python/name:nerfstudio.data.utils.nerfstudio_collate.nerfstudio_collate ''
+    data: null
+    dataparser: !!python/object:NeRAF.NeRAF_dataparser.SoundSpacesDataParserConfig
+      _target: !!python/name:NeRAF.NeRAF_dataparser.SoundSpacesDataParser ''
+      data: !!python/object/apply:pathlib.PosixPath []
+    eval_num_rays_per_batch: 2048
+    fs: 22050
+    hop_len: 128
+    images_on_gpu: false
+    masks_on_gpu: false
+    max_len: 78
+    train_num_rays_per_batch: 2048
+  audio_model: !!python/object:NeRAF.NeRAF_model.NeRAFAudioModelConfig
+    N_features: 1024
+    N_freq_stft: 257
+    W_field: 512
+    _target: !!python/name:NeRAF.NeRAF_model.NeRAFAudioModel ''
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    criterion: SC+SLMSE
+    dataset: SoundSpaces
+    enable_collider: true
+    eval_num_rays_per_chunk: 4096
+    fs: 22050
+    grid_step: 0.0078125
+    hop_len: 128
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    loss_factor: 0.001
+    max_len: 78
+    prompt: null
+    use_grid: true
+    use_multiple_viewing_directions: true
+    win_len: 512
+  datamanager: !!python/object:nerfstudio.data.datamanagers.parallel_datamanager.ParallelDataManagerConfig
+    _target: !!python/name:nerfstudio.data.datamanagers.parallel_datamanager.ParallelDataManager ''
+    camera_optimizer: null
+    camera_res_scale_factor: 1.0
+    collate_fn: *id003
+    data: *id004
+    dataparser: !!python/object:nerfstudio.data.dataparsers.nerfstudio_dataparser.NerfstudioDataParserConfig
+      _target: !!python/name:nerfstudio.data.dataparsers.nerfstudio_dataparser.Nerfstudio ''
+      auto_scale_poses: true
+      center_method: poses
+      data: !!python/object/apply:pathlib.PosixPath []
+      depth_unit_scale_factor: 0.001
+      downscale_factor: null
+      eval_interval: 8
+      eval_mode: filename
+      load_3D_points: false
+      mask_color: null
+      orientation_method: up
+      scale_factor: 1.0
+      scene_scale: 1.0
+      train_split_fraction: 0.9
+    eval_image_indices: !!python/tuple
+    - 0
+    eval_num_images_to_sample_from: -1
+    eval_num_rays_per_batch: 4096
+    eval_num_times_to_repeat_images: -1
+    images_on_gpu: true
+    masks_on_gpu: true
+    max_thread_workers: null
+    num_processes: 1
+    patch_size: 1
+    pixel_sampler: !!python/object:nerfstudio.data.pixel_samplers.PixelSamplerConfig
+      _target: !!python/name:nerfstudio.data.pixel_samplers.PixelSampler ''
+      fisheye_crop_radius: null
+      ignore_mask: false
+      is_equirectangular: false
+      keep_full_image: false
+      max_num_iterations: 100
+      num_rays_per_batch: 4096
+      rejection_sample_mask: true
+    queue_size: 2
+    train_num_images_to_sample_from: -1
+    train_num_rays_per_batch: 4096
+    train_num_times_to_repeat_images: -1
+  model: !!python/object:nerfstudio.models.base_model.ModelConfig
+    _target: !!python/name:nerfstudio.models.base_model.Model ''
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    enable_collider: true
+    eval_num_rays_per_chunk: 4096
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    prompt: null
+  save_eval_audio_path: null
+  start_step_audio: 2000
+  vision_model: !!python/object:NeRAF.NeRAF_model.NeRAFVisionModelConfig
+    _target: !!python/name:NeRAF.NeRAF_model.NeRAFVisionModel ''
+    appearance_embed_dim: 32
+    average_init_density: 0.01
+    background_color: last_sample
+    base_res: 16
+    camera_optimizer: !!python/object:nerfstudio.cameras.camera_optimizers.CameraOptimizerConfig
+      _target: !!python/name:nerfstudio.cameras.camera_optimizers.CameraOptimizer ''
+      mode: SO3xR3
+      optimizer: null
+      rot_l2_penalty: 0.001
+      scheduler: null
+      trans_l2_penalty: 0.01
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    disable_scene_contraction: false
+    distortion_loss_mult: 0.002
+    enable_collider: true
+    eval_num_rays_per_chunk: 32768
+    far_plane: 1000.0
+    features_per_level: 2
+    hidden_dim: 64
+    hidden_dim_color: 64
+    hidden_dim_transient: 64
+    implementation: tcnn
+    interlevel_loss_mult: 1.0
+    log2_hashmap_size: 19
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    max_res: 2048
+    near_plane: 0.05
+    num_levels: 16
+    num_nerf_samples_per_ray: 48
+    num_proposal_iterations: 2
+    num_proposal_samples_per_ray: !!python/tuple
+    - 256
+    - 96
+    orientation_loss_mult: 0.0001
+    pred_normal_loss_mult: 0.001
+    predict_normals: false
+    prompt: null
+    proposal_initial_sampler: piecewise
+    proposal_net_args_list:
+    - hidden_dim: 16
+      log2_hashmap_size: 17
+      max_res: 128
+      num_levels: 5
+      use_linear: false
+    - hidden_dim: 16
+      log2_hashmap_size: 17
+      max_res: 256
+      num_levels: 5
+      use_linear: false
+    proposal_update_every: 5
+    proposal_warmup: 5000
+    proposal_weights_anneal_max_num_iters: 1000
+    proposal_weights_anneal_slope: 10.0
+    use_appearance_embedding: true
+    use_average_appearance_embedding: true
+    use_gradient_scaling: false
+    use_proposal_weight_anneal: true
+    use_same_proposal_network: false
+    use_single_jitter: true
+project_name: nerfstudio-project
+prompt: null
+relative_model_dir: !!python/object/apply:pathlib.PosixPath
+- nerfstudio_models
+save_only_latest_checkpoint: false
+steps_per_eval_all_images: 10000
+steps_per_eval_batch: 10000
+steps_per_eval_image: 10000
+steps_per_save: 10000
+timestamp: 2024-10-11_101120
+use_grad_scaler: false
+viewer: !!python/object:nerfstudio.configs.base_config.ViewerConfig
+  camera_frustum_scale: 0.1
+  default_composite_depth: true
+  image_format: jpeg
+  jpeg_quality: 75
+  make_share_url: false
+  max_num_display_images: 512
+  num_rays_per_chunk: 32768
+  quit_on_train_completion: false
+  relative_log_filename: viewer_log_filename.txt
+  websocket_host: 0.0.0.0
+  websocket_port: null
+  websocket_port_default: 7007
+vis: tensorboard

SoundSpaces/office_4_NeRAF/NeRAF/2024-10-11_101120/dataparser_transforms.json

@@ -0,0 +1,23 @@
+{
+    "transform": [
+        [
+            1.0,
+            0.0,
+            0.0,
+            2.089837074279785
+        ],
+        [
+            0.0,
+            1.0,
+            0.0,
+            1.5238884687423706
+        ],
+        [
+            0.0,
+            0.0,
+            1.0,
+            1.1040260791778564
+        ]
+    ],
+    "scale": 0.36437187805494425
+}

SoundSpaces/office_4_NeRAF/NeRAF/2024-10-11_101120/nerfstudio_models/step-000460000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6de3e2336f43dbee1b93ab77094e6b7b883fb229aec1738d1ed29eaed4eab82b
+size 782750627

SoundSpaces/room_2_NeRAF/NeRAF/2024-10-14_045651/config.yml

@@ -0,0 +1,294 @@
+!!python/object:nerfstudio.engine.trainer.TrainerConfig
+_target: !!python/name:nerfstudio.engine.trainer.Trainer ''
+data: &id004 !!python/object/apply:pathlib.PosixPath
+- ../
+- data
+- SoundSpaces
+- room_2
+experiment_name: room_2_NeRAF
+gradient_accumulation_steps: {}
+load_checkpoint: null
+load_config: null
+load_dir: null
+load_scheduler: true
+load_step: null
+log_gradients: false
+logging: !!python/object:nerfstudio.configs.base_config.LoggingConfig
+  local_writer: !!python/object:nerfstudio.configs.base_config.LocalWriterConfig
+    _target: !!python/name:nerfstudio.utils.writer.LocalWriter ''
+    enable: true
+    max_log_size: 10
+    stats_to_track: !!python/tuple
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Train Iter (time)
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Train Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Test PSNR
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Vis Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - Test Rays / Sec
+    - !!python/object/apply:nerfstudio.utils.writer.EventName
+      - ETA (time)
+  max_buffer_size: 20
+  profiler: basic
+  relative_log_dir: !!python/object/apply:pathlib.PosixPath []
+  steps_per_log: 10
+machine: !!python/object:nerfstudio.configs.base_config.MachineConfig
+  device_type: cuda
+  dist_url: auto
+  machine_rank: 0
+  num_devices: 1
+  num_machines: 1
+  seed: 42
+max_num_iterations: 500001
+method_name: NeRAF
+mixed_precision: true
+optimizers:
+  audio_fields:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: &id001 !!python/name:torch.optim.adam.Adam ''
+      eps: 1.0e-15
+      lr: 0.0001
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: &id002 !!python/name:nerfstudio.engine.schedulers.ExponentialDecayScheduler ''
+      lr_final: 1.0e-08
+      lr_pre_warmup: 1.0e-08
+      max_steps: 1002000
+      ramp: cosine
+      warmup_steps: 2000
+  camera_opt:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.001
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 5000
+      ramp: cosine
+      warmup_steps: 0
+  fields:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.01
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 200000
+      ramp: cosine
+      warmup_steps: 0
+  proposal_networks:
+    optimizer: !!python/object:nerfstudio.engine.optimizers.AdamOptimizerConfig
+      _target: *id001
+      eps: 1.0e-15
+      lr: 0.01
+      max_norm: null
+      weight_decay: 0
+    scheduler: !!python/object:nerfstudio.engine.schedulers.ExponentialDecaySchedulerConfig
+      _target: *id002
+      lr_final: 0.0001
+      lr_pre_warmup: 1.0e-08
+      max_steps: 200000
+      ramp: cosine
+      warmup_steps: 0
+output_dir: !!python/object/apply:pathlib.PosixPath
+- ../
+- outputs
+pipeline: !!python/object:NeRAF.NeRAF_pipeline.NeRAFPipelineConfig
+  _target: !!python/name:NeRAF.NeRAF_pipeline.NeRAFPipeline ''
+  audio_datamanager: !!python/object:NeRAF.NeRAF_datamanager.SoundSpacesDataManagerConfig
+    _target: !!python/name:NeRAF.NeRAF_datamanager.SoundSpacesDataManager ''
+    collate_fn: &id003 !!python/name:nerfstudio.data.utils.nerfstudio_collate.nerfstudio_collate ''
+    data: null
+    dataparser: !!python/object:NeRAF.NeRAF_dataparser.SoundSpacesDataParserConfig
+      _target: !!python/name:NeRAF.NeRAF_dataparser.SoundSpacesDataParser ''
+      data: !!python/object/apply:pathlib.PosixPath []
+    eval_num_rays_per_batch: 2048
+    fs: 22050
+    hop_len: 128
+    images_on_gpu: false
+    masks_on_gpu: false
+    max_len: 84
+    train_num_rays_per_batch: 2048
+  audio_model: !!python/object:NeRAF.NeRAF_model.NeRAFAudioModelConfig
+    N_features: 1024
+    N_freq_stft: 257
+    W_field: 512
+    _target: !!python/name:NeRAF.NeRAF_model.NeRAFAudioModel ''
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    criterion: SC+SLMSE
+    dataset: SoundSpaces
+    enable_collider: true
+    eval_num_rays_per_chunk: 4096
+    fs: 22050
+    grid_step: 0.0078125
+    hop_len: 128
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    loss_factor: 0.001
+    max_len: 84
+    prompt: null
+    use_grid: true
+    use_multiple_viewing_directions: true
+    win_len: 512
+  datamanager: !!python/object:nerfstudio.data.datamanagers.parallel_datamanager.ParallelDataManagerConfig
+    _target: !!python/name:nerfstudio.data.datamanagers.parallel_datamanager.ParallelDataManager ''
+    camera_optimizer: null
+    camera_res_scale_factor: 1.0
+    collate_fn: *id003
+    data: *id004
+    dataparser: !!python/object:nerfstudio.data.dataparsers.nerfstudio_dataparser.NerfstudioDataParserConfig
+      _target: !!python/name:nerfstudio.data.dataparsers.nerfstudio_dataparser.Nerfstudio ''
+      auto_scale_poses: true
+      center_method: poses
+      data: !!python/object/apply:pathlib.PosixPath []
+      depth_unit_scale_factor: 0.001
+      downscale_factor: null
+      eval_interval: 8
+      eval_mode: filename
+      load_3D_points: false
+      mask_color: null
+      orientation_method: up
+      scale_factor: 1.0
+      scene_scale: 1.0
+      train_split_fraction: 0.9
+    eval_image_indices: !!python/tuple
+    - 0
+    eval_num_images_to_sample_from: -1
+    eval_num_rays_per_batch: 4096
+    eval_num_times_to_repeat_images: -1
+    images_on_gpu: true
+    masks_on_gpu: true
+    max_thread_workers: null
+    num_processes: 1
+    patch_size: 1
+    pixel_sampler: !!python/object:nerfstudio.data.pixel_samplers.PixelSamplerConfig
+      _target: !!python/name:nerfstudio.data.pixel_samplers.PixelSampler ''
+      fisheye_crop_radius: null
+      ignore_mask: false
+      is_equirectangular: false
+      keep_full_image: false
+      max_num_iterations: 100
+      num_rays_per_batch: 4096
+      rejection_sample_mask: true
+    queue_size: 2
+    train_num_images_to_sample_from: -1
+    train_num_rays_per_batch: 4096
+    train_num_times_to_repeat_images: -1
+  model: !!python/object:nerfstudio.models.base_model.ModelConfig
+    _target: !!python/name:nerfstudio.models.base_model.Model ''
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    enable_collider: true
+    eval_num_rays_per_chunk: 4096
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    prompt: null
+  save_eval_audio_path: null
+  start_step_audio: 2000
+  vision_model: !!python/object:NeRAF.NeRAF_model.NeRAFVisionModelConfig
+    _target: !!python/name:NeRAF.NeRAF_model.NeRAFVisionModel ''
+    appearance_embed_dim: 32
+    average_init_density: 0.01
+    background_color: last_sample
+    base_res: 16
+    camera_optimizer: !!python/object:nerfstudio.cameras.camera_optimizers.CameraOptimizerConfig
+      _target: !!python/name:nerfstudio.cameras.camera_optimizers.CameraOptimizer ''
+      mode: SO3xR3
+      optimizer: null
+      rot_l2_penalty: 0.001
+      scheduler: null
+      trans_l2_penalty: 0.01
+    collider_params:
+      far_plane: 6.0
+      near_plane: 2.0
+    disable_scene_contraction: false
+    distortion_loss_mult: 0.002
+    enable_collider: true
+    eval_num_rays_per_chunk: 32768
+    far_plane: 1000.0
+    features_per_level: 2
+    hidden_dim: 64
+    hidden_dim_color: 64
+    hidden_dim_transient: 64
+    implementation: tcnn
+    interlevel_loss_mult: 1.0
+    log2_hashmap_size: 19
+    loss_coefficients:
+      rgb_loss_coarse: 1.0
+      rgb_loss_fine: 1.0
+    max_res: 2048
+    near_plane: 0.05
+    num_levels: 16
+    num_nerf_samples_per_ray: 48
+    num_proposal_iterations: 2
+    num_proposal_samples_per_ray: !!python/tuple
+    - 256
+    - 96
+    orientation_loss_mult: 0.0001
+    pred_normal_loss_mult: 0.001
+    predict_normals: false
+    prompt: null
+    proposal_initial_sampler: piecewise
+    proposal_net_args_list:
+    - hidden_dim: 16
+      log2_hashmap_size: 17
+      max_res: 128
+      num_levels: 5
+      use_linear: false
+    - hidden_dim: 16
+      log2_hashmap_size: 17
+      max_res: 256
+      num_levels: 5
+      use_linear: false
+    proposal_update_every: 5
+    proposal_warmup: 5000
+    proposal_weights_anneal_max_num_iters: 1000
+    proposal_weights_anneal_slope: 10.0
+    use_appearance_embedding: true
+    use_average_appearance_embedding: true
+    use_gradient_scaling: false
+    use_proposal_weight_anneal: true
+    use_same_proposal_network: false
+    use_single_jitter: true
+project_name: nerfstudio-project
+prompt: null
+relative_model_dir: !!python/object/apply:pathlib.PosixPath
+- nerfstudio_models
+save_only_latest_checkpoint: false
+steps_per_eval_all_images: 10000
+steps_per_eval_batch: 10000
+steps_per_eval_image: 10000
+steps_per_save: 10000
+timestamp: 2024-10-14_045651
+use_grad_scaler: false
+viewer: !!python/object:nerfstudio.configs.base_config.ViewerConfig
+  camera_frustum_scale: 0.1
+  default_composite_depth: true
+  image_format: jpeg
+  jpeg_quality: 75
+  make_share_url: false
+  max_num_display_images: 512
+  num_rays_per_chunk: 32768
+  quit_on_train_completion: false
+  relative_log_filename: viewer_log_filename.txt
+  websocket_host: 0.0.0.0
+  websocket_port: null
+  websocket_port_default: 7007
+vis: tensorboard

SoundSpaces/room_2_NeRAF/NeRAF/2024-10-14_045651/dataparser_transforms.json

@@ -0,0 +1,23 @@
+{
+    "transform": [
+        [
+            1.0,
+            0.0,
+            0.0,
+            2.031975269317627
+        ],
+        [
+            0.0,
+            1.0,
+            0.0,
+            -0.7323977947235107
+        ],
+        [
+            0.0,
+            0.0,
+            1.0,
+            2.499042510986328
+        ]
+    ],
+    "scale": 0.3277654238635945
+}

SoundSpaces/room_2_NeRAF/NeRAF/2024-10-14_045651/nerfstudio_models/step-000160000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64ea2d345cf504f531885609c19644e63a7314ad7be57dd764600564c8e8d658
+size 782750627