diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..d3b19d9aeee00545081163dae05351572b01aae8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +wandb/run-20250916_181645-hudaedu3/files/media/images/activation_visualization_2_1c027e12f9cd9c41c974.png filter=lfs diff=lfs merge=lfs -text +wandb/run-20250917_081730-bcwwe2gj/files/media/images/attention_map_252_97d63d8f0f1539361cfc.png filter=lfs diff=lfs merge=lfs -text +wandb/run-20250917_081730-bcwwe2gj/files/media/images/attention_map_52_73035baf92fe6d3bd927.png filter=lfs diff=lfs merge=lfs -text +wandb/run-20250917_081730-bcwwe2gj/files/media/images/comparison_374_900ec676100c245d3ca5.png filter=lfs diff=lfs merge=lfs -text +wandb/run-20250916_194552-aaciqyuf/files/media/images/comparison_74_b43a71ca0b755c8287ea.png filter=lfs diff=lfs merge=lfs -text +wandb/run-20250916_194552-aaciqyuf/files/media/images/activation_visualization_2_e7083dae4b9eda33b264.png filter=lfs diff=lfs merge=lfs -text diff --git a/wandb/run-20250916_175907-b2gr1gxl/files/config.yaml b/wandb/run-20250916_175907-b2gr1gxl/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2af29cdee8a98f84b66afaf02825637710461b08 --- /dev/null +++ b/wandb/run-20250916_175907-b2gr1gxl/files/config.yaml @@ -0,0 +1,255 @@ +_wandb: + value: + cli_version: 0.19.9 + m: + - "1": trainer/global_step + "6": + - 3 + "7": [] + python_version: 3.12.11 + t: + "1": + - 1 + - 41 + - 49 + - 50 + - 55 + - 106 + "2": + - 1 + - 41 + - 49 + - 50 + - 55 + - 106 + "3": + - 7 + - 13 + - 15 + - 16 + - 23 + - 55 + - 66 + "4": 3.12.11 + "5": 0.19.9 + "8": + - 5 + "12": 0.19.9 + "13": linux-x86_64 +checkpointing: + value: + every_n_train_steps: 9375 + load: null + save_top_k: 1 + save_weights_only: true +data_loader: + value: + test: + batch_size: 1 + num_workers: 4 + persistent_workers: false + seed: 2345 + train: + batch_size: 16 + num_workers: 16 + persistent_workers: true + seed: 1234 + val: + batch_size: 1 + num_workers: 1 + persistent_workers: true + seed: 3456 +dataset: + value: + re10k: + augment: true + background_color: + - 0 + - 0 + - 0 + baseline_max: 1e+10 + baseline_min: 0.001 + cameras_are_circular: false + input_image_shape: + - 256 + - 256 + make_baseline_1: true + max_fov: 100 + name: re10k + original_image_shape: + - 360 + - 640 + overfit_to_scene: null + relative_pose: true + roots: + - datasets/re10k + skip_bad_shape: true + view_sampler: + distill_steps: 0 + initial_max_distance_between_context_views: 25 + initial_min_distance_between_context_views: 25 + max_distance_between_context_views: 90 + min_distance_between_context_views: 45 + min_distance_to_context_views: 0 + name: bounded + num_context_views: 2 + num_target_views: 4 + warm_up_steps: 9375 +loss: + value: + lpips: + apply_after_step: 0 + weight: 0.05 + mse: + weight: 1 +mode: + value: train +model: + value: + decoder: + background_color: + - 0 + - 0 + - 0 + make_scale_invariant: false + name: splatting_cuda + use_gsplat: true + encoder: + apply_bounds_shim: true + backbone: + asymmetry_decoder: true + intrinsics_embed_degree: 4 + intrinsics_embed_loc: encoder + intrinsics_embed_type: token + model: ViTLarge_BaseDecoder + name: croco + patch_embed_cls: PatchEmbedDust3R + d_feature: 128 + gaussian_adapter: + gaussian_scale_max: 15 + gaussian_scale_min: 0.5 + sh_degree: 4 + gaussians_per_pixel: 1 + gs_params_head_type: dpt_gs + name: noposplat + num_monocular_samples: 32 + num_surfaces: 1 + opacity_mapping: + final: 0 + initial: 0 + warm_up: 1 + pose_free: true + predict_opacity: false + pretrained_weights: "" + visualizer: + export_ply: false + min_resolution: 256 + num_samples: 8 + vggt: + camera_prompt: false + detr: + base_HW: 64 + debug_mode: 0 + dec_layers: 1 + dec_n_points: 4 + decoder_type: conv + dim_down_factor: 1 + dim_feedforward_factor: 4 + dropout: 0.1 + enc_layers: 6 + enc_n_points: 4 + features_num_channel: + - 128 + - 32 + grad_mode: absgrad + hidden_dim: 32 + name: deformable_detr + num_feature_levels: 2 + num_level: 3 + num_queries: 256 + only_last_level: false + position_embedding: sine + pre_norm: false + same_split_head: false + self_layer_num: 1 + split_threshold: 1 + split_threshold_range: + - 0 + - 2 + use_mean_features: true + head_mode: pcd + input_image_shape: + - 518 + - 518 + num_visual_prompt: 0 + same_head: false + use_extrinsic_token: false + use_intrinsic_token: false +optimizer: + value: + accumulate: 1 + backbone_lr_multiplier: 0.1 + backbone_trainable: GB+H + lr: 0.0002 + warm_up_steps: 125 +seed: + value: 111123 +test: + value: + align_pose: false + compute_scores: true + output_path: test + pose_align_steps: 100 + pred_intrinsic: false + rot_opt_lr: 0.005 + save_compare: false + save_image: false + save_video: false + split_threshold_list: + - 0 + - 0.1 + - 0.2 + - 0.3 + - 0.4 + trans_opt_lr: 0.005 +train: + value: + align_corners: false + aux_loss: 0 + backbone_checkpoint: none + camera_consis_loss: 0 + camera_loss: 1 + debug_mode: 0 + depth_mode: null + distill_max_steps: 1000000 + distiller: "" + extended_visualization: false + extrinsic_drop: 0.1 + intrinsic_drop: 0.1 + intrinsic_scaling: true + one_sample_train: false + print_log_every_n_steps: 10 + random_split_strategy: score + return_level_gs: true + scaling_detach: true + split_loss: 0.01 + split_score_grad_scale: 10000 + split_score_log_scale: true + target_scale_mode: 2 + train_context: false + use_pred_point_focal: false +trainer: + value: + gradient_clip_val: 0.5 + max_steps: 18751 + num_nodes: 1 + val_check_interval: 500 +wandb: + value: + entity: scene-representation-group + mode: online + name: re10k_hpc + project: noposplat + tags: + - re10k + - 256x256 diff --git a/wandb/run-20250916_175907-b2gr1gxl/files/output.log b/wandb/run-20250916_175907-b2gr1gxl/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..c9f7049b96557956ce336fb2a0ea497e568c8e7d --- /dev/null +++ b/wandb/run-20250916_175907-b2gr1gxl/files/output.log @@ -0,0 +1,92 @@ +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0] + + | Name | Type | Params | Mode +--------------------------------------------------------- +0 | vggt | VGGT | 585 M | train +1 | decoder | DecoderSplattingCUDA | 0 | train +2 | losses | ModuleList | 0 | train +--------------------------------------------------------- +585 M Trainable params +0 Non-trainable params +585 M Total params +2,340.472 Total estimated model params size (MB) +773 Modules in train mode +980 Modules in eval mode +Sanity Checking: | | 0/? [00:00 + lambda data: self._console_raw_callback("stdout", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-09-16 17:59:13,133 ERROR MainThread:2554 [redirect.py:_on_write():661] [no run ID] error in stdout callback +Traceback (most recent call last): + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 2375, in + lambda data: self._console_raw_callback("stdout", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe diff --git a/wandb/run-20250916_180036-llem5odx/logs/debug-core.log b/wandb/run-20250916_180036-llem5odx/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..e0955a12af0712ba9494035c31b07c8d6c103273 --- /dev/null +++ b/wandb/run-20250916_180036-llem5odx/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-09-16T18:00:36.437776856+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpat1rmnr6/port-3084.txt","pid":3084,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-09-16T18:00:36.441984643+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":3084} +{"time":"2025-09-16T18:00:36.441728127+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":46097,"Zone":""}} +{"time":"2025-09-16T18:00:36.514549133+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:58060"} +{"time":"2025-09-16T18:00:36.799649358+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"llem5odx","id":"127.0.0.1:58060"} +{"time":"2025-09-16T18:00:37.347088092+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"llem5odx","id":"127.0.0.1:58060"} +{"time":"2025-09-16T18:02:13.014947086+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:58060"} +{"time":"2025-09-16T18:02:13.015128964+08:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:58060"} +{"time":"2025-09-16T18:02:13.015330601+08:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-09-16T18:02:13.01546968+08:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:58060"} +{"time":"2025-09-16T18:02:13.993533797+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:58060"} +{"time":"2025-09-16T18:02:13.993587876+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:58060"} +{"time":"2025-09-16T18:02:13.993646155+08:00","level":"INFO","msg":"server is closed"} diff --git a/wandb/run-20250916_180036-llem5odx/logs/debug-internal.log b/wandb/run-20250916_180036-llem5odx/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e51b53315c033b35ff747e86fc812038288def6d --- /dev/null +++ b/wandb/run-20250916_180036-llem5odx/logs/debug-internal.log @@ -0,0 +1,15 @@ +{"time":"2025-09-16T18:00:36.802061687+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc/wandb/run-20250916_180036-llem5odx/logs/debug-core.log"} +{"time":"2025-09-16T18:00:37.346862695+08:00","level":"INFO","msg":"created new stream","id":"llem5odx"} +{"time":"2025-09-16T18:00:37.347045132+08:00","level":"INFO","msg":"stream: started","id":"llem5odx"} +{"time":"2025-09-16T18:00:37.34726721+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"llem5odx"} +{"time":"2025-09-16T18:00:37.347474287+08:00","level":"INFO","msg":"sender: started","stream_id":"llem5odx"} +{"time":"2025-09-16T18:00:37.347713204+08:00","level":"INFO","msg":"handler: started","stream_id":"llem5odx"} +{"time":"2025-09-16T18:00:37.723439452+08:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-09-16T18:02:13.015316961+08:00","level":"INFO","msg":"stream: closing","id":"llem5odx"} +{"time":"2025-09-16T18:02:13.01544048+08:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-09-16T18:02:13.015647877+08:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-09-16T18:02:13.368444821+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2025-09-16T18:02:13.992962884+08:00","level":"INFO","msg":"handler: closed","stream_id":"llem5odx"} +{"time":"2025-09-16T18:02:13.993117282+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"llem5odx"} +{"time":"2025-09-16T18:02:13.993147081+08:00","level":"INFO","msg":"sender: closed","stream_id":"llem5odx"} +{"time":"2025-09-16T18:02:13.99327548+08:00","level":"INFO","msg":"stream: closed","id":"llem5odx"} diff --git a/wandb/run-20250916_180427-9sp0azf6/files/output.log b/wandb/run-20250916_180427-9sp0azf6/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8100e12832b498ab79fcc472502dbb75b4af03c2 --- /dev/null +++ b/wandb/run-20250916_180427-9sp0azf6/files/output.log @@ -0,0 +1,20 @@ +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [2] + + | Name | Type | Params | Mode +--------------------------------------------------------- +0 | vggt | VGGT | 586 M | train +1 | decoder | DecoderSplattingCUDA | 0 | train +2 | losses | ModuleList | 0 | train +--------------------------------------------------------- +586 M Trainable params +0 Non-trainable params +586 M Total params +2,345.910 Total estimated model params size (MB) +813 Modules in train mode +980 Modules in eval mode +Sanity Checking: | | 0/? [00:00 + lambda data: self._console_raw_callback("stdout", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-09-16 18:07:50,610 ERROR MainThread:7974 [redirect.py:_on_write():661] [no run ID] error in stdout callback +Traceback (most recent call last): + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 2375, in + lambda data: self._console_raw_callback("stdout", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe diff --git a/wandb/run-20250916_180500-j938t4jj/files/config.yaml b/wandb/run-20250916_180500-j938t4jj/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32e5cdf982262024f32548d45ee2db4cd53ff637 --- /dev/null +++ b/wandb/run-20250916_180500-j938t4jj/files/config.yaml @@ -0,0 +1,531 @@ +_wandb: + value: + cli_version: 0.19.9 + m: + - "1": loss/aux_1/mse + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": trainer/global_step + "6": + - 3 + "7": [] + - "1": info/global_step + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": loss/camera + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": loss/aux_0/lpips + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": comparison.format + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": points_context_imgs.captions + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": points_context_imgs.width + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": attention_map._type + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": loss/split_score + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": loss/aux_1/lpips + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": comparison.captions + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": points_context_imgs.count + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": attention_map.captions + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": loss/aux_2/mse + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": activation_visualization.height + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": attention_map.format + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": attention_map.count + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": loss/aux_2/lpips + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": points_context_imgs.format + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": activation_visualization.format + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": activation_visualization.count + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": activation_visualization._type + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": epoch + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": loss/total + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": loss/aux_0/mse + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": lr-AdamW/pg2 + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": loss/3dgs_2_0/lpips + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": loss/3dgs_2_0/mse + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": train/psnr_probabilistic + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": comparison.filenames + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": points_context_imgs.filenames + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": activation_visualization.captions + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": attention_map.height + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": lr-AdamW/pg1 + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": activation_visualization.filenames + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": activation_visualization.width + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": lr-AdamW/pg1-momentum + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": comparison.count + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": attention_map.filenames + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": attention_map.width + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": comparison.width + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": points_context_imgs.height + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": lr-AdamW/pg2-momentum + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": comparison._type + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": comparison.height + "5": 2 + "6": + - 1 + - 3 + "7": [] + - "1": points_context_imgs._type + "5": 2 + "6": + - 1 + - 3 + "7": [] + python_version: 3.12.11 + t: + "1": + - 1 + - 41 + - 49 + - 50 + - 55 + - 106 + "2": + - 1 + - 41 + - 49 + - 50 + - 55 + - 106 + "3": + - 7 + - 13 + - 15 + - 16 + - 23 + - 55 + - 66 + "4": 3.12.11 + "5": 0.19.9 + "8": + - 5 + "12": 0.19.9 + "13": linux-x86_64 +checkpointing: + value: + every_n_train_steps: 9375 + load: null + save_top_k: 1 + save_weights_only: true +data_loader: + value: + test: + batch_size: 1 + num_workers: 4 + persistent_workers: false + seed: 2345 + train: + batch_size: 16 + num_workers: 16 + persistent_workers: true + seed: 1234 + val: + batch_size: 1 + num_workers: 1 + persistent_workers: true + seed: 3456 +dataset: + value: + re10k: + augment: true + background_color: + - 0 + - 0 + - 0 + baseline_max: 1e+10 + baseline_min: 0.001 + cameras_are_circular: false + input_image_shape: + - 256 + - 256 + make_baseline_1: true + max_fov: 100 + name: re10k + original_image_shape: + - 360 + - 640 + overfit_to_scene: null + relative_pose: true + roots: + - datasets/re10k + skip_bad_shape: true + view_sampler: + distill_steps: 0 + initial_max_distance_between_context_views: 25 + initial_min_distance_between_context_views: 25 + max_distance_between_context_views: 90 + min_distance_between_context_views: 45 + min_distance_to_context_views: 0 + name: bounded + num_context_views: 2 + num_target_views: 4 + warm_up_steps: 9375 +loss: + value: + lpips: + apply_after_step: 0 + weight: 0.05 + mse: + weight: 1 +mode: + value: train +model: + value: + decoder: + background_color: + - 0 + - 0 + - 0 + make_scale_invariant: false + name: splatting_cuda + use_gsplat: true + encoder: + apply_bounds_shim: true + backbone: + asymmetry_decoder: true + intrinsics_embed_degree: 4 + intrinsics_embed_loc: encoder + intrinsics_embed_type: token + model: ViTLarge_BaseDecoder + name: croco + patch_embed_cls: PatchEmbedDust3R + d_feature: 128 + gaussian_adapter: + gaussian_scale_max: 15 + gaussian_scale_min: 0.5 + sh_degree: 4 + gaussians_per_pixel: 1 + gs_params_head_type: dpt_gs + name: noposplat + num_monocular_samples: 32 + num_surfaces: 1 + opacity_mapping: + final: 0 + initial: 0 + warm_up: 1 + pose_free: true + predict_opacity: false + pretrained_weights: "" + visualizer: + export_ply: false + min_resolution: 256 + num_samples: 8 + vggt: + camera_prompt: false + detr: + base_HW: 64 + debug_mode: 0 + dec_layers: 1 + dec_n_points: 4 + decoder_type: conv + dim_down_factor: 1 + dim_feedforward_factor: 4 + dropout: 0.1 + enc_layers: 6 + enc_n_points: 4 + features_num_channel: + - 128 + - 32 + grad_mode: absgrad + hidden_dim: 32 + name: deformable_detr + num_feature_levels: 2 + num_level: 3 + num_queries: 256 + only_last_level: false + position_embedding: sine + pre_norm: false + same_split_head: false + self_layer_num: 1 + split_threshold: 1 + split_threshold_range: + - 0 + - 2 + use_mean_features: true + head_mode: pcd + input_image_shape: + - 518 + - 518 + num_visual_prompt: 0 + same_head: false + use_extrinsic_token: false + use_intrinsic_token: false +optimizer: + value: + accumulate: 1 + backbone_lr_multiplier: 0.1 + backbone_trainable: GB+H + lr: 0.0002 + warm_up_steps: 125 +seed: + value: 111123 +test: + value: + align_pose: false + compute_scores: true + output_path: test + pose_align_steps: 100 + pred_intrinsic: false + rot_opt_lr: 0.005 + save_compare: false + save_image: false + save_video: false + split_threshold_list: + - 0 + - 0.1 + - 0.2 + - 0.3 + - 0.4 + trans_opt_lr: 0.005 +train: + value: + align_corners: false + aux_loss: 0 + backbone_checkpoint: none + camera_consis_loss: 0 + camera_loss: 1 + debug_mode: 0 + depth_mode: null + distill_max_steps: 1000000 + distiller: "" + extended_visualization: false + extrinsic_drop: 0.1 + intrinsic_drop: 0.1 + intrinsic_scaling: true + one_sample_train: false + print_log_every_n_steps: 10 + random_split_strategy: score + return_level_gs: true + scaling_detach: true + split_loss: 0.01 + split_score_grad_scale: 10000 + split_score_log_scale: true + target_scale_mode: 2 + train_context: false + use_pred_point_focal: false +trainer: + value: + gradient_clip_val: 0.5 + max_steps: 18751 + num_nodes: 1 + val_check_interval: 500 +wandb: + value: + entity: scene-representation-group + mode: online + name: re10k_hpc + project: noposplat + tags: + - re10k + - 256x256 diff --git a/wandb/run-20250916_180500-j938t4jj/files/output.log b/wandb/run-20250916_180500-j938t4jj/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..118a048d4fd822412372e57ecc029d76b36eb143 --- /dev/null +++ b/wandb/run-20250916_180500-j938t4jj/files/output.log @@ -0,0 +1,45 @@ +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [2] + + | Name | Type | Params | Mode +--------------------------------------------------------- +0 | vggt | VGGT | 585 M | train +1 | decoder | DecoderSplattingCUDA | 0 | train +2 | losses | ModuleList | 0 | train +--------------------------------------------------------- +585 M Trainable params +0 Non-trainable params +585 M Total params +2,340.472 Total estimated model params size (MB) +773 Modules in train mode +980 Modules in eval mode +Sanity Checking: | | 0/? [00:00 + lambda data: self._console_raw_callback("stdout", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe +2025-09-16 18:07:49,170 ERROR MainThread:8256 [redirect.py:_on_write():661] [no run ID] error in stdout callback +Traceback (most recent call last): + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write + cb(written_data) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 2375, in + lambda data: self._console_raw_callback("stdout", data), + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback + self._backend.interface.publish_output_raw(name, data) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw + self._publish_output_raw(o) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw + self._publish(rec) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish + self._sock_client.send_record_publish(record) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish + self.send_server_request(server_req) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request + self._send_message(msg) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message + self._sendall_with_error_handle(header + data) + File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle + sent = self._sock.send(data) + ^^^^^^^^^^^^^^^^^^^^^ +BrokenPipeError: [Errno 32] Broken pipe diff --git a/wandb/run-20250916_181645-hudaedu3/files/media/images/activation_visualization_26_85d83182a450f101d94a.png b/wandb/run-20250916_181645-hudaedu3/files/media/images/activation_visualization_26_85d83182a450f101d94a.png new file mode 100644 index 0000000000000000000000000000000000000000..40a02308165957ce50e4122dbc9c5c7961e46f7b Binary files /dev/null and b/wandb/run-20250916_181645-hudaedu3/files/media/images/activation_visualization_26_85d83182a450f101d94a.png differ diff --git a/wandb/run-20250916_181645-hudaedu3/files/media/images/activation_visualization_2_1c027e12f9cd9c41c974.png b/wandb/run-20250916_181645-hudaedu3/files/media/images/activation_visualization_2_1c027e12f9cd9c41c974.png new file mode 100644 index 0000000000000000000000000000000000000000..160b6f83082f8d4d40f8f1706ef84c8e08587af8 --- /dev/null +++ b/wandb/run-20250916_181645-hudaedu3/files/media/images/activation_visualization_2_1c027e12f9cd9c41c974.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c027e12f9cd9c41c9740092c453ccd32c02deae4c073727f5166e256b098fbe +size 142087 diff --git a/wandb/run-20250916_181645-hudaedu3/files/media/images/activation_visualization_51_3b93f26267630963185e.png b/wandb/run-20250916_181645-hudaedu3/files/media/images/activation_visualization_51_3b93f26267630963185e.png new file mode 100644 index 0000000000000000000000000000000000000000..21fffbf00917b4e9aabd34c6220248543355e909 Binary files /dev/null and b/wandb/run-20250916_181645-hudaedu3/files/media/images/activation_visualization_51_3b93f26267630963185e.png differ diff --git a/wandb/run-20250916_181645-hudaedu3/files/output.log b/wandb/run-20250916_181645-hudaedu3/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..6187781a3158f1b30be93954e0cafca884823a5e --- /dev/null +++ b/wandb/run-20250916_181645-hudaedu3/files/output.log @@ -0,0 +1,176 @@ +LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7] + + | Name | Type | Params | Mode +--------------------------------------------------------- +0 | vggt | VGGT | 586 M | train +1 | decoder | DecoderSplattingCUDA | 0 | train +2 | losses | ModuleList | 0 | train +--------------------------------------------------------- +586 M Trainable params +0 Non-trainable params +586 M Total params +2,345.910 Total estimated model params size (MB) +813 Modules in train mode +980 Modules in eval mode +Sanity Checking: | | 0/? [00:00