Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250916_181645-hudaedu3/files/media/images/activation_visualization_2_1c027e12f9cd9c41c974.png filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250917_081730-bcwwe2gj/files/media/images/attention_map_252_97d63d8f0f1539361cfc.png filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250917_081730-bcwwe2gj/files/media/images/attention_map_52_73035baf92fe6d3bd927.png filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250917_081730-bcwwe2gj/files/media/images/comparison_374_900ec676100c245d3ca5.png filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250916_194552-aaciqyuf/files/media/images/comparison_74_b43a71ca0b755c8287ea.png filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250916_194552-aaciqyuf/files/media/images/activation_visualization_2_e7083dae4b9eda33b264.png filter=lfs diff=lfs merge=lfs -text

wandb/run-20250916_175907-b2gr1gxl/files/config.yaml

@@ -0,0 +1,255 @@
+_wandb:
+    value:
+        cli_version: 0.19.9
+        m:
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+        python_version: 3.12.11
+        t:
+            "1":
+                - 1
+                - 41
+                - 49
+                - 50
+                - 55
+                - 106
+            "2":
+                - 1
+                - 41
+                - 49
+                - 50
+                - 55
+                - 106
+            "3":
+                - 7
+                - 13
+                - 15
+                - 16
+                - 23
+                - 55
+                - 66
+            "4": 3.12.11
+            "5": 0.19.9
+            "8":
+                - 5
+            "12": 0.19.9
+            "13": linux-x86_64
+checkpointing:
+    value:
+        every_n_train_steps: 9375
+        load: null
+        save_top_k: 1
+        save_weights_only: true
+data_loader:
+    value:
+        test:
+            batch_size: 1
+            num_workers: 4
+            persistent_workers: false
+            seed: 2345
+        train:
+            batch_size: 16
+            num_workers: 16
+            persistent_workers: true
+            seed: 1234
+        val:
+            batch_size: 1
+            num_workers: 1
+            persistent_workers: true
+            seed: 3456
+dataset:
+    value:
+        re10k:
+            augment: true
+            background_color:
+                - 0
+                - 0
+                - 0
+            baseline_max: 1e+10
+            baseline_min: 0.001
+            cameras_are_circular: false
+            input_image_shape:
+                - 256
+                - 256
+            make_baseline_1: true
+            max_fov: 100
+            name: re10k
+            original_image_shape:
+                - 360
+                - 640
+            overfit_to_scene: null
+            relative_pose: true
+            roots:
+                - datasets/re10k
+            skip_bad_shape: true
+            view_sampler:
+                distill_steps: 0
+                initial_max_distance_between_context_views: 25
+                initial_min_distance_between_context_views: 25
+                max_distance_between_context_views: 90
+                min_distance_between_context_views: 45
+                min_distance_to_context_views: 0
+                name: bounded
+                num_context_views: 2
+                num_target_views: 4
+                warm_up_steps: 9375
+loss:
+    value:
+        lpips:
+            apply_after_step: 0
+            weight: 0.05
+        mse:
+            weight: 1
+mode:
+    value: train
+model:
+    value:
+        decoder:
+            background_color:
+                - 0
+                - 0
+                - 0
+            make_scale_invariant: false
+            name: splatting_cuda
+            use_gsplat: true
+        encoder:
+            apply_bounds_shim: true
+            backbone:
+                asymmetry_decoder: true
+                intrinsics_embed_degree: 4
+                intrinsics_embed_loc: encoder
+                intrinsics_embed_type: token
+                model: ViTLarge_BaseDecoder
+                name: croco
+                patch_embed_cls: PatchEmbedDust3R
+            d_feature: 128
+            gaussian_adapter:
+                gaussian_scale_max: 15
+                gaussian_scale_min: 0.5
+                sh_degree: 4
+            gaussians_per_pixel: 1
+            gs_params_head_type: dpt_gs
+            name: noposplat
+            num_monocular_samples: 32
+            num_surfaces: 1
+            opacity_mapping:
+                final: 0
+                initial: 0
+                warm_up: 1
+            pose_free: true
+            predict_opacity: false
+            pretrained_weights: ""
+            visualizer:
+                export_ply: false
+                min_resolution: 256
+                num_samples: 8
+        vggt:
+            camera_prompt: false
+            detr:
+                base_HW: 64
+                debug_mode: 0
+                dec_layers: 1
+                dec_n_points: 4
+                decoder_type: conv
+                dim_down_factor: 1
+                dim_feedforward_factor: 4
+                dropout: 0.1
+                enc_layers: 6
+                enc_n_points: 4
+                features_num_channel:
+                    - 128
+                    - 32
+                grad_mode: absgrad
+                hidden_dim: 32
+                name: deformable_detr
+                num_feature_levels: 2
+                num_level: 3
+                num_queries: 256
+                only_last_level: false
+                position_embedding: sine
+                pre_norm: false
+                same_split_head: false
+                self_layer_num: 1
+                split_threshold: 1
+                split_threshold_range:
+                    - 0
+                    - 2
+                use_mean_features: true
+            head_mode: pcd
+            input_image_shape:
+                - 518
+                - 518
+            num_visual_prompt: 0
+            same_head: false
+            use_extrinsic_token: false
+            use_intrinsic_token: false
+optimizer:
+    value:
+        accumulate: 1
+        backbone_lr_multiplier: 0.1
+        backbone_trainable: GB+H
+        lr: 0.0002
+        warm_up_steps: 125
+seed:
+    value: 111123
+test:
+    value:
+        align_pose: false
+        compute_scores: true
+        output_path: test
+        pose_align_steps: 100
+        pred_intrinsic: false
+        rot_opt_lr: 0.005
+        save_compare: false
+        save_image: false
+        save_video: false
+        split_threshold_list:
+            - 0
+            - 0.1
+            - 0.2
+            - 0.3
+            - 0.4
+        trans_opt_lr: 0.005
+train:
+    value:
+        align_corners: false
+        aux_loss: 0
+        backbone_checkpoint: none
+        camera_consis_loss: 0
+        camera_loss: 1
+        debug_mode: 0
+        depth_mode: null
+        distill_max_steps: 1000000
+        distiller: ""
+        extended_visualization: false
+        extrinsic_drop: 0.1
+        intrinsic_drop: 0.1
+        intrinsic_scaling: true
+        one_sample_train: false
+        print_log_every_n_steps: 10
+        random_split_strategy: score
+        return_level_gs: true
+        scaling_detach: true
+        split_loss: 0.01
+        split_score_grad_scale: 10000
+        split_score_log_scale: true
+        target_scale_mode: 2
+        train_context: false
+        use_pred_point_focal: false
+trainer:
+    value:
+        gradient_clip_val: 0.5
+        max_steps: 18751
+        num_nodes: 1
+        val_check_interval: 500
+wandb:
+    value:
+        entity: scene-representation-group
+        mode: online
+        name: re10k_hpc
+        project: noposplat
+        tags:
+            - re10k
+            - 256x256

wandb/run-20250916_175907-b2gr1gxl/files/output.log

@@ -0,0 +1,92 @@
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]
+
+  | Name    | Type                 | Params | Mode
+---------------------------------------------------------
+0 | vggt    | VGGT                 | 585 M  | train
+1 | decoder | DecoderSplattingCUDA | 0      | train
+2 | losses  | ModuleList           | 0      | train
+---------------------------------------------------------
+585 M     Trainable params
+0         Non-trainable params
+585 M     Total params
+2,340.472 Total estimated model params size (MB)
+773       Modules in train mode
+980       Modules in eval mode
+Sanity Checking: |                                                                                                                                                                                                    | 0/? [00:00<?, ?it/s]
+Error executing job with overrides: ['+experiment=re10k', 'wandb.mode=online', 'wandb.name=re10k_hpc']
+Traceback (most recent call last):
+  File "/root/highspeedstorage/injae/code/ICLR2026_HCP/src/main.py", line 192, in train
+    trainer.fit(model_wrapper, datamodule=data_module, ckpt_path=checkpoint_path)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/trainer/trainer.py", line 561, in fit
+    call._call_and_handle_interrupt(
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/trainer/call.py", line 48, in _call_and_handle_interrupt
+    return trainer_fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/trainer/trainer.py", line 599, in _fit_impl
+    self._run(model, ckpt_path=ckpt_path)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/trainer/trainer.py", line 1012, in _run
+    results = self._run_stage()
+              ^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/trainer/trainer.py", line 1054, in _run_stage
+    self._run_sanity_check()
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/trainer/trainer.py", line 1083, in _run_sanity_check
+    val_loop.run()
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/loops/utilities.py", line 179, in _decorator
+    return loop_run(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/loops/evaluation_loop.py", line 120, in run
+    self.setup_data()
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/loops/evaluation_loop.py", line 176, in setup_data
+    dataloaders = _request_dataloader(source)
+                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/data_connector.py", line 326, in _request_dataloader
+    return data_source.dataloader()
+           ^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/data_connector.py", line 293, in dataloader
+    return call._call_lightning_datamodule_hook(self.instance.trainer, self.name)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/trainer/call.py", line 198, in _call_lightning_datamodule_hook
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/jaxtyping/_decorator.py", line 562, in wrapped_fn
+    return wrapped_fn_impl(args, kwargs, bound, memos)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/jaxtyping/_decorator.py", line 486, in wrapped_fn_impl
+    out = fn(*args, **kwargs)
+          ^^^^^^^^^^^^^^^^^^^
+  File "/root/highspeedstorage/injae/code/ICLR2026_HCP/src/dataset/data_module.py", line 161, in val_dataloader
+    datasets = get_dataset(self.dataset_cfgs, "val", self.step_tracker)
+               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/jaxtyping/_decorator.py", line 562, in wrapped_fn
+    return wrapped_fn_impl(args, kwargs, bound, memos)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/jaxtyping/_decorator.py", line 486, in wrapped_fn_impl
+    out = fn(*args, **kwargs)
+          ^^^^^^^^^^^^^^^^^^^
+  File "/root/highspeedstorage/injae/code/ICLR2026_HCP/src/dataset/__init__.py", line 129, in get_dataset
+    datasets.extend(get_dataset_re10k(cfgs, stage, step_tracker))
+                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/jaxtyping/_decorator.py", line 562, in wrapped_fn
+    return wrapped_fn_impl(args, kwargs, bound, memos)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/jaxtyping/_decorator.py", line 486, in wrapped_fn_impl
+    out = fn(*args, **kwargs)
+          ^^^^^^^^^^^^^^^^^^^
+  File "/root/highspeedstorage/injae/code/ICLR2026_HCP/src/dataset/__init__.py", line 74, in get_dataset_re10k
+    dataset = DATASETS[cfg.name](cfg, stage, view_sampler)
+              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/jaxtyping/_decorator.py", line 562, in wrapped_fn
+    return wrapped_fn_impl(args, kwargs, bound, memos)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/jaxtyping/_decorator.py", line 486, in wrapped_fn_impl
+    out = fn(*args, **kwargs)
+          ^^^^^^^^^^^^^^^^^^^
+  File "/root/highspeedstorage/injae/code/ICLR2026_HCP/src/dataset/dataset_re10k.py", line 75, in __init__
+    [path for path in root.iterdir() if path.suffix == ".torch"]
+                      ^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/pathlib.py", line 1056, in iterdir
+    for name in os.listdir(self):
+                ^^^^^^^^^^^^^^^^
+FileNotFoundError: [Errno 2] No such file or directory: 'datasets/re10k/test'
+
+Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.

wandb/run-20250916_175907-b2gr1gxl/files/requirements.txt

@@ -0,0 +1,129 @@
+ruff==0.11.4
+PyYAML==6.0.2
+sympy==1.13.3
+fsspec==2024.6.1
+imageio-ffmpeg==0.6.0
+setuptools==78.1.1
+timm==1.0.15
+fonttools==4.57.0
+nvidia-cublas-cu12==12.8.3.14
+nvidia-cuda-nvrtc-cu12==12.8.61
+dacite==1.9.2
+hydra-core==1.3.2
+certifi==2022.12.7
+rich==14.0.0
+filelock==3.13.1
+idna==3.4
+nvidia-cuda-cupti-cu12==12.8.57
+pip==25.2
+wandb==0.19.9
+ninja==1.11.1.4
+networkx==3.3
+GitPython==3.1.44
+moviepy==1.0.3
+markdown-it-py==3.0.0
+lightning==2.5.1
+Jinja2==3.1.4
+nvidia-cusparse-cu12==12.5.7.53
+tabulate==0.9.0
+tifffile==2025.3.30
+nvidia-nvtx-cu12==12.8.55
+jaxtyping==0.2.37
+aiohappyeyeballs==2.6.1
+MarkupSafe==2.1.5
+pydantic_core==2.33.1
+mypy-extensions==1.0.0
+lazy_loader==0.4
+pydantic==2.11.2
+typing-inspection==0.4.0
+pillow==10.4.0
+nvidia-nvjitlink-cu12==12.8.61
+omegaconf==2.3.0
+scikit-image==0.25.2
+jaraco.context==5.3.0
+typing_extensions==4.12.2
+mpmath==1.3.0
+opencv-python==4.11.0.86
+backports.tarfile==1.2.0
+async-timeout==5.0.1
+opt_einsum==3.4.0
+nvidia-cuda-runtime-cu12==12.8.57
+numpy==1.26.4
+torch==2.7.1+cu128
+psutil==7.0.0
+pytorch-lightning==2.5.1
+protobuf==5.29.4
+imageio==2.37.0
+colorama==0.4.6
+attrs==25.3.0
+importlib_metadata==8.0.0
+frozenlist==1.5.0
+beartype==0.19.0
+propcache==0.3.1
+typeguard==4.3.0
+torchaudio==2.7.1+cu128
+decorator==4.4.2
+nvidia-cudnn-cu12==9.7.1.26
+torchvision==0.22.1+cu128
+sentry-sdk==2.25.1
+mdurl==0.1.2
+more-itertools==10.3.0
+einops==0.8.1
+zipp==3.19.2
+antlr4-python3-runtime==4.9.3
+cycler==0.12.1
+nvidia-cufile-cu12==1.13.0.11
+jaraco.collections==5.1.0
+jaraco.text==3.12.1
+torchmetrics==1.8.2
+packaging==24.2
+black==25.1.0
+setproctitle==1.3.5
+Pygments==2.19.2
+tomli==2.2.1
+six==1.17.0
+docker-pycreds==0.4.0
+inflect==7.3.1
+requests==2.28.1
+svg.py==1.6.0
+python-dateutil==2.9.0.post0
+proglog==0.1.11
+gsplat==1.5.3
+e3nn==0.5.6
+pathspec==0.12.1
+aiosignal==1.3.2
+wheel==0.45.1
+multidict==6.3.2
+pyparsing==3.2.3
+plyfile==1.1
+jaraco.functools==4.0.1
+nvidia-nccl-cu12==2.26.2
+triton==3.3.1
+nvidia-cusparselt-cu12==0.6.3
+scikit-video==1.1.11
+huggingface-hub==0.30.1
+lightning-utilities==0.14.3
+wadler_lindig==0.1.4
+nvidia-curand-cu12==10.3.9.55
+matplotlib==3.10.1
+kiwisolver==1.4.8
+tqdm==4.67.1
+safetensors==0.5.3
+scipy==1.15.2
+lpips==0.1.4
+nvidia-cusolver-cu12==11.7.2.55
+click==8.1.8
+charset-normalizer==2.1.1
+platformdirs==4.3.7
+opt-einsum-fx==0.1.4
+annotated-types==0.7.0
+smmap==5.0.2
+gitdb==4.0.12
+nvidia-cufft-cu12==11.3.3.41
+contourpy==1.3.1
+urllib3==1.26.13
+autocommand==2.2.2
+yarl==1.19.0
+aiohttp==3.11.16
+colorspacious==1.1.2
+python-dotenv==1.1.0

wandb/run-20250916_175907-b2gr1gxl/files/wandb-metadata.json

@@ -0,0 +1,79 @@
+{
+  "os": "Linux-5.15.0-153-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.12.11",
+  "startedAt": "2025-09-16T09:59:07.967608Z",
+  "args": [
+    "+experiment=re10k",
+    "wandb.mode=online",
+    "wandb.name=re10k_hpc"
+  ],
+  "program": "-m src.main",
+  "git": {
+    "remote": "git@github.com:K-nowing/ICLR2026_HCP.git",
+    "commit": "6c0080eae0295d43dd7f18fd71e742d924238f89"
+  },
+  "email": "dna9041@korea.ac.kr",
+  "root": "/root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc",
+  "host": "notebook-72e474dd-92e1-11f0-a6ea-7ab79f1c53cb-0",
+  "executable": "/opt/conda/envs/ours/bin/python",
+  "cpu_count": 144,
+  "cpu_count_logical": 288,
+  "gpu": "NVIDIA B200",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1081101176832",
+      "used": "211821346816"
+    }
+  },
+  "memory": {
+    "total": "2434690400256"
+  },
+  "cpu": {
+    "count": 144,
+    "countLogical": 288
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    }
+  ],
+  "cudaVersion": "12.8"
+}

wandb/run-20250916_175907-b2gr1gxl/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":4}}

wandb/run-20250916_175907-b2gr1gxl/logs/debug-core.log

@@ -0,0 +1,13 @@
+{"time":"2025-09-16T17:58:58.528899618+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpn7ko7qag/port-2554.txt","pid":2554,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-09-16T17:58:58.533807236+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":36197,"Zone":""}}
+{"time":"2025-09-16T17:58:58.534069473+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2554}
+{"time":"2025-09-16T17:58:58.600621916+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:34378"}
+{"time":"2025-09-16T17:59:07.971235929+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"b2gr1gxl","id":"127.0.0.1:34378"}
+{"time":"2025-09-16T17:59:08.453561187+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"b2gr1gxl","id":"127.0.0.1:34378"}
+{"time":"2025-09-16T17:59:12.231310869+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:34378"}
+{"time":"2025-09-16T17:59:12.231500987+08:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:34378"}
+{"time":"2025-09-16T17:59:12.231691945+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-16T17:59:12.231793863+08:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:34378"}
+{"time":"2025-09-16T17:59:12.708066678+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:34378"}
+{"time":"2025-09-16T17:59:12.708153197+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:34378"}
+{"time":"2025-09-16T17:59:12.708195956+08:00","level":"INFO","msg":"server is closed"}

wandb/run-20250916_175907-b2gr1gxl/logs/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2025-09-16T17:59:07.974641846+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc/wandb/run-20250916_175907-b2gr1gxl/logs/debug-core.log"}
+{"time":"2025-09-16T17:59:08.453330869+08:00","level":"INFO","msg":"created new stream","id":"b2gr1gxl"}
+{"time":"2025-09-16T17:59:08.453519747+08:00","level":"INFO","msg":"stream: started","id":"b2gr1gxl"}
+{"time":"2025-09-16T17:59:08.453742784+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"b2gr1gxl"}
+{"time":"2025-09-16T17:59:08.453927222+08:00","level":"INFO","msg":"sender: started","stream_id":"b2gr1gxl"}
+{"time":"2025-09-16T17:59:08.4541101+08:00","level":"INFO","msg":"handler: started","stream_id":"b2gr1gxl"}
+{"time":"2025-09-16T17:59:08.91082464+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-09-16T17:59:12.231634945+08:00","level":"INFO","msg":"stream: closing","id":"b2gr1gxl"}
+{"time":"2025-09-16T17:59:12.231734944+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-09-16T17:59:12.231892902+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-09-16T17:59:12.578002072+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-09-16T17:59:12.707399806+08:00","level":"INFO","msg":"handler: closed","stream_id":"b2gr1gxl"}
+{"time":"2025-09-16T17:59:12.707531005+08:00","level":"INFO","msg":"sender: closed","stream_id":"b2gr1gxl"}
+{"time":"2025-09-16T17:59:12.707515925+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"b2gr1gxl"}
+{"time":"2025-09-16T17:59:12.707836441+08:00","level":"INFO","msg":"stream: closed","id":"b2gr1gxl"}

wandb/run-20250916_175907-b2gr1gxl/logs/debug.log

@@ -0,0 +1,85 @@
+2025-09-16 17:59:07,945 INFO    MainThread:2554 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9
+2025-09-16 17:59:07,945 INFO    MainThread:2554 [wandb_setup.py:_flush():67] Configure stats pid to 2554
+2025-09-16 17:59:07,945 INFO    MainThread:2554 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings
+2025-09-16 17:59:07,945 INFO    MainThread:2554 [wandb_setup.py:_flush():67] Loading settings from /root/highspeedstorage/injae/code/ICLR2026_HCP/wandb/settings
+2025-09-16 17:59:07,945 INFO    MainThread:2554 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-09-16 17:59:07,945 INFO    MainThread:2554 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc/wandb/run-20250916_175907-b2gr1gxl/logs/debug.log
+2025-09-16 17:59:07,946 INFO    MainThread:2554 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc/wandb/run-20250916_175907-b2gr1gxl/logs/debug-internal.log
+2025-09-16 17:59:07,946 INFO    MainThread:2554 [wandb_init.py:init():781] calling init triggers
+2025-09-16 17:59:07,946 INFO    MainThread:2554 [wandb_init.py:init():786] wandb.init called with sweep_config: {}
+config: {'model': {'encoder': {'backbone': {'name': 'croco', 'model': 'ViTLarge_BaseDecoder', 'patch_embed_cls': 'PatchEmbedDust3R', 'asymmetry_decoder': True, 'intrinsics_embed_loc': 'encoder', 'intrinsics_embed_degree': 4, 'intrinsics_embed_type': 'token'}, 'name': 'noposplat', 'opacity_mapping': {'initial': 0.0, 'final': 0.0, 'warm_up': 1}, 'num_monocular_samples': 32, 'num_surfaces': 1, 'predict_opacity': False, 'gaussians_per_pixel': 1, 'gaussian_adapter': {'gaussian_scale_min': 0.5, 'gaussian_scale_max': 15.0, 'sh_degree': 4}, 'd_feature': 128, 'visualizer': {'num_samples': 8, 'min_resolution': 256, 'export_ply': False}, 'apply_bounds_shim': True, 'gs_params_head_type': 'dpt_gs', 'pose_free': True, 'pretrained_weights': ''}, 'decoder': {'name': 'splatting_cuda', 'background_color': [0.0, 0.0, 0.0], 'make_scale_invariant': False, 'use_gsplat': True}, 'vggt': {'input_image_shape': [518, 518], 'head_mode': 'pcd', 'same_head': False, 'camera_prompt': False, 'num_visual_prompt': 0, 'use_extrinsic_token': False, 'use_intrinsic_token': False, 'detr': {'name': 'deformable_detr', 'features_num_channel': [128, 32], 'hidden_dim': 32, 'dim_down_factor': 1, 'dim_feedforward_factor': 4, 'enc_layers': 6, 'dec_layers': 1, 'dropout': 0.1, 'num_feature_levels': 2, 'dec_n_points': 4, 'enc_n_points': 4, 'split_threshold': 1.0, 'num_queries': 256, 'base_HW': 64, 'position_embedding': 'sine', 'pre_norm': False, 'self_layer_num': 1, 'num_level': 3, 'debug_mode': 0, 'decoder_type': 'conv', 'split_threshold_range': [0.0, 2.0], 'only_last_level': False, 'grad_mode': 'absgrad', 'same_split_head': False, 'use_mean_features': True}}}, 'loss': {'mse': {'weight': 1.0}, 'lpips': {'weight': 0.05, 'apply_after_step': 0}}, 'wandb': {'project': 'noposplat', 'entity': 'scene-representation-group', 'name': 're10k_hpc', 'mode': 'online', 'tags': ['re10k', '256x256']}, 'mode': 'train', 'data_loader': {'train': {'num_workers': 16, 'persistent_workers': True, 'batch_size': 16, 'seed': 1234}, 'test': {'num_workers': 4, 'persistent_workers': False, 'batch_size': 1, 'seed': 2345}, 'val': {'num_workers': 1, 'persistent_workers': True, 'batch_size': 1, 'seed': 3456}}, 'optimizer': {'lr': 0.0002, 'warm_up_steps': 125, 'backbone_lr_multiplier': 0.1, 'backbone_trainable': 'GB+H', 'accumulate': 1}, 'checkpointing': {'load': None, 'every_n_train_steps': 9375, 'save_top_k': 1, 'save_weights_only': True}, 'train': {'depth_mode': None, 'extended_visualization': False, 'print_log_every_n_steps': 10, 'distiller': '', 'distill_max_steps': 1000000, 'debug_mode': 0, 'camera_consis_loss': 0.0, 'camera_loss': 1.0, 'aux_loss': 0.0, 'one_sample_train': False, 'target_scale_mode': 2, 'extrinsic_drop': 0.1, 'intrinsic_drop': 0.1, 'align_corners': False, 'backbone_checkpoint': 'none', 'intrinsic_scaling': True, 'return_level_gs': True, 'train_context': False, 'use_pred_point_focal': False, 'scaling_detach': True, 'split_score_log_scale': True, 'split_score_grad_scale': 10000.0, 'split_loss': 0.01, 'random_split_strategy': 'score'}, 'test': {'output_path': 'test', 'align_pose': False, 'pose_align_steps': 100, 'rot_opt_lr': 0.005, 'trans_opt_lr': 0.005, 'compute_scores': True, 'save_image': False, 'save_video': False, 'save_compare': False, 'split_threshold_list': [0.0, 0.1, 0.2, 0.3, 0.4], 'pred_intrinsic': False}, 'seed': 111123, 'trainer': {'max_steps': 18751, 'val_check_interval': 500, 'gradient_clip_val': 0.5, 'num_nodes': 1}, 'dataset': {'re10k': {'make_baseline_1': True, 'relative_pose': True, 'augment': True, 'background_color': [0.0, 0.0, 0.0], 'overfit_to_scene': None, 'skip_bad_shape': True, 'view_sampler': {'name': 'bounded', 'num_target_views': 4, 'num_context_views': 2, 'min_distance_between_context_views': 45, 'max_distance_between_context_views': 90, 'min_distance_to_context_views': 0, 'warm_up_steps': 9375, 'initial_min_distance_between_context_views': 25, 'initial_max_distance_between_context_views': 25, 'distill_steps': 0}, 'name': 're10k', 'roots': ['datasets/re10k'], 'input_image_shape': [256, 256], 'original_image_shape': [360, 640], 'cameras_are_circular': False, 'baseline_min': 0.001, 'baseline_max': 10000000000.0, 'max_fov': 100.0}}, '_wandb': {}}
+2025-09-16 17:59:07,946 INFO    MainThread:2554 [wandb_init.py:init():809] starting backend
+2025-09-16 17:59:07,946 INFO    MainThread:2554 [wandb_init.py:init():813] sending inform_init request
+2025-09-16 17:59:07,966 INFO    MainThread:2554 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-09-16 17:59:07,966 INFO    MainThread:2554 [wandb_init.py:init():823] backend started and connected
+2025-09-16 17:59:07,981 INFO    MainThread:2554 [wandb_init.py:init():915] updated telemetry
+2025-09-16 17:59:08,037 INFO    MainThread:2554 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout
+2025-09-16 17:59:08,898 INFO    MainThread:2554 [wandb_init.py:init():1014] starting run threads in backend
+2025-09-16 17:59:09,076 INFO    MainThread:2554 [wandb_run.py:_console_start():2454] atexit reg
+2025-09-16 17:59:09,076 INFO    MainThread:2554 [wandb_run.py:_redirect():2306] redirect: wrap_raw
+2025-09-16 17:59:09,079 INFO    MainThread:2554 [wandb_run.py:_redirect():2371] Wrapping output streams.
+2025-09-16 17:59:09,080 INFO    MainThread:2554 [wandb_run.py:_redirect():2394] Redirects installed.
+2025-09-16 17:59:09,082 INFO    MainThread:2554 [wandb_init.py:init():1056] run started, returning control to user process
+2025-09-16 17:59:12,229 INFO    MsgRouterThr:2554 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.
+2025-09-16 17:59:13,127 ERROR   MainThread:2554 [redirect.py:_on_write():661] [no run ID] error in stdout callback
+Traceback (most recent call last):
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write
+    cb(written_data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 2375, in <lambda>
+    lambda data: self._console_raw_callback("stdout", data),
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback
+    self._backend.interface.publish_output_raw(name, data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw
+    self._publish_output_raw(o)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
+    self._publish(rec)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
+    self._sock_client.send_record_publish(record)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
+    self.send_server_request(server_req)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
+    self._send_message(msg)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
+    self._sendall_with_error_handle(header + data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
+    sent = self._sock.send(data)
+           ^^^^^^^^^^^^^^^^^^^^^
+BrokenPipeError: [Errno 32] Broken pipe
+2025-09-16 17:59:13,133 ERROR   MainThread:2554 [redirect.py:_on_write():661] [no run ID] error in stdout callback
+Traceback (most recent call last):
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write
+    cb(written_data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 2375, in <lambda>
+    lambda data: self._console_raw_callback("stdout", data),
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback
+    self._backend.interface.publish_output_raw(name, data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw
+    self._publish_output_raw(o)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
+    self._publish(rec)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
+    self._sock_client.send_record_publish(record)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
+    self.send_server_request(server_req)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
+    self._send_message(msg)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
+    self._sendall_with_error_handle(header + data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
+    sent = self._sock.send(data)
+           ^^^^^^^^^^^^^^^^^^^^^
+BrokenPipeError: [Errno 32] Broken pipe

wandb/run-20250916_180036-llem5odx/logs/debug-core.log

@@ -0,0 +1,13 @@
+{"time":"2025-09-16T18:00:36.437776856+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpat1rmnr6/port-3084.txt","pid":3084,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-09-16T18:00:36.441984643+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":3084}
+{"time":"2025-09-16T18:00:36.441728127+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":46097,"Zone":""}}
+{"time":"2025-09-16T18:00:36.514549133+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:58060"}
+{"time":"2025-09-16T18:00:36.799649358+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"llem5odx","id":"127.0.0.1:58060"}
+{"time":"2025-09-16T18:00:37.347088092+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"llem5odx","id":"127.0.0.1:58060"}
+{"time":"2025-09-16T18:02:13.014947086+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:58060"}
+{"time":"2025-09-16T18:02:13.015128964+08:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:58060"}
+{"time":"2025-09-16T18:02:13.015330601+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-16T18:02:13.01546968+08:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:58060"}
+{"time":"2025-09-16T18:02:13.993533797+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:58060"}
+{"time":"2025-09-16T18:02:13.993587876+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:58060"}
+{"time":"2025-09-16T18:02:13.993646155+08:00","level":"INFO","msg":"server is closed"}

wandb/run-20250916_180036-llem5odx/logs/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2025-09-16T18:00:36.802061687+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc/wandb/run-20250916_180036-llem5odx/logs/debug-core.log"}
+{"time":"2025-09-16T18:00:37.346862695+08:00","level":"INFO","msg":"created new stream","id":"llem5odx"}
+{"time":"2025-09-16T18:00:37.347045132+08:00","level":"INFO","msg":"stream: started","id":"llem5odx"}
+{"time":"2025-09-16T18:00:37.34726721+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"llem5odx"}
+{"time":"2025-09-16T18:00:37.347474287+08:00","level":"INFO","msg":"sender: started","stream_id":"llem5odx"}
+{"time":"2025-09-16T18:00:37.347713204+08:00","level":"INFO","msg":"handler: started","stream_id":"llem5odx"}
+{"time":"2025-09-16T18:00:37.723439452+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-09-16T18:02:13.015316961+08:00","level":"INFO","msg":"stream: closing","id":"llem5odx"}
+{"time":"2025-09-16T18:02:13.01544048+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-09-16T18:02:13.015647877+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-09-16T18:02:13.368444821+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-09-16T18:02:13.992962884+08:00","level":"INFO","msg":"handler: closed","stream_id":"llem5odx"}
+{"time":"2025-09-16T18:02:13.993117282+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"llem5odx"}
+{"time":"2025-09-16T18:02:13.993147081+08:00","level":"INFO","msg":"sender: closed","stream_id":"llem5odx"}
+{"time":"2025-09-16T18:02:13.99327548+08:00","level":"INFO","msg":"stream: closed","id":"llem5odx"}

wandb/run-20250916_180427-9sp0azf6/files/output.log

@@ -0,0 +1,20 @@
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [2]
+
+  | Name    | Type                 | Params | Mode
+---------------------------------------------------------
+0 | vggt    | VGGT                 | 586 M  | train
+1 | decoder | DecoderSplattingCUDA | 0      | train
+2 | losses  | ModuleList           | 0      | train
+---------------------------------------------------------
+586 M     Trainable params
+0         Non-trainable params
+586 M     Total params
+2,345.910 Total estimated model params size (MB)
+813       Modules in train mode
+980       Modules in eval mode
+Sanity Checking: |                                                                                                                                                                                                    | 0/? [00:00<?, ?it/s][2025-09-16 18:04:29,494][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:425: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=279` in the `DataLoader` to improve performance.
+
+Validation epoch start on rank 0
+Sanity Checking DataLoader 0:   0%|                                                                                                                                                                                   | 0/1 [00:00<?, ?it/s]validation step 0; scene = ['306e2b7785657539']; context = [[48, 73]]
+target intrinsic:  tensor(0.8595, device='cuda:0') tensor(0.8597, device='cuda:0')
+pred intrinsic:  tensor(0.8780, device='cuda:0') tensor(0.8776, device='cuda:0')

wandb/run-20250916_180427-9sp0azf6/files/requirements.txt

@@ -0,0 +1,129 @@
+ruff==0.11.4
+PyYAML==6.0.2
+sympy==1.13.3
+fsspec==2024.6.1
+imageio-ffmpeg==0.6.0
+setuptools==78.1.1
+timm==1.0.15
+fonttools==4.57.0
+nvidia-cublas-cu12==12.8.3.14
+nvidia-cuda-nvrtc-cu12==12.8.61
+dacite==1.9.2
+hydra-core==1.3.2
+certifi==2022.12.7
+rich==14.0.0
+filelock==3.13.1
+idna==3.4
+nvidia-cuda-cupti-cu12==12.8.57
+pip==25.2
+wandb==0.19.9
+ninja==1.11.1.4
+networkx==3.3
+GitPython==3.1.44
+moviepy==1.0.3
+markdown-it-py==3.0.0
+lightning==2.5.1
+Jinja2==3.1.4
+nvidia-cusparse-cu12==12.5.7.53
+tabulate==0.9.0
+tifffile==2025.3.30
+nvidia-nvtx-cu12==12.8.55
+jaxtyping==0.2.37
+aiohappyeyeballs==2.6.1
+MarkupSafe==2.1.5
+pydantic_core==2.33.1
+mypy-extensions==1.0.0
+lazy_loader==0.4
+pydantic==2.11.2
+typing-inspection==0.4.0
+pillow==10.4.0
+nvidia-nvjitlink-cu12==12.8.61
+omegaconf==2.3.0
+scikit-image==0.25.2
+jaraco.context==5.3.0
+typing_extensions==4.12.2
+mpmath==1.3.0
+opencv-python==4.11.0.86
+backports.tarfile==1.2.0
+async-timeout==5.0.1
+opt_einsum==3.4.0
+nvidia-cuda-runtime-cu12==12.8.57
+numpy==1.26.4
+torch==2.7.1+cu128
+psutil==7.0.0
+pytorch-lightning==2.5.1
+protobuf==5.29.4
+imageio==2.37.0
+colorama==0.4.6
+attrs==25.3.0
+importlib_metadata==8.0.0
+frozenlist==1.5.0
+beartype==0.19.0
+propcache==0.3.1
+typeguard==4.3.0
+torchaudio==2.7.1+cu128
+decorator==4.4.2
+nvidia-cudnn-cu12==9.7.1.26
+torchvision==0.22.1+cu128
+sentry-sdk==2.25.1
+mdurl==0.1.2
+more-itertools==10.3.0
+einops==0.8.1
+zipp==3.19.2
+antlr4-python3-runtime==4.9.3
+cycler==0.12.1
+nvidia-cufile-cu12==1.13.0.11
+jaraco.collections==5.1.0
+jaraco.text==3.12.1
+torchmetrics==1.8.2
+packaging==24.2
+black==25.1.0
+setproctitle==1.3.5
+Pygments==2.19.2
+tomli==2.2.1
+six==1.17.0
+docker-pycreds==0.4.0
+inflect==7.3.1
+requests==2.28.1
+svg.py==1.6.0
+python-dateutil==2.9.0.post0
+proglog==0.1.11
+gsplat==1.5.3
+e3nn==0.5.6
+pathspec==0.12.1
+aiosignal==1.3.2
+wheel==0.45.1
+multidict==6.3.2
+pyparsing==3.2.3
+plyfile==1.1
+jaraco.functools==4.0.1
+nvidia-nccl-cu12==2.26.2
+triton==3.3.1
+nvidia-cusparselt-cu12==0.6.3
+scikit-video==1.1.11
+huggingface-hub==0.30.1
+lightning-utilities==0.14.3
+wadler_lindig==0.1.4
+nvidia-curand-cu12==10.3.9.55
+matplotlib==3.10.1
+kiwisolver==1.4.8
+tqdm==4.67.1
+safetensors==0.5.3
+scipy==1.15.2
+lpips==0.1.4
+nvidia-cusolver-cu12==11.7.2.55
+click==8.1.8
+charset-normalizer==2.1.1
+platformdirs==4.3.7
+opt-einsum-fx==0.1.4
+annotated-types==0.7.0
+smmap==5.0.2
+gitdb==4.0.12
+nvidia-cufft-cu12==11.3.3.41
+contourpy==1.3.1
+urllib3==1.26.13
+autocommand==2.2.2
+yarl==1.19.0
+aiohttp==3.11.16
+colorspacious==1.1.2
+python-dotenv==1.1.0

wandb/run-20250916_180427-9sp0azf6/files/wandb-metadata.json

@@ -0,0 +1,79 @@
+{
+  "os": "Linux-5.15.0-153-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.12.11",
+  "startedAt": "2025-09-16T10:04:27.261199Z",
+  "args": [
+    "+experiment=re10k",
+    "wandb.mode=online",
+    "wandb.name=re10k_hpc"
+  ],
+  "program": "-m src.main",
+  "git": {
+    "remote": "git@github.com:K-nowing/ICLR2026_HCP.git",
+    "commit": "6c0080eae0295d43dd7f18fd71e742d924238f89"
+  },
+  "email": "dna9041@korea.ac.kr",
+  "root": "/root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc",
+  "host": "notebook-72e474dd-92e1-11f0-a6ea-7ab79f1c53cb-0",
+  "executable": "/opt/conda/envs/ours/bin/python",
+  "cpu_count": 144,
+  "cpu_count_logical": 288,
+  "gpu": "NVIDIA B200",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1081101176832",
+      "used": "211821617152"
+    }
+  },
+  "memory": {
+    "total": "2434690400256"
+  },
+  "cpu": {
+    "count": 144,
+    "countLogical": 288
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    }
+  ],
+  "cudaVersion": "12.8"
+}

wandb/run-20250916_180427-9sp0azf6/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-09-16T18:04:26.94500103+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp5b1b0abs/port-7308.txt","pid":7308,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-09-16T18:04:26.94900602+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":46089,"Zone":""}}
+{"time":"2025-09-16T18:04:26.949280017+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":7308}
+{"time":"2025-09-16T18:04:27.027130604+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:41308"}
+{"time":"2025-09-16T18:04:27.26441636+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"9sp0azf6","id":"127.0.0.1:41308"}
+{"time":"2025-09-16T18:04:27.732986187+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"9sp0azf6","id":"127.0.0.1:41308"}
+{"time":"2025-09-16T18:04:31.492209193+08:00","level":"INFO","msg":"Parent process exited, terminating service process."}

wandb/run-20250916_180427-9sp0azf6/logs/debug-internal.log

@@ -0,0 +1,7 @@
+{"time":"2025-09-16T18:04:27.266529494+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc/wandb/run-20250916_180427-9sp0azf6/logs/debug-core.log"}
+{"time":"2025-09-16T18:04:27.73276599+08:00","level":"INFO","msg":"created new stream","id":"9sp0azf6"}
+{"time":"2025-09-16T18:04:27.732944628+08:00","level":"INFO","msg":"stream: started","id":"9sp0azf6"}
+{"time":"2025-09-16T18:04:27.733220025+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"9sp0azf6"}
+{"time":"2025-09-16T18:04:27.733455822+08:00","level":"INFO","msg":"handler: started","stream_id":"9sp0azf6"}
+{"time":"2025-09-16T18:04:27.733497061+08:00","level":"INFO","msg":"sender: started","stream_id":"9sp0azf6"}
+{"time":"2025-09-16T18:04:28.133354147+08:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20250916_180427-9sp0azf6/logs/debug.log

@@ -0,0 +1,22 @@
+2025-09-16 18:04:27,238 INFO    MainThread:7308 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9
+2025-09-16 18:04:27,238 INFO    MainThread:7308 [wandb_setup.py:_flush():67] Configure stats pid to 7308
+2025-09-16 18:04:27,238 INFO    MainThread:7308 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings
+2025-09-16 18:04:27,238 INFO    MainThread:7308 [wandb_setup.py:_flush():67] Loading settings from /root/highspeedstorage/injae/code/ICLR2026_HCP/wandb/settings
+2025-09-16 18:04:27,238 INFO    MainThread:7308 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-09-16 18:04:27,239 INFO    MainThread:7308 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc/wandb/run-20250916_180427-9sp0azf6/logs/debug.log
+2025-09-16 18:04:27,239 INFO    MainThread:7308 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc/wandb/run-20250916_180427-9sp0azf6/logs/debug-internal.log
+2025-09-16 18:04:27,239 INFO    MainThread:7308 [wandb_init.py:init():781] calling init triggers
+2025-09-16 18:04:27,240 INFO    MainThread:7308 [wandb_init.py:init():786] wandb.init called with sweep_config: {}
+config: {'model': {'encoder': {'backbone': {'name': 'croco', 'model': 'ViTLarge_BaseDecoder', 'patch_embed_cls': 'PatchEmbedDust3R', 'asymmetry_decoder': True, 'intrinsics_embed_loc': 'encoder', 'intrinsics_embed_degree': 4, 'intrinsics_embed_type': 'token'}, 'name': 'noposplat', 'opacity_mapping': {'initial': 0.0, 'final': 0.0, 'warm_up': 1}, 'num_monocular_samples': 32, 'num_surfaces': 1, 'predict_opacity': False, 'gaussians_per_pixel': 1, 'gaussian_adapter': {'gaussian_scale_min': 0.5, 'gaussian_scale_max': 15.0, 'sh_degree': 4}, 'd_feature': 128, 'visualizer': {'num_samples': 8, 'min_resolution': 256, 'export_ply': False}, 'apply_bounds_shim': True, 'gs_params_head_type': 'dpt_gs', 'pose_free': True, 'pretrained_weights': ''}, 'decoder': {'name': 'splatting_cuda', 'background_color': [0.0, 0.0, 0.0], 'make_scale_invariant': False, 'use_gsplat': True}, 'vggt': {'input_image_shape': [518, 518], 'head_mode': 'pcd', 'same_head': False, 'camera_prompt': False, 'num_visual_prompt': 0, 'use_extrinsic_token': False, 'use_intrinsic_token': False, 'detr': {'name': 'deformable_detr', 'features_num_channel': [128, 32], 'hidden_dim': 32, 'dim_down_factor': 1, 'dim_feedforward_factor': 4, 'enc_layers': 6, 'dec_layers': 3, 'dropout': 0.1, 'num_feature_levels': 2, 'dec_n_points': 4, 'enc_n_points': 4, 'split_threshold': 1.0, 'num_queries': 256, 'base_HW': 64, 'position_embedding': 'sine', 'pre_norm': False, 'self_layer_num': 1, 'num_level': 3, 'debug_mode': 0, 'decoder_type': 'conv', 'split_threshold_range': [0.0, 2.0], 'only_last_level': False, 'grad_mode': 'absgrad', 'same_split_head': False, 'use_mean_features': True}}}, 'loss': {'mse': {'weight': 1.0}, 'lpips': {'weight': 0.05, 'apply_after_step': 0}}, 'wandb': {'project': 'noposplat', 'entity': 'scene-representation-group', 'name': 're10k_hpc', 'mode': 'online', 'tags': ['re10k', '256x256']}, 'mode': 'train', 'data_loader': {'train': {'num_workers': 16, 'persistent_workers': True, 'batch_size': 16, 'seed': 1234}, 'test': {'num_workers': 4, 'persistent_workers': False, 'batch_size': 1, 'seed': 2345}, 'val': {'num_workers': 1, 'persistent_workers': True, 'batch_size': 1, 'seed': 3456}}, 'optimizer': {'lr': 0.0002, 'warm_up_steps': 125, 'backbone_lr_multiplier': 0.1, 'backbone_trainable': 'GB+H', 'accumulate': 1}, 'checkpointing': {'load': None, 'every_n_train_steps': 9375, 'save_top_k': 1, 'save_weights_only': True}, 'train': {'depth_mode': None, 'extended_visualization': False, 'print_log_every_n_steps': 10, 'distiller': '', 'distill_max_steps': 1000000, 'debug_mode': 0, 'camera_consis_loss': 0.0, 'camera_loss': 1.0, 'aux_loss': 0.0, 'one_sample_train': False, 'target_scale_mode': 2, 'extrinsic_drop': 0.1, 'intrinsic_drop': 0.1, 'align_corners': False, 'backbone_checkpoint': 'none', 'intrinsic_scaling': True, 'return_level_gs': True, 'train_context': False, 'use_pred_point_focal': False, 'scaling_detach': True, 'split_score_log_scale': True, 'split_score_grad_scale': 10000.0, 'split_loss': 0.01, 'random_split_strategy': 'score'}, 'test': {'output_path': 'test', 'align_pose': False, 'pose_align_steps': 100, 'rot_opt_lr': 0.005, 'trans_opt_lr': 0.005, 'compute_scores': True, 'save_image': False, 'save_video': False, 'save_compare': False, 'split_threshold_list': [0.0, 0.1, 0.2, 0.3, 0.4], 'pred_intrinsic': False}, 'seed': 111123, 'trainer': {'max_steps': 18751, 'val_check_interval': 500, 'gradient_clip_val': 0.5, 'num_nodes': 1}, 'dataset': {'re10k': {'make_baseline_1': True, 'relative_pose': True, 'augment': True, 'background_color': [0.0, 0.0, 0.0], 'overfit_to_scene': None, 'skip_bad_shape': True, 'view_sampler': {'name': 'bounded', 'num_target_views': 4, 'num_context_views': 2, 'min_distance_between_context_views': 45, 'max_distance_between_context_views': 90, 'min_distance_to_context_views': 0, 'warm_up_steps': 9375, 'initial_min_distance_between_context_views': 25, 'initial_max_distance_between_context_views': 25, 'distill_steps': 0}, 'name': 're10k', 'roots': ['datasets/re10k'], 'input_image_shape': [256, 256], 'original_image_shape': [360, 640], 'cameras_are_circular': False, 'baseline_min': 0.001, 'baseline_max': 10000000000.0, 'max_fov': 100.0}}, '_wandb': {}}
+2025-09-16 18:04:27,240 INFO    MainThread:7308 [wandb_init.py:init():809] starting backend
+2025-09-16 18:04:27,240 INFO    MainThread:7308 [wandb_init.py:init():813] sending inform_init request
+2025-09-16 18:04:27,259 INFO    MainThread:7308 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-09-16 18:04:27,260 INFO    MainThread:7308 [wandb_init.py:init():823] backend started and connected
+2025-09-16 18:04:27,278 INFO    MainThread:7308 [wandb_init.py:init():915] updated telemetry
+2025-09-16 18:04:27,323 INFO    MainThread:7308 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout
+2025-09-16 18:04:28,122 INFO    MainThread:7308 [wandb_init.py:init():1014] starting run threads in backend
+2025-09-16 18:04:28,297 INFO    MainThread:7308 [wandb_run.py:_console_start():2454] atexit reg
+2025-09-16 18:04:28,298 INFO    MainThread:7308 [wandb_run.py:_redirect():2306] redirect: wrap_raw
+2025-09-16 18:04:28,300 INFO    MainThread:7308 [wandb_run.py:_redirect():2371] Wrapping output streams.
+2025-09-16 18:04:28,300 INFO    MainThread:7308 [wandb_run.py:_redirect():2394] Redirects installed.
+2025-09-16 18:04:28,302 INFO    MainThread:7308 [wandb_init.py:init():1056] run started, returning control to user process

wandb/run-20250916_180450-bbofdbql/files/config.yaml

@@ -0,0 +1,531 @@
+_wandb:
+    value:
+        cli_version: 0.19.9
+        m:
+            - "1": lr-AdamW/pg2
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+            - "1": loss/camera
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/aux_0/lpips
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.width
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": activation_visualization.width
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": activation_visualization.filenames
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": attention_map.count
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/aux_2/mse
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/split_score
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": points_context_imgs._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": points_context_imgs.format
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": activation_visualization.format
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.height
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": points_context_imgs.height
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": lr-AdamW/pg2-momentum
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train/psnr_probabilistic
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": info/global_step
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": points_context_imgs.width
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": activation_visualization.captions
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": attention_map.filenames
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": attention_map.captions
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": attention_map._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": attention_map.height
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": attention_map.format
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.captions
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/3dgs_2_0/lpips
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/3dgs_2_0/mse
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/total
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/aux_0/mse
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": epoch
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.filenames
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": lr-AdamW/pg1-momentum
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/aux_1/lpips
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": points_context_imgs.captions
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": activation_visualization.height
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": points_context_imgs.filenames
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": points_context_imgs.count
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": activation_visualization.count
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": activation_visualization._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": attention_map.width
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": lr-AdamW/pg1
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.format
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.count
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/aux_2/lpips
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/aux_1/mse
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+        python_version: 3.12.11
+        t:
+            "1":
+                - 1
+                - 41
+                - 49
+                - 50
+                - 55
+                - 106
+            "2":
+                - 1
+                - 41
+                - 49
+                - 50
+                - 55
+                - 106
+            "3":
+                - 7
+                - 13
+                - 15
+                - 16
+                - 23
+                - 55
+                - 66
+            "4": 3.12.11
+            "5": 0.19.9
+            "8":
+                - 5
+            "12": 0.19.9
+            "13": linux-x86_64
+checkpointing:
+    value:
+        every_n_train_steps: 9375
+        load: null
+        save_top_k: 1
+        save_weights_only: true
+data_loader:
+    value:
+        test:
+            batch_size: 1
+            num_workers: 4
+            persistent_workers: false
+            seed: 2345
+        train:
+            batch_size: 16
+            num_workers: 16
+            persistent_workers: true
+            seed: 1234
+        val:
+            batch_size: 1
+            num_workers: 1
+            persistent_workers: true
+            seed: 3456
+dataset:
+    value:
+        re10k:
+            augment: true
+            background_color:
+                - 0
+                - 0
+                - 0
+            baseline_max: 1e+10
+            baseline_min: 0.001
+            cameras_are_circular: false
+            input_image_shape:
+                - 256
+                - 256
+            make_baseline_1: true
+            max_fov: 100
+            name: re10k
+            original_image_shape:
+                - 360
+                - 640
+            overfit_to_scene: null
+            relative_pose: true
+            roots:
+                - datasets/re10k
+            skip_bad_shape: true
+            view_sampler:
+                distill_steps: 0
+                initial_max_distance_between_context_views: 25
+                initial_min_distance_between_context_views: 25
+                max_distance_between_context_views: 90
+                min_distance_between_context_views: 45
+                min_distance_to_context_views: 0
+                name: bounded
+                num_context_views: 2
+                num_target_views: 4
+                warm_up_steps: 9375
+loss:
+    value:
+        lpips:
+            apply_after_step: 0
+            weight: 0.05
+        mse:
+            weight: 1
+mode:
+    value: train
+model:
+    value:
+        decoder:
+            background_color:
+                - 0
+                - 0
+                - 0
+            make_scale_invariant: false
+            name: splatting_cuda
+            use_gsplat: true
+        encoder:
+            apply_bounds_shim: true
+            backbone:
+                asymmetry_decoder: true
+                intrinsics_embed_degree: 4
+                intrinsics_embed_loc: encoder
+                intrinsics_embed_type: token
+                model: ViTLarge_BaseDecoder
+                name: croco
+                patch_embed_cls: PatchEmbedDust3R
+            d_feature: 128
+            gaussian_adapter:
+                gaussian_scale_max: 15
+                gaussian_scale_min: 0.5
+                sh_degree: 4
+            gaussians_per_pixel: 1
+            gs_params_head_type: dpt_gs
+            name: noposplat
+            num_monocular_samples: 32
+            num_surfaces: 1
+            opacity_mapping:
+                final: 0
+                initial: 0
+                warm_up: 1
+            pose_free: true
+            predict_opacity: false
+            pretrained_weights: ""
+            visualizer:
+                export_ply: false
+                min_resolution: 256
+                num_samples: 8
+        vggt:
+            camera_prompt: false
+            detr:
+                base_HW: 64
+                debug_mode: 0
+                dec_layers: 3
+                dec_n_points: 4
+                decoder_type: conv
+                dim_down_factor: 1
+                dim_feedforward_factor: 4
+                dropout: 0.1
+                enc_layers: 6
+                enc_n_points: 4
+                features_num_channel:
+                    - 128
+                    - 32
+                grad_mode: absgrad
+                hidden_dim: 32
+                name: deformable_detr
+                num_feature_levels: 2
+                num_level: 3
+                num_queries: 256
+                only_last_level: false
+                position_embedding: sine
+                pre_norm: false
+                same_split_head: false
+                self_layer_num: 1
+                split_threshold: 1
+                split_threshold_range:
+                    - 0
+                    - 2
+                use_mean_features: true
+            head_mode: pcd
+            input_image_shape:
+                - 518
+                - 518
+            num_visual_prompt: 0
+            same_head: false
+            use_extrinsic_token: false
+            use_intrinsic_token: false
+optimizer:
+    value:
+        accumulate: 1
+        backbone_lr_multiplier: 0.1
+        backbone_trainable: GB+H
+        lr: 0.0002
+        warm_up_steps: 125
+seed:
+    value: 111123
+test:
+    value:
+        align_pose: false
+        compute_scores: true
+        output_path: test
+        pose_align_steps: 100
+        pred_intrinsic: false
+        rot_opt_lr: 0.005
+        save_compare: false
+        save_image: false
+        save_video: false
+        split_threshold_list:
+            - 0
+            - 0.1
+            - 0.2
+            - 0.3
+            - 0.4
+        trans_opt_lr: 0.005
+train:
+    value:
+        align_corners: false
+        aux_loss: 0
+        backbone_checkpoint: none
+        camera_consis_loss: 0
+        camera_loss: 1
+        debug_mode: 0
+        depth_mode: null
+        distill_max_steps: 1000000
+        distiller: ""
+        extended_visualization: false
+        extrinsic_drop: 0.1
+        intrinsic_drop: 0.1
+        intrinsic_scaling: true
+        one_sample_train: false
+        print_log_every_n_steps: 10
+        random_split_strategy: score
+        return_level_gs: true
+        scaling_detach: true
+        split_loss: 0.01
+        split_score_grad_scale: 10000
+        split_score_log_scale: true
+        target_scale_mode: 2
+        train_context: false
+        use_pred_point_focal: false
+trainer:
+    value:
+        gradient_clip_val: 0.5
+        max_steps: 18751
+        num_nodes: 1
+        val_check_interval: 500
+wandb:
+    value:
+        entity: scene-representation-group
+        mode: online
+        name: re10k_hpc
+        project: noposplat
+        tags:
+            - re10k
+            - 256x256

wandb/run-20250916_180450-bbofdbql/files/output.log

@@ -0,0 +1,45 @@
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [1]
+
+  | Name    | Type                 | Params | Mode
+---------------------------------------------------------
+0 | vggt    | VGGT                 | 586 M  | train
+1 | decoder | DecoderSplattingCUDA | 0      | train
+2 | losses  | ModuleList           | 0      | train
+---------------------------------------------------------
+586 M     Trainable params
+0         Non-trainable params
+586 M     Total params
+2,345.910 Total estimated model params size (MB)
+813       Modules in train mode
+980       Modules in eval mode
+Sanity Checking: |                                                                                                                                                                                                    | 0/? [00:00<?, ?it/s][2025-09-16 18:04:52,934][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:425: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=279` in the `DataLoader` to improve performance.
+
+Validation epoch start on rank 0
+Sanity Checking DataLoader 0:   0%|                                                                                                                                                                                   | 0/1 [00:00<?, ?it/s]validation step 0; scene = ['306e2b7785657539']; context = [[48, 73]]
+target intrinsic:  tensor(0.8595, device='cuda:0') tensor(0.8597, device='cuda:0')
+pred intrinsic:  tensor(0.8780, device='cuda:0') tensor(0.8776, device='cuda:0')
+[2025-09-16 18:04:55,463][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/torch/utils/cpp_extension.py:2356: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation.
+If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'].
+  warnings.warn(
+
+[2025-09-16 18:04:55,703][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/utilities/data.py:79: Trying to infer the `batch_size` from an ambiguous collection. The batch size we found is 1. To avoid any miscalculations, use `self.log(..., batch_size=batch_size)`.
+
+Setting up [LPIPS] perceptual loss: trunk [vgg], v[0.1], spatial [off]
+[2025-09-16 18:04:55,705][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
+  warnings.warn(
+
+[2025-09-16 18:04:55,706][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
+  warnings.warn(msg)
+
+Loading model from: /opt/conda/envs/ours/lib/python3.12/site-packages/lpips/weights/v0.1/vgg.pth
+[2025-09-16 18:04:57,296][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/torch/functional.py:554: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at /pytorch/aten/src/ATen/native/TensorShape.cpp:4314.)
+  return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]
+
+Epoch 0: |                                                                                                                                                                                        | 9/? [00:26<00:00,  0.33it/s, v_num=dbql]train step 10; scene = ['78784c6ffe218407', '2144f1fd5b00dc66', 'f568ab2fa64994e9', 'eac6cedeba1f720b', '216d5454333d29b5', 'b628b7d46a07cb57', '3b07094586f67024', '044df93e59024ee2', '167c67586a8c0846', '3931c6a7e69ffff1', 'c9c67636b9d521be', '0908c09364f54c42', '2d92b7321e1039c5', '3bc0044f0678f39f', 'e51ef9945ae527c4', '31250ac58b3e32c9']; context = [[31, 56], [10, 35], [7, 32], [27, 52], [249, 274], [57, 82], [15, 40], [114, 139], [6, 31], [6, 31], [137, 162], [14, 39], [10, 35], [95, 120], [55, 80], [89, 114]]; loss = 0.774127
+Epoch 0: |                                                                                                                                                                                       | 19/? [00:56<00:00,  0.34it/s, v_num=dbql]train step 20; scene = ['4581bdd74b6ee0a9', 'd56e9c764f637e1c', '1f9574f1e0c1c2b2', 'd7e2a0e5718dc803', 'b2186a798ad4fb49', 'db1723893a3a6277', 'b3edac66be582f84', '6546295c3cec469a', 'b3232677872b2974', '2f27acaf69f3c7a7', '486673d483fe843a', '488358b53866b7aa', 'eedd397acee4cb4f', 'c18ea6a77949e219', '5140967fcd51092e', '08234c2181c4db46']; context = [[6, 31], [26, 51], [45, 70], [209, 234], [31, 56], [9, 34], [30, 55], [247, 272], [58, 83], [0, 25], [62, 87], [10, 35], [113, 138], [92, 117], [43, 68], [162, 187]]; loss = 0.659746
+Epoch 0: |                                                                                                                                                                                       | 29/? [01:24<00:00,  0.34it/s, v_num=dbql]train step 30; scene = ['6dd4b348223c880c', 'dff181c503e5c202', 'fd4c9fb0fae56ccd', '66c3771cf7cb96c2', '75790c62002de4f2', 'c7e94713ef1f1ace', '0cbf1454a4391523', '9bb0448e7cb2f733', 'a8cef6a851fbea3c', '9c763036f1d84e18', '7759eccd562145b9', 'a597e94e50eb917a', '67e69f3a4701dfd8', 'b91dfcb3102d1482', '5a2ad43377e9d18d', '0e4916077d990d1e']; context = [[4, 29], [102, 127], [26, 51], [247, 272], [39, 64], [16, 41], [129, 154], [10, 35], [38, 63], [101, 126], [8, 33], [111, 136], [10, 35], [76, 101], [52, 77], [21, 46]]; loss = 0.462477
+Epoch 0: |                                                                                                                                                                                       | 39/? [01:53<00:00,  0.34it/s, v_num=dbql]train step 40; scene = ['af9d978c9fca5752', '55cf2bbe9e017ea4', '437474fa4196e1b7', '0207b0ec0cc851f6', '1684dbfef3d182a9', '512cd4ea0fef0198', 'd1772c09b4b6d95f', '629f07baec1cbad5', '99e9f09b456ef23e', 'd066040e500bd58b', '7e7b53917bcf337b', '0066ed3711a7240d', '0f0bbb6510ca3018', 'c49bd62c183dd925', 'd7ca14c855d58875', 'b07a54eda9cdaccb']; context = [[175, 200], [3, 28], [122, 147], [24, 49], [17, 42], [35, 60], [53, 78], [110, 135], [31, 56], [13, 38], [54, 79], [71, 96], [37, 62], [11, 36], [45, 70], [106, 131]]; loss = 0.312660
+Epoch 0: |                                                                                                                                                                                       | 49/? [02:23<00:00,  0.34it/s, v_num=dbql]train step 50; scene = ['c1bcd66d5a2d51b6', 'ea5a39bd855fdd91', 'f3135a7492955a8a', '9794641b7e015578', '5041701491a4b930', 'c2fd6e5ebbff57b3', '604b0838aa3321c5', '86d9acf576d8ec0b', '2dd4d2a252e34405', 'fe1cf5504260d4ea', '7e58b6857e275547', '7dfeaf87a8145697', '96b163f47895beee', '83c9baf5bbc941ee', '23d17b97a9d3674e', '36846d6fa22bba62']; context = [[113, 138], [33, 58], [127, 152], [125, 150], [124, 149], [27, 52], [26, 51], [157, 182], [30, 55], [3, 28], [132, 157], [8, 33], [115, 140], [106, 131], [69, 94], [135, 160]]; loss = 0.248496
+Epoch 0: |                                                                                                                                                                                       | 55/? [02:40<00:00,  0.34it/s, v_num=dbql]
+
+Detected KeyboardInterrupt, attempting graceful shutdown ...

wandb/run-20250916_180450-bbofdbql/files/requirements.txt

@@ -0,0 +1,129 @@
+ruff==0.11.4
+PyYAML==6.0.2
+sympy==1.13.3
+fsspec==2024.6.1
+imageio-ffmpeg==0.6.0
+setuptools==78.1.1
+timm==1.0.15
+fonttools==4.57.0
+nvidia-cublas-cu12==12.8.3.14
+nvidia-cuda-nvrtc-cu12==12.8.61
+dacite==1.9.2
+hydra-core==1.3.2
+certifi==2022.12.7
+rich==14.0.0
+filelock==3.13.1
+idna==3.4
+nvidia-cuda-cupti-cu12==12.8.57
+pip==25.2
+wandb==0.19.9
+ninja==1.11.1.4
+networkx==3.3
+GitPython==3.1.44
+moviepy==1.0.3
+markdown-it-py==3.0.0
+lightning==2.5.1
+Jinja2==3.1.4
+nvidia-cusparse-cu12==12.5.7.53
+tabulate==0.9.0
+tifffile==2025.3.30
+nvidia-nvtx-cu12==12.8.55
+jaxtyping==0.2.37
+aiohappyeyeballs==2.6.1
+MarkupSafe==2.1.5
+pydantic_core==2.33.1
+mypy-extensions==1.0.0
+lazy_loader==0.4
+pydantic==2.11.2
+typing-inspection==0.4.0
+pillow==10.4.0
+nvidia-nvjitlink-cu12==12.8.61
+omegaconf==2.3.0
+scikit-image==0.25.2
+jaraco.context==5.3.0
+typing_extensions==4.12.2
+mpmath==1.3.0
+opencv-python==4.11.0.86
+backports.tarfile==1.2.0
+async-timeout==5.0.1
+opt_einsum==3.4.0
+nvidia-cuda-runtime-cu12==12.8.57
+numpy==1.26.4
+torch==2.7.1+cu128
+psutil==7.0.0
+pytorch-lightning==2.5.1
+protobuf==5.29.4
+imageio==2.37.0
+colorama==0.4.6
+attrs==25.3.0
+importlib_metadata==8.0.0
+frozenlist==1.5.0
+beartype==0.19.0
+propcache==0.3.1
+typeguard==4.3.0
+torchaudio==2.7.1+cu128
+decorator==4.4.2
+nvidia-cudnn-cu12==9.7.1.26
+torchvision==0.22.1+cu128
+sentry-sdk==2.25.1
+mdurl==0.1.2
+more-itertools==10.3.0
+einops==0.8.1
+zipp==3.19.2
+antlr4-python3-runtime==4.9.3
+cycler==0.12.1
+nvidia-cufile-cu12==1.13.0.11
+jaraco.collections==5.1.0
+jaraco.text==3.12.1
+torchmetrics==1.8.2
+packaging==24.2
+black==25.1.0
+setproctitle==1.3.5
+Pygments==2.19.2
+tomli==2.2.1
+six==1.17.0
+docker-pycreds==0.4.0
+inflect==7.3.1
+requests==2.28.1
+svg.py==1.6.0
+python-dateutil==2.9.0.post0
+proglog==0.1.11
+gsplat==1.5.3
+e3nn==0.5.6
+pathspec==0.12.1
+aiosignal==1.3.2
+wheel==0.45.1
+multidict==6.3.2
+pyparsing==3.2.3
+plyfile==1.1
+jaraco.functools==4.0.1
+nvidia-nccl-cu12==2.26.2
+triton==3.3.1
+nvidia-cusparselt-cu12==0.6.3
+scikit-video==1.1.11
+huggingface-hub==0.30.1
+lightning-utilities==0.14.3
+wadler_lindig==0.1.4
+nvidia-curand-cu12==10.3.9.55
+matplotlib==3.10.1
+kiwisolver==1.4.8
+tqdm==4.67.1
+safetensors==0.5.3
+scipy==1.15.2
+lpips==0.1.4
+nvidia-cusolver-cu12==11.7.2.55
+click==8.1.8
+charset-normalizer==2.1.1
+platformdirs==4.3.7
+opt-einsum-fx==0.1.4
+annotated-types==0.7.0
+smmap==5.0.2
+gitdb==4.0.12
+nvidia-cufft-cu12==11.3.3.41
+contourpy==1.3.1
+urllib3==1.26.13
+autocommand==2.2.2
+yarl==1.19.0
+aiohttp==3.11.16
+colorspacious==1.1.2
+python-dotenv==1.1.0

wandb/run-20250916_180450-bbofdbql/files/wandb-metadata.json

@@ -0,0 +1,79 @@
+{
+  "os": "Linux-5.15.0-153-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.12.11",
+  "startedAt": "2025-09-16T10:04:50.228862Z",
+  "args": [
+    "+experiment=re10k",
+    "wandb.mode=online",
+    "wandb.name=re10k_hpc"
+  ],
+  "program": "-m src.main",
+  "git": {
+    "remote": "git@github.com:K-nowing/ICLR2026_HCP.git",
+    "commit": "6c0080eae0295d43dd7f18fd71e742d924238f89"
+  },
+  "email": "dna9041@korea.ac.kr",
+  "root": "/root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc",
+  "host": "notebook-72e474dd-92e1-11f0-a6ea-7ab79f1c53cb-0",
+  "executable": "/opt/conda/envs/ours/bin/python",
+  "cpu_count": 144,
+  "cpu_count_logical": 288,
+  "gpu": "NVIDIA B200",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1081101176832",
+      "used": "211821682688"
+    }
+  },
+  "memory": {
+    "total": "2434690400256"
+  },
+  "cpu": {
+    "count": 144,
+    "countLogical": 288
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    }
+  ],
+  "cudaVersion": "12.8"
+}

wandb/run-20250916_180450-bbofdbql/files/wandb-summary.json

@@ -0,0 +1 @@
+{"loss/3dgs_2_0/mse":0.01890556514263153,"loss/aux_2/lpips":0.023699283599853516,"_step":5,"train/psnr_probabilistic":17.52679443359375,"points_context_imgs":{"format":"png","count":1,"filenames":["media/images/points_context_imgs_1_3431a5aef1476f516c25.png"],"captions":["306e2b7785657539"],"_type":"images/separated","width":536,"height":800},"loss/3dgs_2_0/lpips":0.023520031943917274,"loss/aux_0/lpips":0.036791641265153885,"_runtime":156.298987369,"loss/split_score":0.007056367117911577,"comparison":{"format":"png","count":1,"filenames":["media/images/comparison_0_271f44c5f8402c370b36.png"],"captions":["306e2b7785657539"],"_type":"images/separated","width":1064,"height":1098},"trainer/global_step":49,"loss/camera":0.0005199297447688878,"lr-AdamW/pg1-momentum":0.9,"loss/aux_1/mse":0.030908813700079918,"loss/total":0.24849636852741241,"loss/aux_1/lpips":0.03214797377586365,"lr-AdamW/pg2-momentum":0.9,"_timestamp":1.758017246526744e+09,"activation_visualization":{"filenames":["media/images/activation_visualization_2_1c027e12f9cd9c41c974.png"],"captions":["306e2b7785657539"],"_type":"images/separated","width":256,"height":256,"format":"png","count":1},"lr-AdamW/pg1":7.937280000000003e-05,"info/global_step":50,"loss/aux_0/mse":0.05457072705030441,"loss/aux_2/mse":0.020376034080982208,"lr-AdamW/pg2":7.937280000000003e-06,"epoch":0,"_wandb":{"runtime":179},"attention_map":{"format":"png","count":1,"filenames":["media/images/attention_map_3_26d8c2041170f5fec521.png"],"captions":["306e2b7785657539"],"_type":"images/separated","width":800,"height":536}}

wandb/run-20250916_180450-bbofdbql/logs/debug-core.log

@@ -0,0 +1,13 @@
+{"time":"2025-09-16T18:04:49.906817983+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpdi5btku6/port-7974.txt","pid":7974,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-09-16T18:04:49.91264959+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":46535,"Zone":""}}
+{"time":"2025-09-16T18:04:49.912897107+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":7974}
+{"time":"2025-09-16T18:04:49.979818591+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:34414"}
+{"time":"2025-09-16T18:04:50.23218552+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"bbofdbql","id":"127.0.0.1:34414"}
+{"time":"2025-09-16T18:04:50.79838477+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"bbofdbql","id":"127.0.0.1:34414"}
+{"time":"2025-09-16T18:07:49.556125089+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:34414"}
+{"time":"2025-09-16T18:07:49.556534164+08:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:34414"}
+{"time":"2025-09-16T18:07:49.556672562+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-16T18:07:49.55682568+08:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:34414"}
+{"time":"2025-09-16T18:07:50.12178276+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:34414"}
+{"time":"2025-09-16T18:07:50.121867559+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:34414"}
+{"time":"2025-09-16T18:07:50.121927478+08:00","level":"INFO","msg":"server is closed"}

wandb/run-20250916_180450-bbofdbql/logs/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2025-09-16T18:04:50.23380822+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc/wandb/run-20250916_180450-bbofdbql/logs/debug-core.log"}
+{"time":"2025-09-16T18:04:50.798198612+08:00","level":"INFO","msg":"created new stream","id":"bbofdbql"}
+{"time":"2025-09-16T18:04:50.798355611+08:00","level":"INFO","msg":"stream: started","id":"bbofdbql"}
+{"time":"2025-09-16T18:04:50.798610927+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"bbofdbql"}
+{"time":"2025-09-16T18:04:50.798856364+08:00","level":"INFO","msg":"sender: started","stream_id":"bbofdbql"}
+{"time":"2025-09-16T18:04:50.799137521+08:00","level":"INFO","msg":"handler: started","stream_id":"bbofdbql"}
+{"time":"2025-09-16T18:04:51.373683227+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-09-16T18:07:49.556476444+08:00","level":"INFO","msg":"stream: closing","id":"bbofdbql"}
+{"time":"2025-09-16T18:07:49.556603443+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-09-16T18:07:49.556883199+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-09-16T18:07:49.900010843+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-09-16T18:07:50.121008889+08:00","level":"INFO","msg":"handler: closed","stream_id":"bbofdbql"}
+{"time":"2025-09-16T18:07:50.121179847+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"bbofdbql"}
+{"time":"2025-09-16T18:07:50.121222647+08:00","level":"INFO","msg":"sender: closed","stream_id":"bbofdbql"}
+{"time":"2025-09-16T18:07:50.121490203+08:00","level":"INFO","msg":"stream: closed","id":"bbofdbql"}

wandb/run-20250916_180450-bbofdbql/logs/debug.log

@@ -0,0 +1,85 @@
+2025-09-16 18:04:50,204 INFO    MainThread:7974 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9
+2025-09-16 18:04:50,204 INFO    MainThread:7974 [wandb_setup.py:_flush():67] Configure stats pid to 7974
+2025-09-16 18:04:50,204 INFO    MainThread:7974 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings
+2025-09-16 18:04:50,204 INFO    MainThread:7974 [wandb_setup.py:_flush():67] Loading settings from /root/highspeedstorage/injae/code/ICLR2026_HCP/wandb/settings
+2025-09-16 18:04:50,204 INFO    MainThread:7974 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-09-16 18:04:50,205 INFO    MainThread:7974 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc/wandb/run-20250916_180450-bbofdbql/logs/debug.log
+2025-09-16 18:04:50,205 INFO    MainThread:7974 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc/wandb/run-20250916_180450-bbofdbql/logs/debug-internal.log
+2025-09-16 18:04:50,205 INFO    MainThread:7974 [wandb_init.py:init():781] calling init triggers
+2025-09-16 18:04:50,206 INFO    MainThread:7974 [wandb_init.py:init():786] wandb.init called with sweep_config: {}
+config: {'model': {'encoder': {'backbone': {'name': 'croco', 'model': 'ViTLarge_BaseDecoder', 'patch_embed_cls': 'PatchEmbedDust3R', 'asymmetry_decoder': True, 'intrinsics_embed_loc': 'encoder', 'intrinsics_embed_degree': 4, 'intrinsics_embed_type': 'token'}, 'name': 'noposplat', 'opacity_mapping': {'initial': 0.0, 'final': 0.0, 'warm_up': 1}, 'num_monocular_samples': 32, 'num_surfaces': 1, 'predict_opacity': False, 'gaussians_per_pixel': 1, 'gaussian_adapter': {'gaussian_scale_min': 0.5, 'gaussian_scale_max': 15.0, 'sh_degree': 4}, 'd_feature': 128, 'visualizer': {'num_samples': 8, 'min_resolution': 256, 'export_ply': False}, 'apply_bounds_shim': True, 'gs_params_head_type': 'dpt_gs', 'pose_free': True, 'pretrained_weights': ''}, 'decoder': {'name': 'splatting_cuda', 'background_color': [0.0, 0.0, 0.0], 'make_scale_invariant': False, 'use_gsplat': True}, 'vggt': {'input_image_shape': [518, 518], 'head_mode': 'pcd', 'same_head': False, 'camera_prompt': False, 'num_visual_prompt': 0, 'use_extrinsic_token': False, 'use_intrinsic_token': False, 'detr': {'name': 'deformable_detr', 'features_num_channel': [128, 32], 'hidden_dim': 32, 'dim_down_factor': 1, 'dim_feedforward_factor': 4, 'enc_layers': 6, 'dec_layers': 3, 'dropout': 0.1, 'num_feature_levels': 2, 'dec_n_points': 4, 'enc_n_points': 4, 'split_threshold': 1.0, 'num_queries': 256, 'base_HW': 64, 'position_embedding': 'sine', 'pre_norm': False, 'self_layer_num': 1, 'num_level': 3, 'debug_mode': 0, 'decoder_type': 'conv', 'split_threshold_range': [0.0, 2.0], 'only_last_level': False, 'grad_mode': 'absgrad', 'same_split_head': False, 'use_mean_features': True}}}, 'loss': {'mse': {'weight': 1.0}, 'lpips': {'weight': 0.05, 'apply_after_step': 0}}, 'wandb': {'project': 'noposplat', 'entity': 'scene-representation-group', 'name': 're10k_hpc', 'mode': 'online', 'tags': ['re10k', '256x256']}, 'mode': 'train', 'data_loader': {'train': {'num_workers': 16, 'persistent_workers': True, 'batch_size': 16, 'seed': 1234}, 'test': {'num_workers': 4, 'persistent_workers': False, 'batch_size': 1, 'seed': 2345}, 'val': {'num_workers': 1, 'persistent_workers': True, 'batch_size': 1, 'seed': 3456}}, 'optimizer': {'lr': 0.0002, 'warm_up_steps': 125, 'backbone_lr_multiplier': 0.1, 'backbone_trainable': 'GB+H', 'accumulate': 1}, 'checkpointing': {'load': None, 'every_n_train_steps': 9375, 'save_top_k': 1, 'save_weights_only': True}, 'train': {'depth_mode': None, 'extended_visualization': False, 'print_log_every_n_steps': 10, 'distiller': '', 'distill_max_steps': 1000000, 'debug_mode': 0, 'camera_consis_loss': 0.0, 'camera_loss': 1.0, 'aux_loss': 0.0, 'one_sample_train': False, 'target_scale_mode': 2, 'extrinsic_drop': 0.1, 'intrinsic_drop': 0.1, 'align_corners': False, 'backbone_checkpoint': 'none', 'intrinsic_scaling': True, 'return_level_gs': True, 'train_context': False, 'use_pred_point_focal': False, 'scaling_detach': True, 'split_score_log_scale': True, 'split_score_grad_scale': 10000.0, 'split_loss': 0.01, 'random_split_strategy': 'score'}, 'test': {'output_path': 'test', 'align_pose': False, 'pose_align_steps': 100, 'rot_opt_lr': 0.005, 'trans_opt_lr': 0.005, 'compute_scores': True, 'save_image': False, 'save_video': False, 'save_compare': False, 'split_threshold_list': [0.0, 0.1, 0.2, 0.3, 0.4], 'pred_intrinsic': False}, 'seed': 111123, 'trainer': {'max_steps': 18751, 'val_check_interval': 500, 'gradient_clip_val': 0.5, 'num_nodes': 1}, 'dataset': {'re10k': {'make_baseline_1': True, 'relative_pose': True, 'augment': True, 'background_color': [0.0, 0.0, 0.0], 'overfit_to_scene': None, 'skip_bad_shape': True, 'view_sampler': {'name': 'bounded', 'num_target_views': 4, 'num_context_views': 2, 'min_distance_between_context_views': 45, 'max_distance_between_context_views': 90, 'min_distance_to_context_views': 0, 'warm_up_steps': 9375, 'initial_min_distance_between_context_views': 25, 'initial_max_distance_between_context_views': 25, 'distill_steps': 0}, 'name': 're10k', 'roots': ['datasets/re10k'], 'input_image_shape': [256, 256], 'original_image_shape': [360, 640], 'cameras_are_circular': False, 'baseline_min': 0.001, 'baseline_max': 10000000000.0, 'max_fov': 100.0}}, '_wandb': {}}
+2025-09-16 18:04:50,206 INFO    MainThread:7974 [wandb_init.py:init():809] starting backend
+2025-09-16 18:04:50,206 INFO    MainThread:7974 [wandb_init.py:init():813] sending inform_init request
+2025-09-16 18:04:50,227 INFO    MainThread:7974 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-09-16 18:04:50,227 INFO    MainThread:7974 [wandb_init.py:init():823] backend started and connected
+2025-09-16 18:04:50,245 INFO    MainThread:7974 [wandb_init.py:init():915] updated telemetry
+2025-09-16 18:04:50,293 INFO    MainThread:7974 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout
+2025-09-16 18:04:51,320 INFO    MainThread:7974 [wandb_init.py:init():1014] starting run threads in backend
+2025-09-16 18:04:51,534 INFO    MainThread:7974 [wandb_run.py:_console_start():2454] atexit reg
+2025-09-16 18:04:51,535 INFO    MainThread:7974 [wandb_run.py:_redirect():2306] redirect: wrap_raw
+2025-09-16 18:04:51,535 INFO    MainThread:7974 [wandb_run.py:_redirect():2371] Wrapping output streams.
+2025-09-16 18:04:51,535 INFO    MainThread:7974 [wandb_run.py:_redirect():2394] Redirects installed.
+2025-09-16 18:04:51,538 INFO    MainThread:7974 [wandb_init.py:init():1056] run started, returning control to user process
+2025-09-16 18:07:49,554 INFO    MsgRouterThr:7974 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.
+2025-09-16 18:07:50,602 ERROR   MainThread:7974 [redirect.py:_on_write():661] [no run ID] error in stdout callback
+Traceback (most recent call last):
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write
+    cb(written_data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 2375, in <lambda>
+    lambda data: self._console_raw_callback("stdout", data),
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback
+    self._backend.interface.publish_output_raw(name, data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw
+    self._publish_output_raw(o)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
+    self._publish(rec)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
+    self._sock_client.send_record_publish(record)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
+    self.send_server_request(server_req)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
+    self._send_message(msg)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
+    self._sendall_with_error_handle(header + data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
+    sent = self._sock.send(data)
+           ^^^^^^^^^^^^^^^^^^^^^
+BrokenPipeError: [Errno 32] Broken pipe
+2025-09-16 18:07:50,610 ERROR   MainThread:7974 [redirect.py:_on_write():661] [no run ID] error in stdout callback
+Traceback (most recent call last):
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write
+    cb(written_data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 2375, in <lambda>
+    lambda data: self._console_raw_callback("stdout", data),
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback
+    self._backend.interface.publish_output_raw(name, data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw
+    self._publish_output_raw(o)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
+    self._publish(rec)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
+    self._sock_client.send_record_publish(record)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
+    self.send_server_request(server_req)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
+    self._send_message(msg)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
+    self._sendall_with_error_handle(header + data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
+    sent = self._sock.send(data)
+           ^^^^^^^^^^^^^^^^^^^^^
+BrokenPipeError: [Errno 32] Broken pipe

wandb/run-20250916_180500-j938t4jj/files/config.yaml

@@ -0,0 +1,531 @@
+_wandb:
+    value:
+        cli_version: 0.19.9
+        m:
+            - "1": loss/aux_1/mse
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+            - "1": info/global_step
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/camera
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/aux_0/lpips
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.format
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": points_context_imgs.captions
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": points_context_imgs.width
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": attention_map._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/split_score
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/aux_1/lpips
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.captions
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": points_context_imgs.count
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": attention_map.captions
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/aux_2/mse
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": activation_visualization.height
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": attention_map.format
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": attention_map.count
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/aux_2/lpips
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": points_context_imgs.format
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": activation_visualization.format
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": activation_visualization.count
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": activation_visualization._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": epoch
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/total
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/aux_0/mse
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": lr-AdamW/pg2
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/3dgs_2_0/lpips
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": loss/3dgs_2_0/mse
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train/psnr_probabilistic
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.filenames
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": points_context_imgs.filenames
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": activation_visualization.captions
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": attention_map.height
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": lr-AdamW/pg1
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": activation_visualization.filenames
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": activation_visualization.width
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": lr-AdamW/pg1-momentum
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.count
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": attention_map.filenames
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": attention_map.width
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.width
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": points_context_imgs.height
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": lr-AdamW/pg2-momentum
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.height
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": points_context_imgs._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+        python_version: 3.12.11
+        t:
+            "1":
+                - 1
+                - 41
+                - 49
+                - 50
+                - 55
+                - 106
+            "2":
+                - 1
+                - 41
+                - 49
+                - 50
+                - 55
+                - 106
+            "3":
+                - 7
+                - 13
+                - 15
+                - 16
+                - 23
+                - 55
+                - 66
+            "4": 3.12.11
+            "5": 0.19.9
+            "8":
+                - 5
+            "12": 0.19.9
+            "13": linux-x86_64
+checkpointing:
+    value:
+        every_n_train_steps: 9375
+        load: null
+        save_top_k: 1
+        save_weights_only: true
+data_loader:
+    value:
+        test:
+            batch_size: 1
+            num_workers: 4
+            persistent_workers: false
+            seed: 2345
+        train:
+            batch_size: 16
+            num_workers: 16
+            persistent_workers: true
+            seed: 1234
+        val:
+            batch_size: 1
+            num_workers: 1
+            persistent_workers: true
+            seed: 3456
+dataset:
+    value:
+        re10k:
+            augment: true
+            background_color:
+                - 0
+                - 0
+                - 0
+            baseline_max: 1e+10
+            baseline_min: 0.001
+            cameras_are_circular: false
+            input_image_shape:
+                - 256
+                - 256
+            make_baseline_1: true
+            max_fov: 100
+            name: re10k
+            original_image_shape:
+                - 360
+                - 640
+            overfit_to_scene: null
+            relative_pose: true
+            roots:
+                - datasets/re10k
+            skip_bad_shape: true
+            view_sampler:
+                distill_steps: 0
+                initial_max_distance_between_context_views: 25
+                initial_min_distance_between_context_views: 25
+                max_distance_between_context_views: 90
+                min_distance_between_context_views: 45
+                min_distance_to_context_views: 0
+                name: bounded
+                num_context_views: 2
+                num_target_views: 4
+                warm_up_steps: 9375
+loss:
+    value:
+        lpips:
+            apply_after_step: 0
+            weight: 0.05
+        mse:
+            weight: 1
+mode:
+    value: train
+model:
+    value:
+        decoder:
+            background_color:
+                - 0
+                - 0
+                - 0
+            make_scale_invariant: false
+            name: splatting_cuda
+            use_gsplat: true
+        encoder:
+            apply_bounds_shim: true
+            backbone:
+                asymmetry_decoder: true
+                intrinsics_embed_degree: 4
+                intrinsics_embed_loc: encoder
+                intrinsics_embed_type: token
+                model: ViTLarge_BaseDecoder
+                name: croco
+                patch_embed_cls: PatchEmbedDust3R
+            d_feature: 128
+            gaussian_adapter:
+                gaussian_scale_max: 15
+                gaussian_scale_min: 0.5
+                sh_degree: 4
+            gaussians_per_pixel: 1
+            gs_params_head_type: dpt_gs
+            name: noposplat
+            num_monocular_samples: 32
+            num_surfaces: 1
+            opacity_mapping:
+                final: 0
+                initial: 0
+                warm_up: 1
+            pose_free: true
+            predict_opacity: false
+            pretrained_weights: ""
+            visualizer:
+                export_ply: false
+                min_resolution: 256
+                num_samples: 8
+        vggt:
+            camera_prompt: false
+            detr:
+                base_HW: 64
+                debug_mode: 0
+                dec_layers: 1
+                dec_n_points: 4
+                decoder_type: conv
+                dim_down_factor: 1
+                dim_feedforward_factor: 4
+                dropout: 0.1
+                enc_layers: 6
+                enc_n_points: 4
+                features_num_channel:
+                    - 128
+                    - 32
+                grad_mode: absgrad
+                hidden_dim: 32
+                name: deformable_detr
+                num_feature_levels: 2
+                num_level: 3
+                num_queries: 256
+                only_last_level: false
+                position_embedding: sine
+                pre_norm: false
+                same_split_head: false
+                self_layer_num: 1
+                split_threshold: 1
+                split_threshold_range:
+                    - 0
+                    - 2
+                use_mean_features: true
+            head_mode: pcd
+            input_image_shape:
+                - 518
+                - 518
+            num_visual_prompt: 0
+            same_head: false
+            use_extrinsic_token: false
+            use_intrinsic_token: false
+optimizer:
+    value:
+        accumulate: 1
+        backbone_lr_multiplier: 0.1
+        backbone_trainable: GB+H
+        lr: 0.0002
+        warm_up_steps: 125
+seed:
+    value: 111123
+test:
+    value:
+        align_pose: false
+        compute_scores: true
+        output_path: test
+        pose_align_steps: 100
+        pred_intrinsic: false
+        rot_opt_lr: 0.005
+        save_compare: false
+        save_image: false
+        save_video: false
+        split_threshold_list:
+            - 0
+            - 0.1
+            - 0.2
+            - 0.3
+            - 0.4
+        trans_opt_lr: 0.005
+train:
+    value:
+        align_corners: false
+        aux_loss: 0
+        backbone_checkpoint: none
+        camera_consis_loss: 0
+        camera_loss: 1
+        debug_mode: 0
+        depth_mode: null
+        distill_max_steps: 1000000
+        distiller: ""
+        extended_visualization: false
+        extrinsic_drop: 0.1
+        intrinsic_drop: 0.1
+        intrinsic_scaling: true
+        one_sample_train: false
+        print_log_every_n_steps: 10
+        random_split_strategy: score
+        return_level_gs: true
+        scaling_detach: true
+        split_loss: 0.01
+        split_score_grad_scale: 10000
+        split_score_log_scale: true
+        target_scale_mode: 2
+        train_context: false
+        use_pred_point_focal: false
+trainer:
+    value:
+        gradient_clip_val: 0.5
+        max_steps: 18751
+        num_nodes: 1
+        val_check_interval: 500
+wandb:
+    value:
+        entity: scene-representation-group
+        mode: online
+        name: re10k_hpc
+        project: noposplat
+        tags:
+            - re10k
+            - 256x256

wandb/run-20250916_180500-j938t4jj/files/output.log

@@ -0,0 +1,45 @@
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [2]
+
+  | Name    | Type                 | Params | Mode
+---------------------------------------------------------
+0 | vggt    | VGGT                 | 585 M  | train
+1 | decoder | DecoderSplattingCUDA | 0      | train
+2 | losses  | ModuleList           | 0      | train
+---------------------------------------------------------
+585 M     Trainable params
+0         Non-trainable params
+585 M     Total params
+2,340.472 Total estimated model params size (MB)
+773       Modules in train mode
+980       Modules in eval mode
+Sanity Checking: |                                                                                                                                                                                                    | 0/? [00:00<?, ?it/s][2025-09-16 18:05:03,198][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:425: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=279` in the `DataLoader` to improve performance.
+
+Validation epoch start on rank 0
+Sanity Checking DataLoader 0:   0%|                                                                                                                                                                                   | 0/1 [00:00<?, ?it/s]validation step 0; scene = ['306e2b7785657539']; context = [[48, 73]]
+target intrinsic:  tensor(0.8595, device='cuda:0') tensor(0.8597, device='cuda:0')
+pred intrinsic:  tensor(0.8780, device='cuda:0') tensor(0.8776, device='cuda:0')
+[2025-09-16 18:05:05,392][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/torch/utils/cpp_extension.py:2356: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation.
+If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'].
+  warnings.warn(
+
+[2025-09-16 18:05:05,581][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/utilities/data.py:79: Trying to infer the `batch_size` from an ambiguous collection. The batch size we found is 1. To avoid any miscalculations, use `self.log(..., batch_size=batch_size)`.
+
+Setting up [LPIPS] perceptual loss: trunk [vgg], v[0.1], spatial [off]
+[2025-09-16 18:05:05,583][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
+  warnings.warn(
+
+[2025-09-16 18:05:05,583][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
+  warnings.warn(msg)
+
+Loading model from: /opt/conda/envs/ours/lib/python3.12/site-packages/lpips/weights/v0.1/vgg.pth
+[2025-09-16 18:05:07,261][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/torch/functional.py:554: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at /pytorch/aten/src/ATen/native/TensorShape.cpp:4314.)
+  return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]
+
+Epoch 0: |                                                                                                                                                                                        | 9/? [00:26<00:00,  0.35it/s, v_num=t4jj]train step 10; scene = ['78784c6ffe218407', '2144f1fd5b00dc66', 'f568ab2fa64994e9', 'eac6cedeba1f720b', '216d5454333d29b5', 'b628b7d46a07cb57', '3b07094586f67024', '044df93e59024ee2', '167c67586a8c0846', '3931c6a7e69ffff1', 'c9c67636b9d521be', '0908c09364f54c42', '2d92b7321e1039c5', '3bc0044f0678f39f', 'e51ef9945ae527c4', '31250ac58b3e32c9']; context = [[31, 56], [10, 35], [7, 32], [27, 52], [249, 274], [57, 82], [15, 40], [114, 139], [6, 31], [6, 31], [137, 162], [14, 39], [10, 35], [95, 120], [55, 80], [89, 114]]; loss = 0.770314
+Epoch 0: |                                                                                                                                                                                       | 19/? [00:54<00:00,  0.35it/s, v_num=t4jj]train step 20; scene = ['4581bdd74b6ee0a9', 'd56e9c764f637e1c', '1f9574f1e0c1c2b2', 'd7e2a0e5718dc803', 'b2186a798ad4fb49', 'db1723893a3a6277', 'b3edac66be582f84', '6546295c3cec469a', 'b3232677872b2974', '2f27acaf69f3c7a7', '486673d483fe843a', '488358b53866b7aa', 'eedd397acee4cb4f', 'c18ea6a77949e219', '5140967fcd51092e', '08234c2181c4db46']; context = [[6, 31], [26, 51], [45, 70], [209, 234], [31, 56], [9, 34], [30, 55], [247, 272], [58, 83], [0, 25], [62, 87], [10, 35], [113, 138], [92, 117], [43, 68], [162, 187]]; loss = 0.672848
+Epoch 0: |                                                                                                                                                                                       | 29/? [01:21<00:00,  0.35it/s, v_num=t4jj]train step 30; scene = ['6dd4b348223c880c', 'dff181c503e5c202', 'fd4c9fb0fae56ccd', '66c3771cf7cb96c2', '75790c62002de4f2', 'c7e94713ef1f1ace', '0cbf1454a4391523', '9bb0448e7cb2f733', 'a8cef6a851fbea3c', '9c763036f1d84e18', '7759eccd562145b9', 'a597e94e50eb917a', '67e69f3a4701dfd8', 'b91dfcb3102d1482', '5a2ad43377e9d18d', '0e4916077d990d1e']; context = [[4, 29], [102, 127], [26, 51], [247, 272], [39, 64], [16, 41], [129, 154], [10, 35], [38, 63], [101, 126], [8, 33], [111, 136], [10, 35], [76, 101], [52, 77], [21, 46]]; loss = 0.513662
+Epoch 0: |                                                                                                                                                                                       | 39/? [01:49<00:00,  0.36it/s, v_num=t4jj]train step 40; scene = ['af9d978c9fca5752', '55cf2bbe9e017ea4', '437474fa4196e1b7', '0207b0ec0cc851f6', '1684dbfef3d182a9', '512cd4ea0fef0198', 'd1772c09b4b6d95f', '629f07baec1cbad5', '99e9f09b456ef23e', 'd066040e500bd58b', '7e7b53917bcf337b', '0066ed3711a7240d', '0f0bbb6510ca3018', 'c49bd62c183dd925', 'd7ca14c855d58875', 'b07a54eda9cdaccb']; context = [[175, 200], [3, 28], [122, 147], [24, 49], [17, 42], [35, 60], [53, 78], [110, 135], [31, 56], [13, 38], [54, 79], [71, 96], [37, 62], [11, 36], [45, 70], [106, 131]]; loss = 0.303013
+Epoch 0: |                                                                                                                                                                                       | 49/? [02:17<00:00,  0.36it/s, v_num=t4jj]train step 50; scene = ['c1bcd66d5a2d51b6', 'ea5a39bd855fdd91', 'f3135a7492955a8a', '9794641b7e015578', '5041701491a4b930', 'c2fd6e5ebbff57b3', '604b0838aa3321c5', '86d9acf576d8ec0b', '2dd4d2a252e34405', 'fe1cf5504260d4ea', '7e58b6857e275547', '7dfeaf87a8145697', '96b163f47895beee', '83c9baf5bbc941ee', '23d17b97a9d3674e', '36846d6fa22bba62']; context = [[113, 138], [33, 58], [127, 152], [125, 150], [124, 149], [27, 52], [26, 51], [157, 182], [30, 55], [3, 28], [132, 157], [8, 33], [115, 140], [106, 131], [69, 94], [135, 160]]; loss = 0.345490
+Epoch 0: |                                                                                                                                                                                       | 53/? [02:28<00:00,  0.36it/s, v_num=t4jj]
+
+Detected KeyboardInterrupt, attempting graceful shutdown ...

wandb/run-20250916_180500-j938t4jj/files/requirements.txt

@@ -0,0 +1,129 @@
+ruff==0.11.4
+PyYAML==6.0.2
+sympy==1.13.3
+fsspec==2024.6.1
+imageio-ffmpeg==0.6.0
+setuptools==78.1.1
+timm==1.0.15
+fonttools==4.57.0
+nvidia-cublas-cu12==12.8.3.14
+nvidia-cuda-nvrtc-cu12==12.8.61
+dacite==1.9.2
+hydra-core==1.3.2
+certifi==2022.12.7
+rich==14.0.0
+filelock==3.13.1
+idna==3.4
+nvidia-cuda-cupti-cu12==12.8.57
+pip==25.2
+wandb==0.19.9
+ninja==1.11.1.4
+networkx==3.3
+GitPython==3.1.44
+moviepy==1.0.3
+markdown-it-py==3.0.0
+lightning==2.5.1
+Jinja2==3.1.4
+nvidia-cusparse-cu12==12.5.7.53
+tabulate==0.9.0
+tifffile==2025.3.30
+nvidia-nvtx-cu12==12.8.55
+jaxtyping==0.2.37
+aiohappyeyeballs==2.6.1
+MarkupSafe==2.1.5
+pydantic_core==2.33.1
+mypy-extensions==1.0.0
+lazy_loader==0.4
+pydantic==2.11.2
+typing-inspection==0.4.0
+pillow==10.4.0
+nvidia-nvjitlink-cu12==12.8.61
+omegaconf==2.3.0
+scikit-image==0.25.2
+jaraco.context==5.3.0
+typing_extensions==4.12.2
+mpmath==1.3.0
+opencv-python==4.11.0.86
+backports.tarfile==1.2.0
+async-timeout==5.0.1
+opt_einsum==3.4.0
+nvidia-cuda-runtime-cu12==12.8.57
+numpy==1.26.4
+torch==2.7.1+cu128
+psutil==7.0.0
+pytorch-lightning==2.5.1
+protobuf==5.29.4
+imageio==2.37.0
+colorama==0.4.6
+attrs==25.3.0
+importlib_metadata==8.0.0
+frozenlist==1.5.0
+beartype==0.19.0
+propcache==0.3.1
+typeguard==4.3.0
+torchaudio==2.7.1+cu128
+decorator==4.4.2
+nvidia-cudnn-cu12==9.7.1.26
+torchvision==0.22.1+cu128
+sentry-sdk==2.25.1
+mdurl==0.1.2
+more-itertools==10.3.0
+einops==0.8.1
+zipp==3.19.2
+antlr4-python3-runtime==4.9.3
+cycler==0.12.1
+nvidia-cufile-cu12==1.13.0.11
+jaraco.collections==5.1.0
+jaraco.text==3.12.1
+torchmetrics==1.8.2
+packaging==24.2
+black==25.1.0
+setproctitle==1.3.5
+Pygments==2.19.2
+tomli==2.2.1
+six==1.17.0
+docker-pycreds==0.4.0
+inflect==7.3.1
+requests==2.28.1
+svg.py==1.6.0
+python-dateutil==2.9.0.post0
+proglog==0.1.11
+gsplat==1.5.3
+e3nn==0.5.6
+pathspec==0.12.1
+aiosignal==1.3.2
+wheel==0.45.1
+multidict==6.3.2
+pyparsing==3.2.3
+plyfile==1.1
+jaraco.functools==4.0.1
+nvidia-nccl-cu12==2.26.2
+triton==3.3.1
+nvidia-cusparselt-cu12==0.6.3
+scikit-video==1.1.11
+huggingface-hub==0.30.1
+lightning-utilities==0.14.3
+wadler_lindig==0.1.4
+nvidia-curand-cu12==10.3.9.55
+matplotlib==3.10.1
+kiwisolver==1.4.8
+tqdm==4.67.1
+safetensors==0.5.3
+scipy==1.15.2
+lpips==0.1.4
+nvidia-cusolver-cu12==11.7.2.55
+click==8.1.8
+charset-normalizer==2.1.1
+platformdirs==4.3.7
+opt-einsum-fx==0.1.4
+annotated-types==0.7.0
+smmap==5.0.2
+gitdb==4.0.12
+nvidia-cufft-cu12==11.3.3.41
+contourpy==1.3.1
+urllib3==1.26.13
+autocommand==2.2.2
+yarl==1.19.0
+aiohttp==3.11.16
+colorspacious==1.1.2
+python-dotenv==1.1.0

wandb/run-20250916_180500-j938t4jj/files/wandb-metadata.json

@@ -0,0 +1,79 @@
+{
+  "os": "Linux-5.15.0-153-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.12.11",
+  "startedAt": "2025-09-16T10:05:00.996990Z",
+  "args": [
+    "+experiment=re10k",
+    "wandb.mode=online",
+    "wandb.name=re10k_hpc"
+  ],
+  "program": "-m src.main",
+  "git": {
+    "remote": "git@github.com:K-nowing/ICLR2026_HCP.git",
+    "commit": "6c0080eae0295d43dd7f18fd71e742d924238f89"
+  },
+  "email": "dna9041@korea.ac.kr",
+  "root": "/root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc",
+  "host": "notebook-72e474dd-92e1-11f0-a6ea-7ab79f1c53cb-0",
+  "executable": "/opt/conda/envs/ours/bin/python",
+  "cpu_count": 144,
+  "cpu_count_logical": 288,
+  "gpu": "NVIDIA B200",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1081101176832",
+      "used": "211821752320"
+    }
+  },
+  "memory": {
+    "total": "2434690400256"
+  },
+  "cpu": {
+    "count": 144,
+    "countLogical": 288
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    }
+  ],
+  "cudaVersion": "12.8"
+}

wandb/run-20250916_180500-j938t4jj/files/wandb-summary.json

@@ -0,0 +1 @@
+{"comparison":{"captions":["306e2b7785657539"],"_type":"images/separated","width":1064,"height":1098,"format":"png","count":1,"filenames":["media/images/comparison_0_621c28fb8c1f6ff0dbb4.png"]},"lr-AdamW/pg1":7.937280000000003e-05,"loss/aux_2/mse":0.04114922136068344,"points_context_imgs":{"format":"png","count":1,"filenames":["media/images/points_context_imgs_1_cbb5017831dfea54f21d.png"],"captions":["306e2b7785657539"],"_type":"images/separated","width":536,"height":800},"epoch":0,"_runtime":149.388106731,"loss/aux_1/lpips":0.03612890467047691,"loss/total":0.3454895317554474,"lr-AdamW/pg2-momentum":0.9,"activation_visualization":{"height":256,"format":"png","count":1,"filenames":["media/images/activation_visualization_2_e7083dae4b9eda33b264.png"],"captions":["306e2b7785657539"],"_type":"images/separated","width":256},"loss/aux_2/lpips":0.029589688405394554,"loss/aux_0/mse":0.07302524149417877,"loss/aux_0/lpips":0.03862287849187851,"loss/3dgs_2_0/lpips":0.029533980414271355,"trainer/global_step":49,"lr-AdamW/pg2":7.937280000000003e-06,"info/global_step":50,"loss/split_score":0.007429626304656267,"train/psnr_probabilistic":16.124618530273438,"attention_map":{"_type":"images/separated","width":800,"height":536,"format":"png","count":1,"filenames":["media/images/attention_map_3_4bf1d5f13f9f4f399591.png"],"captions":["306e2b7785657539"]},"loss/aux_1/mse":0.04901320859789848,"loss/3dgs_2_0/mse":0.03985760733485222,"_step":5,"loss/camera":0.0011391781736165285,"lr-AdamW/pg1-momentum":0.9,"_wandb":{"runtime":167},"_timestamp":1.7580172503834121e+09}

wandb/run-20250916_180500-j938t4jj/logs/debug-core.log

@@ -0,0 +1,13 @@
+{"time":"2025-09-16T18:05:00.633924244+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmppslyc4zu/port-8256.txt","pid":8256,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-09-16T18:05:00.640553681+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":8256}
+{"time":"2025-09-16T18:05:00.640388563+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":34891,"Zone":""}}
+{"time":"2025-09-16T18:05:00.719465096+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:57976"}
+{"time":"2025-09-16T18:05:00.999906955+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"j938t4jj","id":"127.0.0.1:57976"}
+{"time":"2025-09-16T18:05:01.461793428+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"j938t4jj","id":"127.0.0.1:57976"}
+{"time":"2025-09-16T18:07:48.278701448+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:57976"}
+{"time":"2025-09-16T18:07:48.279116282+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-16T18:07:48.279088283+08:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:57976"}
+{"time":"2025-09-16T18:07:48.2793282+08:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:57976"}
+{"time":"2025-09-16T18:07:48.792194009+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:57976"}
+{"time":"2025-09-16T18:07:48.792251048+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:57976"}
+{"time":"2025-09-16T18:07:48.792307767+08:00","level":"INFO","msg":"server is closed"}

wandb/run-20250916_180500-j938t4jj/logs/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2025-09-16T18:05:01.001325257+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc/wandb/run-20250916_180500-j938t4jj/logs/debug-core.log"}
+{"time":"2025-09-16T18:05:01.461505952+08:00","level":"INFO","msg":"created new stream","id":"j938t4jj"}
+{"time":"2025-09-16T18:05:01.461751509+08:00","level":"INFO","msg":"stream: started","id":"j938t4jj"}
+{"time":"2025-09-16T18:05:01.461994426+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"j938t4jj"}
+{"time":"2025-09-16T18:05:01.462239143+08:00","level":"INFO","msg":"handler: started","stream_id":"j938t4jj"}
+{"time":"2025-09-16T18:05:01.462366421+08:00","level":"INFO","msg":"sender: started","stream_id":"j938t4jj"}
+{"time":"2025-09-16T18:05:01.981645538+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-09-16T18:07:48.278988564+08:00","level":"INFO","msg":"stream: closing","id":"j938t4jj"}
+{"time":"2025-09-16T18:07:48.279073403+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-09-16T18:07:48.279385159+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-09-16T18:07:48.667849038+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-09-16T18:07:48.791572536+08:00","level":"INFO","msg":"handler: closed","stream_id":"j938t4jj"}
+{"time":"2025-09-16T18:07:48.791670775+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"j938t4jj"}
+{"time":"2025-09-16T18:07:48.791769614+08:00","level":"INFO","msg":"sender: closed","stream_id":"j938t4jj"}
+{"time":"2025-09-16T18:07:48.791910292+08:00","level":"INFO","msg":"stream: closed","id":"j938t4jj"}

wandb/run-20250916_180500-j938t4jj/logs/debug.log

@@ -0,0 +1,85 @@
+2025-09-16 18:05:00,973 INFO    MainThread:8256 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9
+2025-09-16 18:05:00,974 INFO    MainThread:8256 [wandb_setup.py:_flush():67] Configure stats pid to 8256
+2025-09-16 18:05:00,974 INFO    MainThread:8256 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings
+2025-09-16 18:05:00,974 INFO    MainThread:8256 [wandb_setup.py:_flush():67] Loading settings from /root/highspeedstorage/injae/code/ICLR2026_HCP/wandb/settings
+2025-09-16 18:05:00,974 INFO    MainThread:8256 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-09-16 18:05:00,974 INFO    MainThread:8256 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc/wandb/run-20250916_180500-j938t4jj/logs/debug.log
+2025-09-16 18:05:00,975 INFO    MainThread:8256 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc/wandb/run-20250916_180500-j938t4jj/logs/debug-internal.log
+2025-09-16 18:05:00,975 INFO    MainThread:8256 [wandb_init.py:init():781] calling init triggers
+2025-09-16 18:05:00,975 INFO    MainThread:8256 [wandb_init.py:init():786] wandb.init called with sweep_config: {}
+config: {'model': {'encoder': {'backbone': {'name': 'croco', 'model': 'ViTLarge_BaseDecoder', 'patch_embed_cls': 'PatchEmbedDust3R', 'asymmetry_decoder': True, 'intrinsics_embed_loc': 'encoder', 'intrinsics_embed_degree': 4, 'intrinsics_embed_type': 'token'}, 'name': 'noposplat', 'opacity_mapping': {'initial': 0.0, 'final': 0.0, 'warm_up': 1}, 'num_monocular_samples': 32, 'num_surfaces': 1, 'predict_opacity': False, 'gaussians_per_pixel': 1, 'gaussian_adapter': {'gaussian_scale_min': 0.5, 'gaussian_scale_max': 15.0, 'sh_degree': 4}, 'd_feature': 128, 'visualizer': {'num_samples': 8, 'min_resolution': 256, 'export_ply': False}, 'apply_bounds_shim': True, 'gs_params_head_type': 'dpt_gs', 'pose_free': True, 'pretrained_weights': ''}, 'decoder': {'name': 'splatting_cuda', 'background_color': [0.0, 0.0, 0.0], 'make_scale_invariant': False, 'use_gsplat': True}, 'vggt': {'input_image_shape': [518, 518], 'head_mode': 'pcd', 'same_head': False, 'camera_prompt': False, 'num_visual_prompt': 0, 'use_extrinsic_token': False, 'use_intrinsic_token': False, 'detr': {'name': 'deformable_detr', 'features_num_channel': [128, 32], 'hidden_dim': 32, 'dim_down_factor': 1, 'dim_feedforward_factor': 4, 'enc_layers': 6, 'dec_layers': 1, 'dropout': 0.1, 'num_feature_levels': 2, 'dec_n_points': 4, 'enc_n_points': 4, 'split_threshold': 1.0, 'num_queries': 256, 'base_HW': 64, 'position_embedding': 'sine', 'pre_norm': False, 'self_layer_num': 1, 'num_level': 3, 'debug_mode': 0, 'decoder_type': 'conv', 'split_threshold_range': [0.0, 2.0], 'only_last_level': False, 'grad_mode': 'absgrad', 'same_split_head': False, 'use_mean_features': True}}}, 'loss': {'mse': {'weight': 1.0}, 'lpips': {'weight': 0.05, 'apply_after_step': 0}}, 'wandb': {'project': 'noposplat', 'entity': 'scene-representation-group', 'name': 're10k_hpc', 'mode': 'online', 'tags': ['re10k', '256x256']}, 'mode': 'train', 'data_loader': {'train': {'num_workers': 16, 'persistent_workers': True, 'batch_size': 16, 'seed': 1234}, 'test': {'num_workers': 4, 'persistent_workers': False, 'batch_size': 1, 'seed': 2345}, 'val': {'num_workers': 1, 'persistent_workers': True, 'batch_size': 1, 'seed': 3456}}, 'optimizer': {'lr': 0.0002, 'warm_up_steps': 125, 'backbone_lr_multiplier': 0.1, 'backbone_trainable': 'GB+H', 'accumulate': 1}, 'checkpointing': {'load': None, 'every_n_train_steps': 9375, 'save_top_k': 1, 'save_weights_only': True}, 'train': {'depth_mode': None, 'extended_visualization': False, 'print_log_every_n_steps': 10, 'distiller': '', 'distill_max_steps': 1000000, 'debug_mode': 0, 'camera_consis_loss': 0.0, 'camera_loss': 1.0, 'aux_loss': 0.0, 'one_sample_train': False, 'target_scale_mode': 2, 'extrinsic_drop': 0.1, 'intrinsic_drop': 0.1, 'align_corners': False, 'backbone_checkpoint': 'none', 'intrinsic_scaling': True, 'return_level_gs': True, 'train_context': False, 'use_pred_point_focal': False, 'scaling_detach': True, 'split_score_log_scale': True, 'split_score_grad_scale': 10000.0, 'split_loss': 0.01, 'random_split_strategy': 'score'}, 'test': {'output_path': 'test', 'align_pose': False, 'pose_align_steps': 100, 'rot_opt_lr': 0.005, 'trans_opt_lr': 0.005, 'compute_scores': True, 'save_image': False, 'save_video': False, 'save_compare': False, 'split_threshold_list': [0.0, 0.1, 0.2, 0.3, 0.4], 'pred_intrinsic': False}, 'seed': 111123, 'trainer': {'max_steps': 18751, 'val_check_interval': 500, 'gradient_clip_val': 0.5, 'num_nodes': 1}, 'dataset': {'re10k': {'make_baseline_1': True, 'relative_pose': True, 'augment': True, 'background_color': [0.0, 0.0, 0.0], 'overfit_to_scene': None, 'skip_bad_shape': True, 'view_sampler': {'name': 'bounded', 'num_target_views': 4, 'num_context_views': 2, 'min_distance_between_context_views': 45, 'max_distance_between_context_views': 90, 'min_distance_to_context_views': 0, 'warm_up_steps': 9375, 'initial_min_distance_between_context_views': 25, 'initial_max_distance_between_context_views': 25, 'distill_steps': 0}, 'name': 're10k', 'roots': ['datasets/re10k'], 'input_image_shape': [256, 256], 'original_image_shape': [360, 640], 'cameras_are_circular': False, 'baseline_min': 0.001, 'baseline_max': 10000000000.0, 'max_fov': 100.0}}, '_wandb': {}}
+2025-09-16 18:05:00,975 INFO    MainThread:8256 [wandb_init.py:init():809] starting backend
+2025-09-16 18:05:00,975 INFO    MainThread:8256 [wandb_init.py:init():813] sending inform_init request
+2025-09-16 18:05:00,995 INFO    MainThread:8256 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-09-16 18:05:00,996 INFO    MainThread:8256 [wandb_init.py:init():823] backend started and connected
+2025-09-16 18:05:01,012 INFO    MainThread:8256 [wandb_init.py:init():915] updated telemetry
+2025-09-16 18:05:01,056 INFO    MainThread:8256 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout
+2025-09-16 18:05:01,970 INFO    MainThread:8256 [wandb_init.py:init():1014] starting run threads in backend
+2025-09-16 18:05:02,143 INFO    MainThread:8256 [wandb_run.py:_console_start():2454] atexit reg
+2025-09-16 18:05:02,143 INFO    MainThread:8256 [wandb_run.py:_redirect():2306] redirect: wrap_raw
+2025-09-16 18:05:02,145 INFO    MainThread:8256 [wandb_run.py:_redirect():2371] Wrapping output streams.
+2025-09-16 18:05:02,145 INFO    MainThread:8256 [wandb_run.py:_redirect():2394] Redirects installed.
+2025-09-16 18:05:02,147 INFO    MainThread:8256 [wandb_init.py:init():1056] run started, returning control to user process
+2025-09-16 18:07:48,276 INFO    MsgRouterThr:8256 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.
+2025-09-16 18:07:49,166 ERROR   MainThread:8256 [redirect.py:_on_write():661] [no run ID] error in stdout callback
+Traceback (most recent call last):
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write
+    cb(written_data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 2375, in <lambda>
+    lambda data: self._console_raw_callback("stdout", data),
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback
+    self._backend.interface.publish_output_raw(name, data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw
+    self._publish_output_raw(o)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
+    self._publish(rec)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
+    self._sock_client.send_record_publish(record)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
+    self.send_server_request(server_req)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
+    self._send_message(msg)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
+    self._sendall_with_error_handle(header + data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
+    sent = self._sock.send(data)
+           ^^^^^^^^^^^^^^^^^^^^^
+BrokenPipeError: [Errno 32] Broken pipe
+2025-09-16 18:07:49,170 ERROR   MainThread:8256 [redirect.py:_on_write():661] [no run ID] error in stdout callback
+Traceback (most recent call last):
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/redirect.py", line 659, in _on_write
+    cb(written_data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 2375, in <lambda>
+    lambda data: self._console_raw_callback("stdout", data),
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 387, in wrapper
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 435, in wrapper_fn
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1518, in _console_raw_callback
+    self._backend.interface.publish_output_raw(name, data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface.py", line 761, in publish_output_raw
+    self._publish_output_raw(o)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
+    self._publish(rec)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
+    self._sock_client.send_record_publish(record)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
+    self.send_server_request(server_req)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
+    self._send_message(msg)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
+    self._sendall_with_error_handle(header + data)
+  File "/opt/conda/envs/ours/lib/python3.12/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
+    sent = self._sock.send(data)
+           ^^^^^^^^^^^^^^^^^^^^^
+BrokenPipeError: [Errno 32] Broken pipe

wandb/run-20250916_181645-hudaedu3/files/output.log

@@ -0,0 +1,176 @@
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+
+  | Name    | Type                 | Params | Mode
+---------------------------------------------------------
+0 | vggt    | VGGT                 | 586 M  | train
+1 | decoder | DecoderSplattingCUDA | 0      | train
+2 | losses  | ModuleList           | 0      | train
+---------------------------------------------------------
+586 M     Trainable params
+0         Non-trainable params
+586 M     Total params
+2,345.910 Total estimated model params size (MB)
+813       Modules in train mode
+980       Modules in eval mode
+Sanity Checking: |                                                                                                                                                                                                    | 0/? [00:00<?, ?it/s][2025-09-16 18:16:48,200][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:425: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=34` in the `DataLoader` to improve performance.
+
+[2025-09-16 18:16:48,201][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/torch/distributed/distributed_c10d.py:4631: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user.
+  warnings.warn(  # warn only once
+
+Validation epoch start on rank 0
+Sanity Checking DataLoader 0:   0%|                                                                                                                                                                                   | 0/1 [00:00<?, ?it/s]validation step 0; scene = ['306e2b7785657539']; context = [[48, 73]]
+target intrinsic:  tensor(0.8595, device='cuda:0') tensor(0.8597, device='cuda:0')
+pred intrinsic:  tensor(0.8780, device='cuda:0') tensor(0.8776, device='cuda:0')
+[2025-09-16 18:16:51,169][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/torch/utils/cpp_extension.py:2356: UserWarning: TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation.
+If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'].
+  warnings.warn(
+
+[2025-09-16 18:16:51,447][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/utilities/data.py:79: Trying to infer the `batch_size` from an ambiguous collection. The batch size we found is 1. To avoid any miscalculations, use `self.log(..., batch_size=batch_size)`.
+
+Setting up [LPIPS] perceptual loss: trunk [vgg], v[0.1], spatial [off]
+[2025-09-16 18:16:51,448][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
+  warnings.warn(
+
+[2025-09-16 18:16:51,449][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
+  warnings.warn(msg)
+
+Loading model from: /opt/conda/envs/ours/lib/python3.12/site-packages/lpips/weights/v0.1/vgg.pth
+[2025-09-16 18:16:53,384][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/torch/functional.py:554: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at /pytorch/aten/src/ATen/native/TensorShape.cpp:4314.)
+  return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]
+
+Sanity Checking DataLoader 0: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:04<00:00,  0.20it/s][2025-09-16 18:16:53,735][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('val/psnr', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+
+[2025-09-16 18:16:53,738][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('val/lpips', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+
+[2025-09-16 18:16:53,738][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('val/ssim', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+
+[2025-09-16 18:16:53,739][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('val/gaussian_num_ratio', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+
+Epoch 0: |                                                                                                                                                                                                            | 0/? [00:00<?, ?it/s][2025-09-16 18:17:04,137][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/torch/autograd/graph.py:824: UserWarning: Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed.  This is not an error, but may impair performance.
+grad.sizes() = [56, 128, 1, 1], strides() = [128, 1, 128, 128]
+bucket_view.sizes() = [56, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at /pytorch/torch/csrc/distributed/c10d/reducer.cpp:328.)
+  return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+
+Epoch 0: |                                                                                                                                                                                        | 9/? [00:29<00:00,  0.30it/s, v_num=edu3]train step 10; scene = ['78784c6ffe218407', '2144f1fd5b00dc66', 'f568ab2fa64994e9', 'eac6cedeba1f720b', '216d5454333d29b5', 'b628b7d46a07cb57', '3b07094586f67024', '044df93e59024ee2', '167c67586a8c0846', '3931c6a7e69ffff1', 'c9c67636b9d521be', '0908c09364f54c42', '2d92b7321e1039c5', '3bc0044f0678f39f', 'e51ef9945ae527c4', '31250ac58b3e32c9']; context = [[31, 56], [10, 35], [7, 32], [27, 52], [249, 274], [57, 82], [15, 40], [114, 139], [6, 31], [6, 31], [137, 162], [14, 39], [10, 35], [95, 120], [55, 80], [89, 114]]; loss = 0.772814
+Epoch 0: |                                                                                                                                                                                       | 19/? [00:59<00:00,  0.32it/s, v_num=edu3]train step 20; scene = ['4581bdd74b6ee0a9', 'd56e9c764f637e1c', '1f9574f1e0c1c2b2', 'd7e2a0e5718dc803', 'b2186a798ad4fb49', 'db1723893a3a6277', 'b3edac66be582f84', '6546295c3cec469a', 'b3232677872b2974', '2f27acaf69f3c7a7', '486673d483fe843a', '488358b53866b7aa', 'eedd397acee4cb4f', 'c18ea6a77949e219', '5140967fcd51092e', '08234c2181c4db46']; context = [[6, 31], [26, 51], [45, 70], [209, 234], [31, 56], [9, 34], [30, 55], [247, 272], [58, 83], [0, 25], [62, 87], [10, 35], [113, 138], [92, 117], [43, 68], [162, 187]]; loss = 0.654585
+Epoch 0: |                                                                                                                                                                                       | 29/? [01:28<00:00,  0.33it/s, v_num=edu3]train step 30; scene = ['6dd4b348223c880c', 'dff181c503e5c202', 'fd4c9fb0fae56ccd', '66c3771cf7cb96c2', '75790c62002de4f2', 'c7e94713ef1f1ace', '0cbf1454a4391523', '9bb0448e7cb2f733', 'a8cef6a851fbea3c', '9c763036f1d84e18', '7759eccd562145b9', 'a597e94e50eb917a', '67e69f3a4701dfd8', 'b91dfcb3102d1482', '5a2ad43377e9d18d', '0e4916077d990d1e']; context = [[4, 29], [102, 127], [26, 51], [247, 272], [39, 64], [16, 41], [129, 154], [10, 35], [38, 63], [101, 126], [8, 33], [111, 136], [10, 35], [76, 101], [52, 77], [21, 46]]; loss = 0.437110
+Epoch 0: |                                                                                                                                                                                       | 39/? [01:58<00:00,  0.33it/s, v_num=edu3]train step 40; scene = ['af9d978c9fca5752', '55cf2bbe9e017ea4', '437474fa4196e1b7', '0207b0ec0cc851f6', '1684dbfef3d182a9', '512cd4ea0fef0198', 'd1772c09b4b6d95f', '629f07baec1cbad5', '99e9f09b456ef23e', 'd066040e500bd58b', '7e7b53917bcf337b', '0066ed3711a7240d', '0f0bbb6510ca3018', 'c49bd62c183dd925', 'd7ca14c855d58875', 'b07a54eda9cdaccb']; context = [[175, 200], [3, 28], [122, 147], [24, 49], [17, 42], [35, 60], [53, 78], [110, 135], [31, 56], [13, 38], [54, 79], [71, 96], [37, 62], [11, 36], [45, 70], [106, 131]]; loss = 0.284089
+Epoch 0: |                                                                                                                                                                                       | 49/? [02:28<00:00,  0.33it/s, v_num=edu3]train step 50; scene = ['c1bcd66d5a2d51b6', 'ea5a39bd855fdd91', 'f3135a7492955a8a', '9794641b7e015578', '5041701491a4b930', 'c2fd6e5ebbff57b3', '604b0838aa3321c5', '86d9acf576d8ec0b', '2dd4d2a252e34405', 'fe1cf5504260d4ea', '7e58b6857e275547', '7dfeaf87a8145697', '96b163f47895beee', '83c9baf5bbc941ee', '23d17b97a9d3674e', '36846d6fa22bba62']; context = [[113, 138], [33, 58], [127, 152], [125, 150], [124, 149], [27, 52], [26, 51], [157, 182], [30, 55], [3, 28], [132, 157], [8, 33], [115, 140], [106, 131], [69, 94], [135, 160]]; loss = 0.230237
+Epoch 0: |                                                                                                                                                                                       | 59/? [02:58<00:00,  0.33it/s, v_num=edu3]train step 60; scene = ['d8b22b4eb5e28e71', '68e586537c71a833', 'ad24c22967e79c48', '5a6fefaa5fc9fb2b', 'c0e3203eaee46f03', '8e16c8644efeec52', 'b0acc8ab51920ba2', '03d05f69a1cab4f8', '4017d144a78a75f1', '729acff96609b269', '4c9fcaa404a29913', 'c963dfce16d61584', '45a2984105a2b69b', '88758635b5ba64ab', 'cc68efcc2b123690', '5d77d10032b5d35d']; context = [[88, 113], [101, 126], [43, 68], [15, 40], [44, 69], [6, 31], [14, 39], [129, 154], [125, 150], [40, 65], [161, 186], [0, 25], [12, 37], [20, 45], [161, 186], [17, 42]]; loss = 0.202005
+Epoch 0: |                                                                                                                                                                                       | 69/? [03:27<00:00,  0.33it/s, v_num=edu3]train step 70; scene = ['73ecf847059621f6', '6a60c963307639aa', '83f9d8c6d307248a', '6df172462e2547f2', '8dd73309b133b8bf', 'bbbded1a277dd24f', 'c7af45d1166fe6cb', 'f5db0bad3ebe96e3', '57a51e1d7e1b80fe', '5223b08ead451ba3', 'f6894181dbf11e40', 'b24a35dc8884076b', '73b27f4f150327af', '932af4bdfcb8093b', '7bdeaa5ddcb319c8', '6e1a2b5a4f427345']; context = [[5, 30], [0, 25], [18, 43], [133, 158], [47, 72], [5, 30], [23, 48], [2, 27], [51, 76], [100, 125], [0, 25], [19, 44], [41, 66], [178, 203], [1, 26], [59, 84]]; loss = 0.164023
+Epoch 0: |                                                                                                                                                                                       | 79/? [03:56<00:00,  0.33it/s, v_num=edu3]train step 80; scene = ['d628ed946998f4f0', '75335793f866b96d', '634b28ad470cf75a', '04285c62eb6cc081', 'cd2d9ac2a7223cf5', '3aba252e263e7585', '3d90d586b33daa63', '86dfadf971ff9ff5', 'a9936b77895f33b3', '06a16655c8e8ad9c', '1e56c6d50221c84b', 'c5a85f680c4f9c9c', '92bc1a5a284f9e88', '186b6487fea252ab', '6cbbe9075b0d2138', 'fb7776636c2a50b5']; context = [[24, 49], [28, 53], [74, 99], [44, 69], [104, 129], [215, 240], [72, 97], [87, 112], [20, 45], [25, 50], [122, 147], [50, 75], [166, 191], [37, 62], [104, 129], [36, 61]]; loss = 0.148352
+Epoch 0: |                                                                                                                                                                                       | 89/? [04:27<00:00,  0.33it/s, v_num=edu3]train step 90; scene = ['48c804cdf23efb75', 'bdf562da5cc7475c', '31240f5e4b708132', '723f94d150ab09f2', '296063ceba643c07', '6e5505414fd63528', '5a9672017ab85918', 'a67e709b68b7163d', '7e34a8e57c879465', '18aecc2e66ae3112', '0160565c91c8654b', '692a87e2b7ebc8be', '2107f0bc2f004d1e', 'c25b314716aa6b10', '0779daaad5fd0ef7', 'e6cfdadf048f70a1']; context = [[8, 33], [0, 25], [91, 116], [248, 273], [100, 125], [50, 75], [64, 89], [66, 91], [29, 54], [43, 68], [36, 61], [4, 29], [55, 80], [0, 25], [52, 77], [55, 80]]; loss = 0.142850
+Epoch 0: |                                                                                                                                                                                       | 99/? [04:56<00:00,  0.33it/s, v_num=edu3]train step 100; scene = ['b34bb5f53856d34f', 'd110e65023ed8364', 'cb155c027de7a145', '7d9fc989aa9c6149', '6d51b12343179f02', 'cc2226bd7f8faab6', '84f60ca2323e9654', '4e1c106e3effe9bc', '042ddfb169d6bbdb', '342099a48847f4f6', '6ea0ff32c8ea695c', '38a6a1ac5db4c9ca', 'f7a74e0520117195', 'a931b461d6725481', '786cc15dbe60b001', '94841d39236a40ee']; context = [[0, 25], [25, 50], [126, 151], [97, 122], [3, 28], [19, 44], [17, 42], [63, 88], [26, 51], [12, 37], [3, 28], [117, 142], [8, 33], [79, 104], [68, 93], [28, 53]]; loss = 0.125427
+Epoch 0: |                                                                                                                                                                                      | 109/? [05:26<00:00,  0.33it/s, v_num=edu3]train step 110; scene = ['da0fad2070e6c0dd', 'e17208be486cc7a7', '2b15c8704aaa87ba', 'dbac9841157230ce', '0f62ec5c20682f71', '1251592126203554', 'bd99ad1d47bb9005', '04b859d3d26366de', 'f4b06ed6e137cd5d', 'b1de49fea231bb59', 'f92964360d184d99', 'a0b45bab75d35ee0', '3d9f44d7bdd0796d', '42000d5a83b48ee4', '625e3aa0ff734714', 'd2dd3b46d9903314']; context = [[21, 46], [125, 150], [41, 66], [225, 250], [217, 242], [8, 33], [76, 101], [4, 29], [168, 193], [20, 45], [5, 30], [13, 38], [36, 61], [45, 70], [57, 82], [132, 157]]; loss = 0.127871
+Epoch 0: |                                                                                                                                                                                      | 119/? [05:56<00:00,  0.33it/s, v_num=edu3]train step 120; scene = ['65c4746a5f0a43b7', 'ce3ff34509148f55', '2b83ce9b23c5ab7d', 'd729044c166a455f', '84cede8286dcf69b', '3ba79e4a3499b7bc', 'afc970648a89e04a', '5a040cbc8d351ff7', '8278bfd97661dc65', 'bf37ac6cc15e90c4', '2bf30153b26e2060', '814f74b9c3c4d094', '0a609dca037e83a9', 'eda299b8ec5d56ff', '82b8b4c5e983729f', '122d66974706d78c']; context = [[3, 28], [2, 27], [41, 66], [173, 198], [70, 95], [70, 95], [79, 104], [159, 184], [1, 26], [76, 101], [45, 70], [31, 56], [8, 33], [64, 89], [155, 180], [4, 29]]; loss = 0.105100
+Epoch 0: |                                                                                                                                                                                      | 124/? [06:10<00:00,  0.33it/s, v_num=edu3][2025-09-16 18:23:12,050][py.warnings][WARNING] - /opt/conda/envs/ours/lib/python3.12/site-packages/torch/optim/lr_scheduler.py:198: UserWarning: The epoch parameter in `scheduler.step()` was not necessary and is being deprecated where possible. Please use `scheduler.step()` to step the scheduler. During the deprecation, if epoch is different from None, the closed form is used instead of the new chainable form, where available. Please open an issue if you are unable to replicate your use case: https://github.com/pytorch/pytorch/issues/new/choose.
+  warnings.warn(EPOCH_DEPRECATION_WARNING, UserWarning)
+
+Epoch 0: |                                                                                                                                                                                      | 129/? [06:25<00:00,  0.33it/s, v_num=edu3]train step 130; scene = ['7898a828b7203ca4', '6814d469f5cec50d', '68a4606ce0150ce6', '70a343bffc90f4aa', 'c31cf070ae1a75d4', 'e734e244799d0bfd', 'cb734fdc69e9900e', '89e3bea1f3d2137a', '12f1a55c604c66b0', 'f8532bb5eca0e29d', '174ebd189316bd92', '0087cf27aa9e6167', 'be0f77ad0082e93e', 'db6cd90de8fee2ff', '739932d092e06fe2', 'b703bdb45d172fe7']; context = [[32, 57], [78, 103], [64, 89], [45, 70], [73, 98], [143, 168], [24, 49], [132, 157], [239, 264], [25, 50], [39, 64], [32, 57], [221, 246], [60, 85], [68, 93], [72, 97]]; loss = 0.124171
+Epoch 0: |                                                                                                                                                                                      | 139/? [06:55<00:00,  0.33it/s, v_num=edu3]train step 140; scene = ['c206b7bad4e2bd69', '7a82d470ad5d5855', 'c08a5ec56e05c975', 'f928d960cbfae15a', '6c7379b0be88feb0', 'cc2ee5b080780c5e', '93f6c9634e5ad304', '631caf62e5167b66', 'eeb55a861130a21d', 'a47a167922cfc6df', 'd4a97e0a52dc8cb7', '8f110752082f188c', '8b671df4b3b155bd', 'd90000318f9a8442', 'dd3bbf1f7f832e83', '1786d6d8c1da703e']; context = [[19, 44], [48, 73], [108, 133], [48, 73], [250, 275], [0, 25], [1, 26], [76, 101], [18, 43], [55, 80], [53, 78], [21, 46], [38, 63], [56, 81], [3, 28], [38, 63]]; loss = 0.102875
+Epoch 0: |                                                                                                                                                                                      | 149/? [07:25<00:00,  0.33it/s, v_num=edu3]train step 150; scene = ['abab9d091104123a', '58f47cf52ac6cd8e', '496af891d9e2079c', '5b14f5577d88ef9d', 'd317148296e2c9e7', '9fde0d7ec4b06f6c', '225e7ec65db4fff6', '01e4bd13a70c49ac', 'ff6e099a51781a35', 'd354f81b3f57bc62', '27d205da1f0f929c', 'bf2b8a204cdeb69f', 'e5812066c518f8a2', '1d035bb092fa4113', '5da7fa4de4cc06e1', 'f5ac33bf99394312']; context = [[43, 68], [217, 242], [3, 28], [62, 87], [17, 42], [37, 62], [4, 29], [1, 26], [136, 161], [36, 61], [16, 41], [98, 123], [2, 27], [33, 58], [2, 27], [68, 93]]; loss = 0.107159
+Epoch 0: |                                                                                                                                                                                      | 159/? [07:55<00:00,  0.33it/s, v_num=edu3]train step 160; scene = ['b95ffd3039761a94', '43c4c89b3414a0fc', '956df4359a5b3ced', 'bd077bfc739857cd', '252aa7d6ffb0422d', '14af89bdc75382ab', 'fc8deaf763806c66', 'c236887ce05e2c32', '20f3647b9c4356a9', '3abd72e83aac22b2', 'dd9067bcff6354f9', 'ad75e36b74f6b033', 'e0c462f81b501a0f', '0b6bc1cddcc503af', 'e68bf4919dbd4e29', 'a4cc10dede2eba84']; context = [[44, 69], [49, 74], [174, 199], [208, 233], [44, 69], [54, 79], [1, 26], [7, 32], [59, 84], [8, 33], [6, 31], [144, 169], [191, 216], [7, 32], [134, 159], [19, 44]]; loss = 0.099668
+Epoch 0: |                                                                                                                                                                                      | 169/? [08:24<00:00,  0.33it/s, v_num=edu3]train step 170; scene = ['d93de5f6c62c01f3', '3e68b6b7bb281e94', 'cb09920c02a2b6bb', '506fed97ede5adbf', 'fc8aa2c335c15eb9', '25bd820746058fb5', '37578ce7a19d91aa', '9569fcfe82105f38', '70285f05516940b2', 'c7d4d5977857ba5a', '74c2c2c5287346b6', '95bdea61f4c80624', '2bfd7a62ae051639', '716290f59cec96d4', '5460e2c24ab3b848', '9093c09d9919a5ee']; context = [[65, 90], [8, 33], [17, 42], [45, 70], [138, 163], [72, 97], [53, 78], [58, 83], [9, 34], [144, 169], [62, 87], [46, 71], [60, 85], [69, 94], [10, 35], [22, 47]]; loss = 0.103284
+Epoch 0: |                                                                                                                                                                                      | 179/? [08:54<00:00,  0.33it/s, v_num=edu3]train step 180; scene = ['59e55e5b179a50ba', '2ed33656a13829b9', '04d4fee4c437c04e', 'ea0ae256b3a02a33', '7357769abf6e270a', '2243808dcedf81c9', '7572cf78733b5e04', '8e15c3ce3989dc4f', 'cf78433dadf58175', '0832ad3a8f7c0ffc', '12f55e64a1f460c7', 'afba4de2c78eb96f', '7f172f67f7c6fb69', 'cb797cd30542e55c', '88d67718b0e0e025', '1e100eeadbff952c']; context = [[40, 65], [30, 55], [0, 25], [36, 61], [21, 46], [93, 118], [131, 156], [27, 52], [54, 79], [13, 38], [17, 42], [74, 99], [14, 39], [6, 31], [13, 38], [67, 92]]; loss = 0.103037
+Epoch 0: |                                                                                                                                                                                      | 189/? [09:24<00:00,  0.33it/s, v_num=edu3]train step 190; scene = ['dd8e5b4c898f822d', '3645515a07f57fde', '198d644c5230422f', '60c37b519a01205d', 'ee34918918062668', '629a4530b18dc456', 'dd1a25142bac8d29', 'f44cc142d9796ff7', '7a6bd2a4d2688ce2', 'cbb91d90ddc6d3fd', 'c93c5abcaf89dd0d', 'a5d23080c6b858b0', '39cafa06c9c431dc', '3d2772568f41a9b6', '7a564b61233c3fca', '73dcd9322253689e']; context = [[29, 54], [13, 38], [43, 68], [4, 29], [3, 28], [67, 92], [47, 72], [117, 142], [15, 40], [16, 41], [17, 42], [31, 56], [80, 105], [89, 114], [108, 133], [5, 30]]; loss = 0.085640
+Epoch 0: |                                                                                                                                                                                      | 199/? [09:54<00:00,  0.33it/s, v_num=edu3]train step 200; scene = ['1087329ad17df204', '5eca48d9febf5e70', 'bf4e0656a420aa9d', 'eff98653337775a8', 'a2ae856e2faf3097', '4eb73e73ca77fb11', 'a3214062cd1950df', '8f5e074629cedd06', '5be8385a860cd167', '21d88b442db5f97f', '282c3c958edab940', '8c2000c91596ea0d', '0339818f120ea7ab', '3738997e842ffe36', '281928f66836f023', '6dd5bd6445ada623']; context = [[40, 65], [22, 47], [25, 50], [29, 54], [15, 40], [42, 67], [219, 244], [8, 33], [22, 47], [23, 48], [4, 29], [59, 84], [13, 38], [20, 45], [8, 33], [133, 158]]; loss = 0.084745
+Epoch 0: |                                                                                                                                                                                      | 209/? [10:24<00:00,  0.33it/s, v_num=edu3]train step 210; scene = ['5498b21b7b980c1f', '2abe932fd9d76528', '86f3bbf2864c65f0', '2ae7890bd8231e93', '8fabc39ad677dece', '806af4d2d935738b', 'd78079ff1a0f045d', '2b132bbbc4b720b0', 'b551bb88ffea8c1d', 'b569a2f2d95abd90', 'a017de2545c81e26', '63636dcaf9b864e6', '874a7d92f2302f58', '4923656d2871e39c', '901e47c506f0d2c1', '7e7d7e538815535b']; context = [[24, 49], [36, 61], [38, 63], [234, 259], [68, 93], [118, 143], [46, 71], [2, 27], [106, 131], [133, 158], [68, 93], [15, 40], [1, 26], [225, 250], [24, 49], [1, 26]]; loss = 0.078213
+Epoch 0: |                                                                                                                                                                                      | 219/? [10:54<00:00,  0.33it/s, v_num=edu3]train step 220; scene = ['09db4e6feaea4714', 'f38d945b92c6e0e2', '6ba12d097566dbdb', '835a96da6173dd7e', 'ec2e3374601803b6', '458f38fa2ef28e63', '95cc4a86e6e8e823', 'a3687e6319f47730', '2395ecfa5327b13a', '4fcec6671d77987c', 'e308bb209941c309', '3f53a38d77a20a18', 'aad9a3b6cf7ad971', '68acc2522f558367', '6bd6d8270eee0b81', '59af7a148f0573b3']; context = [[36, 61], [236, 261], [35, 60], [88, 113], [38, 63], [84, 109], [44, 69], [41, 66], [11, 36], [232, 257], [3, 28], [26, 51], [43, 68], [190, 215], [71, 96], [6, 31]]; loss = 0.082822
+Epoch 0: |                                                                                                                                                                                      | 229/? [11:24<00:00,  0.33it/s, v_num=edu3]train step 230; scene = ['d7b308d30c7e7630', '8dd198d245a49bd5', 'b52203577bf5ab66', 'a025554e94654c2f', '590d0d46abee63bb', 'de1181d6c642e883', '558684b06f159a0a', 'd6dd35d96ec83e83', '443d63805b3fd4ef', 'b877fceeed5942d8', '125318c9856ce742', '8049eef0ee607a97', '07278d9d2d1e4e6f', 'a394ce027ea94a26', '2392a428453da5e7', '92db89e50720951f']; context = [[42, 67], [58, 83], [26, 51], [6, 31], [40, 65], [115, 140], [10, 35], [19, 44], [107, 132], [52, 77], [19, 44], [15, 40], [210, 235], [5, 30], [228, 253], [3, 28]]; loss = 0.105271
+Epoch 0: |                                                                                                                                                                                      | 239/? [11:53<00:00,  0.33it/s, v_num=edu3]train step 240; scene = ['a026ab82720e6363', '1c30239815f91334', '510c5beb7e6a13e2', '29f66f7d00d5a958', '824a7bbb922965f5', '053e55bc3666cc69', '2a9baa599c00613f', 'aa7d34a0930aaed6', '86961a46929cf91d', '8410b8c23b2c737d', '6789b17ebe4606cd', '12e0d4265e7789b4', '0a91685401b56e22', 'f6a0556897b15d6b', 'd83f2d1ffd5fd125', 'b281bf93286a0573']; context = [[69, 94], [74, 99], [29, 54], [78, 103], [10, 35], [51, 76], [197, 222], [7, 32], [38, 63], [30, 55], [94, 119], [8, 33], [53, 78], [33, 58], [42, 67], [204, 229]]; loss = 0.084950
+Epoch 0: |                                                                                                                                                                                      | 249/? [12:23<00:00,  0.33it/s, v_num=edu3]train step 250; scene = ['08127a499408b79e', '811e5be9bf8e26ca', '15cd8f43f173d0e3', 'f1f5e26cb49fd2a5', 'e1fbf58a49076d36', 'cae0139a521aa052', '19f950b6900d6176', 'eb54a14f301214c4', 'ec1090f15ec68a1e', '82347d56a4a55c27', '3e4608ca989fdf39', 'fee08c59f9bf59f8', '579a16674dbb8aad', '263c8f62a1f40965', '06beb74df9f499ab', '6fcb6588a79e44d1']; context = [[4, 29], [99, 124], [9, 34], [0, 25], [5, 30], [118, 143], [164, 189], [14, 39], [30, 55], [137, 162], [6, 31], [15, 40], [31, 56], [69, 94], [120, 145], [55, 80]]; loss = 0.104924
+Epoch 0: |                                                                                                                                                                                      | 259/? [12:53<00:00,  0.33it/s, v_num=edu3]train step 260; scene = ['7239b2a7554c75fe', '5947dd1e10a23b7c', 'fea1c01a1b763526', 'f0ae1c0ba950cd70', 'b6e320aeec7c0ba2', '3154710017640b49', 'ef1fa8ad46c8a424', '339a2912e861d13e', 'b35aad97d393ba2b', 'abe1b540708162f2', '14f5326ed459f92f', '42832be11a0bf2b5', '8c7d25a2c34e0402', 'b5a4e0980cd9c9f6', '108d63b0e7a60f8b', '5bb09c5a46c372d6']; context = [[57, 82], [57, 82], [9, 34], [16, 41], [25, 50], [10, 35], [51, 76], [38, 63], [9, 34], [4, 29], [31, 56], [67, 92], [71, 96], [9, 34], [18, 43], [38, 63]]; loss = 0.085117
+Epoch 0: |                                                                                                                                                                                      | 269/? [13:22<00:00,  0.34it/s, v_num=edu3]train step 270; scene = ['1a20aac3a317d67e', '4705db5f75ab548e', '0a3c5ae29bf2cb59', '231964ce719fed4c', '62560eba3ba6f77a', '41986a305a19ca1d', '9beaef3924b260b5', 'c63b37ec347f0d0e', '175e1611df44964f', '944e92ff3fea78eb', '52cf0ad63b78dd22', '4993c13bd0a11f24', 'ea97e5ae55e56208', '97caecafbfa7f1f6', '826d8ad6169872ba', 'a95dacbd3ea3db36']; context = [[130, 155], [37, 62], [5, 30], [0, 25], [117, 142], [61, 86], [3, 28], [10, 35], [172, 197], [99, 124], [74, 99], [47, 72], [44, 69], [71, 96], [5, 30], [59, 84]]; loss = 0.096085
+Epoch 0: |                                                                                                                                                                                      | 279/? [13:53<00:00,  0.33it/s, v_num=edu3]train step 280; scene = ['d70347e4bc080e50', '8dda78cf108c2d57', 'f6c2d79ed734958a', 'e3af760082725e5a', '37f2b83c868084df', '5627575be8e76e1d', 'eef721bc7b9f4a92', '3046f14dad8f19f2', 'cf1b9b9835b3d061', 'f0cd80abbd204b40', 'c65e2e20eff15b15', 'f1657c6128d2b332', '20a321699aab25aa', 'b5476504de893f96', 'fc36c85463412c38', 'c25194b2a961a96a']; context = [[9, 34], [134, 159], [6, 31], [6, 31], [214, 239], [67, 92], [10, 35], [18, 43], [83, 108], [12, 37], [7, 32], [5, 30], [10, 35], [104, 129], [76, 101], [13, 38]]; loss = 0.091324
+Epoch 0: |                                                                                                                                                                                      | 289/? [14:23<00:00,  0.33it/s, v_num=edu3]train step 290; scene = ['5b756acd27bfbaa9', 'dd6b426f383d86cf', 'c699a0c7b862e041', 'bd814603b3e38fb1', '774047a695fba862', '3d86d5c255821277', '0fc2d2e45645813b', '41646b4cab9e19e5', 'ab749461af5d5717', 'f22b0491bc90af5b', '67b0fe8f8f97dae4', '257c31a64a016d24', 'e81c3eda0310ba4e', '116dd0818dd9560e', '3a75651c2d3168a1', 'df11b2823a464dda']; context = [[27, 52], [14, 39], [30, 55], [148, 173], [17, 42], [30, 55], [84, 109], [80, 105], [12, 37], [30, 55], [191, 216], [24, 49], [0, 25], [41, 66], [36, 61], [6, 31]]; loss = 0.077103
+Epoch 0: |                                                                                                                                                                                      | 299/? [14:52<00:00,  0.33it/s, v_num=edu3]train step 300; scene = ['1cb3ecb30e3e9d0e', 'f0b193a824ced787', '344dc121e7a7866d', 'd5c7ba9e0a7956ad', 'b7895467f9574841', 'fb526fa7f45a3ae4', 'cfeaf7156a508877', '5692ffde04b99f08', 'ef5c5a590e9d175d', 'a530c8b4fa89b84e', '572f15c53545baaf', '6feb82e28fe110f9', 'debd6aa3bae1d248', '356ba7e96c3d9426', 'c46f3f1ddfd8533a', 'd6bd81b4a34ac00d']; context = [[65, 90], [75, 100], [71, 96], [19, 44], [66, 91], [0, 25], [30, 55], [127, 152], [15, 40], [10, 35], [96, 121], [39, 64], [197, 222], [8, 33], [19, 44], [184, 209]]; loss = 0.091057
+Epoch 0: |                                                                                                                                                                                      | 309/? [15:22<00:00,  0.33it/s, v_num=edu3]train step 310; scene = ['347a65acdefb3ab8', 'b7aeefdcf8c006ec', 'ca04de3c55cd1ca0', '53d2992e5899c3d5', '84359e8e847ad96b', '8625891ed4d0cf38', '5c4acbcb2726ae77', 'ab88a6b3b387dac9', '03a09832e1472391', 'fecda62970485d2b', '882350c0ebe1872a', '12f629409a280733', 'b92f865fbde65b1d', '8dbf79bf8fb56328', 'f81c46fec218b506', '99a11af3dcb2bd13']; context = [[13, 38], [210, 235], [122, 147], [20, 45], [54, 79], [88, 113], [70, 95], [110, 135], [45, 70], [4, 29], [39, 64], [21, 46], [0, 25], [7, 32], [47, 72], [248, 273]]; loss = 0.083637
+Epoch 0: |                                                                                                                                                                                      | 319/? [15:52<00:00,  0.33it/s, v_num=edu3]train step 320; scene = ['73fef5139753e974', '295a4324517ee9bd', 'b09e289b3dc5b7f4', '09223c50d5a39b7a', '2059d1d6c79bc51b', 'db3d51c462a54a7e', 'f7d6cec0bcd35042', '58901334e2d813d9', 'bd8103bcb04d32e1', 'd0ee747ebdd26d7f', '993a85e1b690e97d', 'f26e8a72f69b6153', 'f26c35b6e5dca2b9', '89fbc4149a5d7348', '895fe07be01436f1', 'afefafce8f467d96']; context = [[20, 45], [106, 131], [17, 42], [125, 150], [4, 29], [5, 30], [15, 40], [25, 50], [29, 54], [112, 137], [116, 141], [1, 26], [75, 100], [65, 90], [2, 27], [48, 73]]; loss = 0.084885
+Epoch 0: |                                                                                                                                                                                      | 329/? [16:22<00:00,  0.33it/s, v_num=edu3]train step 330; scene = ['1835f16e1c0b9795', '301fc1c9a4462dd3', 'ea4146e3386ff1ac', '2475b5436e129d89', 'a11880fb50e63985', 'bcd63850409a4c42', '0636f8d5854771ca', '18288a196b6403c0', '0f2eebc01e8dcad3', '4c326f63354ed371', '0684c9b9769fccdf', 'fe1fc09277b8a000', 'c62b22de6a7495be', '094724dd912fed9c', '5b9edc356515b2c4', '103119c8e78252e0']; context = [[12, 37], [33, 58], [16, 41], [73, 98], [1, 26], [23, 48], [27, 52], [39, 64], [66, 91], [39, 64], [5, 30], [32, 57], [34, 59], [13, 38], [104, 129], [39, 64]]; loss = 0.093180
+Epoch 0: |                                                                                                                                                                                      | 339/? [16:52<00:00,  0.33it/s, v_num=edu3]train step 340; scene = ['50d6d19613455b86', '29a47ea47b953e37', '485e4c3a23558aab', '129bba31edcccb2b', '4558408811e71246', 'f39f139a01476176', '0fa24a96d121319d', '45ab71fe50becdee', '8d89803ec0446d0e', '824e5d9fe8f9fe1d', '6549544819976e02', 'f8961efbf6e73ce5', '4c76d28a44171812', 'c9fc763ae456cd6f', 'aa69d5f0ef278bcc', '36e2f4d1de6396fb']; context = [[23, 48], [2, 27], [38, 63], [73, 98], [59, 84], [73, 98], [34, 59], [3, 28], [113, 138], [35, 60], [14, 39], [58, 83], [36, 61], [63, 88], [203, 228], [49, 74]]; loss = 0.075936
+Epoch 0: |                                                                                                                                                                                      | 349/? [17:22<00:00,  0.33it/s, v_num=edu3]train step 350; scene = ['2821faf83e638b87', 'f1c0d2a78d066bc0', '6494828a237a7a69', '1849a41079039a3a', '0762d4d82b19cc37', '64fe2e10e340ebc4', '6a2a2ff9c392fefe', '22b1b9bebc0a0520', 'a270004198eddd9b', 'ad6fa2297bbfa91c', '2c0939fba79df7ad', '4a3ff4b7939ec268', 'a5c9cbbc7e1d69a1', 'dbb7151643a69e0c', 'f1cbd0e2909c8ce7', 'c537c888e9db11fb']; context = [[61, 86], [17, 42], [71, 96], [15, 40], [94, 119], [22, 47], [23, 48], [61, 86], [169, 194], [2, 27], [40, 65], [104, 129], [27, 52], [205, 230], [150, 175], [21, 46]]; loss = 0.089985
+Epoch 0: |                                                                                                                                                                                      | 359/? [17:52<00:00,  0.33it/s, v_num=edu3]train step 360; scene = ['1c87d4883f49da92', '8d75d6f051b8f52e', '2a2e3da0444d2ecd', '9c70b01b66e2fabc', '73d493ff94578c55', 'ffaaeea472f10ee2', '3dc0b3372b2e13dd', '91bab928c4639b09', '6551944a93afc0d0', 'c6fa4e41f75c901c', 'f1a5f8d83353bec9', '8260ddb8f461c1fe', '2b0182cffdc9433b', 'bbaa8e8136f8e184', '6e743856adbecde2', '71fc22ad4a04cf9a']; context = [[23, 48], [120, 145], [74, 99], [133, 158], [15, 40], [51, 76], [96, 121], [44, 69], [16, 41], [41, 66], [6, 31], [102, 127], [123, 148], [36, 61], [32, 57], [0, 25]]; loss = 0.073549
+Epoch 0: |                                                                                                                                                                                      | 369/? [18:22<00:00,  0.33it/s, v_num=edu3]train step 370; scene = ['ad127c962ce28e63', '453abcae519671ba', 'fe1e321d59a67f06', '32e35693a999fb23', '6bef29b74b93e80a', '1b24cf4a586a15e7', '56db1d6fc942bc90', '7194b8d204f4a0b6', '4bbb9fadc993969a', '4e8e3034d0aa307b', '6acaceae77f75500', 'd91a42243deb04e3', 'a1e59c09aca976c4', 'ff87c19446c6325e', '0aa2d2048c581dff', '9e1b58ec292d6690']; context = [[20, 45], [27, 52], [3, 28], [212, 237], [24, 49], [87, 112], [110, 135], [78, 103], [2, 27], [172, 197], [154, 179], [167, 192], [59, 84], [23, 48], [145, 170], [33, 58]]; loss = 0.078481
+Epoch 0: |                                                                                                                                                                                      | 379/? [18:52<00:00,  0.33it/s, v_num=edu3]train step 380; scene = ['e91f5d416c3d2843', '3e80e30faedaa4b6', '674ac829bbf8f50c', '8d5c76362cfbd82e', '1adadfa4ed535f3b', '9037a2323c37f9ef', '102e45db9a58004c', 'eaf4b595a888214f', '9e80f805fd6c7b93', '27abdcbddd0180dd', 'c3062cb8fc7a0efd', 'b46547428ff2b071', '544eb83e4b6b4fc7', '321addf5693a66aa', '662bb59ab9f410a3', 'fe9c6c63360e43c2']; context = [[9, 34], [47, 72], [22, 47], [81, 106], [39, 64], [14, 39], [9, 34], [21, 46], [53, 78], [1, 26], [50, 75], [18, 43], [96, 121], [121, 146], [66, 91], [12, 37]]; loss = 0.089958
+Epoch 0: |                                                                                                                                                                                      | 389/? [19:22<00:00,  0.33it/s, v_num=edu3]train step 390; scene = ['2702020b31c28210', '79a763e5b64e4f45', 'de783427084a7fc8', '3731e5cc943c4ec0', 'cc80de7991d73ab0', 'b71a0cb9f4143a6e', 'bb28a1c09df8a484', '59ae8e2d2504d760', 'cc061ca8d139f499', 'c2162040f6db4d2f', '4fe4fc34f41764a4', '4858dc374c4fca93', '0069fbbde9f77bf8', '45a74c68ea76c91e', '83d4b54aeee30945', '2b58d1bbfac7511a']; context = [[62, 87], [7, 32], [22, 47], [22, 47], [45, 70], [101, 126], [88, 113], [101, 126], [48, 73], [86, 111], [139, 164], [27, 52], [121, 146], [26, 51], [7, 32], [8, 33]]; loss = 0.078213
+Epoch 0: |                                                                                                                                                                                      | 399/? [19:52<00:00,  0.33it/s, v_num=edu3]train step 400; scene = ['efedd66257d1adf0', '34bd9723166c9978', '48c2b63b452555af', '94323b4cd6a95a9a', '80b044471fd710b5', '8f243590d88e28db', 'b2c65f94460ad653', 'aecfc6c8ebc6c15e', '8839e65c6f6bb394', '1fe4a6141969e60a', '509210a9f0f87e8f', 'fceff078ea39c8b4', '17a43ba9a8843508', 'badfd7edf4b25b37', '771c99720c383982', '74e22e2724cb0a9b']; context = [[29, 54], [52, 77], [47, 72], [189, 214], [29, 54], [75, 100], [97, 122], [2, 27], [51, 76], [36, 61], [76, 101], [34, 59], [18, 43], [240, 265], [21, 46], [132, 157]]; loss = 0.059389
+Epoch 0: |                                                                                                                                                                                      | 409/? [20:22<00:00,  0.33it/s, v_num=edu3]train step 410; scene = ['9fce6892002b07be', '1f4f12476af5c871', '4aef5d4b9287e08a', '8db516061dca4e75', '3b560e3220235444', '78bfcb719521a42d', '5c907b99b4003976', '5b86537af9058e2d', '2db04fecf76095b6', 'c05a15168d4c3332', 'e24323b82d0671c2', 'd2ad156576e078cc', 'aa4938d7f2fd0a1d', 'e4fc6fa4bbd6efa1', 'c05fd148d2da6d26', '72d7ff49071fa801']; context = [[88, 113], [8, 33], [18, 43], [18, 43], [0, 25], [82, 107], [183, 208], [64, 89], [6, 31], [54, 79], [24, 49], [3, 28], [137, 162], [4, 29], [59, 84], [74, 99]]; loss = 0.066592
+Epoch 0: |                                                                                                                                                                                      | 419/? [20:52<00:00,  0.33it/s, v_num=edu3]train step 420; scene = ['53b47cb89ee91160', '7220014865edc948', '9296168b518b720b', 'cf14d94f41c6e825', 'd159debd3f3ac5ec', '12fa5b60a2ba9d8f', '64ae75e57c6aa0a4', '430d79082d999336', '6f9088cd9c755fb0', 'c654f15dda47ab5d', '1a0af7635585892a', '3729cab2867e7a06', 'ea51f71d8dc1e7b2', '7e6d320ac85bd57c', '20b2bb20840251c6', '5fcab1686e38f61c']; context = [[94, 119], [223, 248], [26, 51], [13, 38], [1, 26], [37, 62], [136, 161], [67, 92], [40, 65], [9, 34], [53, 78], [12, 37], [28, 53], [20, 45], [43, 68], [19, 44]]; loss = 0.076469
+Epoch 0: |                                                                                                                                                                                      | 429/? [21:21<00:00,  0.33it/s, v_num=edu3]train step 430; scene = ['1c7d5cf7a1e08139', '82d896f5142ee6dd', '764206404e317c51', '61c9b6540aaa5884', '11b74ef19465ab45', '409b268729c0a86d', 'd1f583b4b527fefe', '6570d802b2f69e4b', 'e46623a7d46b50d6', '12503cac61ce326e', 'adff1244e7025d77', '014d6d0fdfb6393e', 'a8dd4730a65f88fd', '08f56d6e5878317b', '723db63d24c84d1d', '8032cb2f4ac9e342']; context = [[7, 32], [72, 97], [15, 40], [36, 61], [2, 27], [14, 39], [77, 102], [222, 247], [12, 37], [150, 175], [54, 79], [15, 40], [30, 55], [16, 41], [90, 115], [56, 81]]; loss = 0.073612
+Epoch 0: |                                                                                                                                                                                      | 439/? [21:51<00:00,  0.33it/s, v_num=edu3]train step 440; scene = ['cee46b2523a45a59', '78f54b16f8204c3a', 'ab964b6b003ae5e8', '3a192914348dbd61', 'b1473420a788c688', '4a43e0185c382de7', '3e5be575ccb0f6dc', '0f8ad14bb4088d81', '4af74f4ff7c54a9b', '31f279e32e497154', 'dc077856d79ff04c', '59689345260b0920', 'c7876e0ce303f492', 'ede1a18c89abe3dd', 'c95f1f846f48d813', '739b8ee920a178e0']; context = [[87, 112], [44, 69], [10, 35], [33, 58], [4, 29], [66, 91], [100, 125], [122, 147], [207, 232], [74, 99], [53, 78], [65, 90], [44, 69], [37, 62], [124, 149], [117, 142]]; loss = 0.066762
+Epoch 0: |                                                                                                                                                                                      | 449/? [22:21<00:00,  0.33it/s, v_num=edu3]train step 450; scene = ['0be2e01e27a683b0', 'a0e4abe253996ea2', '0120521cfa518c34', 'b2a34524405f79ff', 'a1b7ad951995dadd', 'c338cd1e105943ae', '913f87b9648dda52', '3b4d6384abe5561f', '2dd8ae2b71457753', 'f001500643d191d6', '124c9f059bef0478', '23f93bec3ac52e4c', 'f1e8c1b674d290ec', '9bb10fc846b3d2b1', 'fbae7bcc4ca44675', 'ec1dcf652eae675d']; context = [[16, 41], [38, 63], [36, 61], [87, 112], [3, 28], [24, 49], [44, 69], [63, 88], [44, 69], [103, 128], [161, 186], [109, 134], [39, 64], [122, 147], [18, 43], [51, 76]]; loss = 0.109147
+Epoch 0: |                                                                                                                                                                                      | 459/? [22:51<00:00,  0.33it/s, v_num=edu3]train step 460; scene = ['22062ed897320134', '8b8f7cd56fb6a696', '11c47219c153c014', '20fa2a9557a949a9', '402071c2b1550820', 'f8a24a1cae5ae861', '591277853eb83777', '71b9b21424e1cc97', '1d4ffe48542eb14b', '6c5d6b6468bc196d', '94f4529096d17f25', '130e2bd2cf4084d7', '7bc09504a215e592', '98fa652433a9a8b7', 'acea865775134a18', '392b692c8c3f4e5f']; context = [[8, 33], [2, 27], [120, 145], [25, 50], [163, 188], [232, 257], [62, 87], [44, 69], [172, 197], [250, 275], [22, 47], [106, 131], [178, 203], [42, 67], [17, 42], [71, 96]]; loss = 0.072647
+Epoch 0: |                                                                                                                                                                                      | 469/? [23:22<00:00,  0.33it/s, v_num=edu3]train step 470; scene = ['338d0fb7e58dd4a3', '778eaeaaffc6107d', '9dda63b4544559e9', '34461872e645d034', '0684f42098862b74', '0c7171edef36d44d', 'ebde595b532cf60c', '24649be0dd272ffc', '52e13717e8b9393d', 'ee6e5709a57be759', '466666b6242e3081', 'e4e0a2c744fcabd0', '5eeb2c7cd1bd78f7', '409dc0e5cbdbe442', 'db84572032e69dc0', '395b1c86e35e8d93']; context = [[11, 36], [13, 38], [25, 50], [89, 114], [27, 52], [119, 144], [106, 131], [123, 148], [32, 57], [195, 220], [18, 43], [52, 77], [6, 31], [102, 127], [9, 34], [30, 55]]; loss = 0.069648
+Epoch 0: |                                                                                                                                                                                      | 479/? [23:51<00:00,  0.33it/s, v_num=edu3]train step 480; scene = ['a134112256d679ba', 'ea334cb643e07603', '8ccc670b427d277a', '40c5311e3b3accef', '4277d8783b6a046a', '91599919681fac69', 'a01c907f0070d11d', '896854a5d7c5179b', '9af7c151a7ca6d00', '5d14e2fa2974d6df', 'a6161601e3a76aa9', 'caee640343f2dc58', 'b9e8ae6d34748bc1', '496e2618d0fb9556', '1b4bf9a4a458e0df', '9810e48786be9e3d']; context = [[8, 33], [32, 57], [20, 45], [140, 165], [39, 64], [156, 181], [59, 84], [16, 41], [159, 184], [25, 50], [153, 178], [124, 149], [78, 103], [43, 68], [25, 50], [144, 169]]; loss = 0.077833
+Epoch 0: |                                                                                                                                                                                      | 489/? [24:22<00:00,  0.33it/s, v_num=edu3]train step 490; scene = ['d02e6b104723d39a', 'b9b73f46f1e5127a', '1290f3a6418b3dde', 'a535813b18e86b5c', '33e825f8a32b472f', '6a417181b27738ac', '53a2168e29e700dc', '91ac7d7027dcc46d', '57cea8a71cab48cf', 'de8093f620537a4e', '051c6fc0722d51f7', '1318e656455d56d2', 'b86ade7fa8b48715', '289adac94281ccb3', 'b43b9870fd778691', 'ab1ba9f15e793ef9']; context = [[144, 169], [0, 25], [217, 242], [37, 62], [23, 48], [4, 29], [50, 75], [47, 72], [49, 74], [52, 77], [25, 50], [85, 110], [2, 27], [35, 60], [45, 70], [76, 101]]; loss = 0.115178
+Epoch 0: |                                                                                                                                                                                      | 499/? [24:51<00:00,  0.33it/s, v_num=edu3]train step 500; scene = ['0f35ceaf227ff187', 'e23f945703f61bfc', '7277bcf45f9b2f21', 'd77fc60af718d2e2', '9bebde67fd28d7c2', '0645a4e5ad941af4', '99271aa47d803a6f', '4dd424a2e0a97403', '81431041a033fbc8', '6883aebc73cdcd2a', 'e3ab34faf834f40f', '4986da6c3f01c2bb', '33733a6eb9850264', '57a8005cadd17369', '63c3f51245b29666', '25c60ef8ca347ede']; context = [[4, 29], [4, 29], [131, 156], [76, 101], [59, 84], [78, 103], [171, 196], [100, 125], [51, 76], [155, 180], [30, 55], [52, 77], [160, 185], [150, 175], [33, 58], [53, 78]]; loss = 0.069103
+Epoch 0: |                                                                                                                                                                                      | 500/? [24:55<00:00,  0.33it/s, v_num=edu3]Validation epoch start on rank 0
+Validation: |                                                                                                                                                                                                         | 0/? [00:00<?, ?it/s]validation step 500; scene = ['49b8f80c849dc341']; context = [[62, 87]]
+target intrinsic:  tensor(0.8891, device='cuda:0') tensor(0.8894, device='cuda:0')                                                                                                                                    | 0/1 [00:00<?, ?it/s]
+pred intrinsic:  tensor(0.8803, device='cuda:0') tensor(0.8806, device='cuda:0')
+Epoch 0: |                                                                                                                                                                                      | 509/? [25:22<00:00,  0.33it/s, v_num=edu3]train step 510; scene = ['a5ec396329b37a88', 'dc02e89f277cc130', '9af18a1c4c45a179', 'd2443f80e34394ff', '1a7af026101df102', 'ff70172dd4ad43ec', '11dc92ba5c0e1ac3', 'bb0c96ae5026135d', 'a62bd02a1223b0e5', '625ef197fa165c4a', 'c2bf5ee569ba3681', 'a33838d396af29c0', 'c911c92ac370270a', '347f4e7f9f627a12', '5450eb5614d8af42', 'aa2ee5c4626a6c43']; context = [[1, 26], [43, 68], [109, 134], [21, 46], [36, 61], [95, 120], [58, 83], [31, 56], [21, 46], [65, 90], [39, 64], [94, 119], [175, 200], [29, 54], [9, 34], [47, 72]]; loss = 0.068601
+Epoch 0: |                                                                                                                                                                                      | 519/? [25:53<00:00,  0.33it/s, v_num=edu3]train step 520; scene = ['59e7800dec3fa9f5', '3ad7fc9a83e3e883', 'a5bcf7c72eb10c48', '89c03784240911c9', 'a24b9835a33fb46e', '4da0f9a3b1ca2560', '19bd0f9c48f069eb', 'c8e5ea381a7de099', '521aa01f117a1f3d', 'a9a1efdde077861a', '009e573e59c8c393', 'fda481eb3ffc890e', '701c95afe128e0d0', '586e189a8bb53cd8', '4c952c27612a3ac7', '997656bdb430ad43']; context = [[13, 38], [0, 25], [39, 64], [64, 89], [88, 113], [28, 53], [54, 79], [1, 26], [66, 91], [38, 63], [115, 140], [47, 72], [74, 99], [75, 100], [39, 64], [21, 46]]; loss = 0.061505
+Epoch 0: |                                                                                                                                                                                      | 529/? [26:23<00:00,  0.33it/s, v_num=edu3]train step 530; scene = ['d7e416701d1c4272', 'b9076a13bb94860c', '15536452f9b95c1f', '532786fdf5fd727d', 'a8abd4eeea08e1ad', '30bdb9819e21fbd1', 'b1aa2a14c16deaa1', 'b6974cc96a02e9e6', '05efd1fc549ed5f2', '12adbd00ee2b3fcc', 'a48e4e90b76cc3a3', '4eb66b49aeb1d641', '7043e8afce176c8c', '916572369a42b51b', '4e69d635ba8e495f', '12a8561544c97948']; context = [[86, 111], [13, 38], [179, 204], [12, 37], [73, 98], [84, 109], [84, 109], [24, 49], [24, 49], [32, 57], [132, 157], [19, 44], [7, 32], [205, 230], [4, 29], [108, 133]]; loss = 0.186769
+Epoch 0: |                                                                                                                                                                                      | 539/? [26:52<00:00,  0.33it/s, v_num=edu3]train step 540; scene = ['35e2e0c67e738b87', '605c98b35d15551d', 'e9b9ad20889455df', '73df004467108123', '41d454a6a4ccc3c6', 'ab03c2979623d6cb', 'c1628e1f1e308fb6', '5fc14076d0059d8a', 'c3f1d0f6da5ce21d', '3985b66cc91776eb', '36d957c25ce3a65d', '1a402532663d05ad', '4af1f08d7b5d1523', '837d2467631c6e6a', '6545dab9a4699ef0', '7eb0dffcddc1722c']; context = [[168, 193], [121, 146], [38, 63], [0, 25], [98, 123], [72, 97], [7, 32], [114, 139], [217, 242], [71, 96], [189, 214], [144, 169], [35, 60], [26, 51], [35, 60], [60, 85]]; loss = 0.064001
+Epoch 0: |                                                                                                                                                                                      | 549/? [27:22<00:00,  0.33it/s, v_num=edu3]train step 550; scene = ['ab8e80684547f15c', '8de0c4ae0c5bb988', 'aee9e4231eda0fdb', '59ee32c2dccc3dba', 'b210bb589468fb4d', 'dc5f38f005c3ebd6', 'e2ce81d901cdd6c7', '08527853c5cf3f4d', 'e09c7a12828e1b49', 'b113c611f9029c49', '340688cb63ead6f6', 'bae2d6272431c8eb', '7ffd821c6ca83444', '05d0fcfbe36d261f', '50c8b8855897d03a', '5e68140473b7fb10']; context = [[8, 33], [30, 55], [25, 50], [146, 171], [74, 99], [80, 105], [29, 54], [1, 26], [100, 125], [20, 45], [48, 73], [145, 170], [6, 31], [37, 62], [47, 72], [12, 37]]; loss = 0.049298
+Epoch 0: |                                                                                                                                                                                      | 559/? [27:52<00:00,  0.33it/s, v_num=edu3]train step 560; scene = ['60a584cf224950a4', 'c38835db6bec2825', 'b86f88d0d21318cd', 'c0e9d9f4f60169e0', '3ec1eeba0cc084af', 'f8ee5a395e856a9d', 'c532c6f1dbe975b7', '34037b62eef60172', 'b5f39f444ec03cee', '7c6d160c26de6887', '0f0bbb6510ca3018', '99e9f09b456ef23e', 'c49bd62c183dd925', 'd066040e500bd58b', '87c9128b943f1b3f', 'b07a54eda9cdaccb']; context = [[21, 46], [81, 106], [61, 86], [21, 46], [69, 94], [4, 29], [14, 39], [30, 55], [95, 120], [144, 169], [33, 58], [22, 47], [12, 37], [1, 26], [26, 51], [74, 99]]; loss = 0.061458
+Epoch 0: |                                                                                                                                                                                      | 569/? [28:22<00:00,  0.33it/s, v_num=edu3]train step 570; scene = ['43cd8bfd1bada381', '2c3bc0c318fa3771', '9a2d4c60e9585ed7', 'b7873e1ebdb5721f', 'ea2343e4a390441b', '95cd0bad15de18a9', 'aa801c6cfb72fdf8', '51e99330cbe8b36a', 'cb3bac70297d52c0', '04de1a4925ce16a7', 'c2648243586329ba', '699dc0e0cbd1d2d5', 'ff4fb430b2108b32', '6dd73ce2d44e63de', 'd73dae9e6ffe86dc', 'f0db130b452071e8']; context = [[0, 25], [20, 45], [96, 121], [76, 101], [37, 62], [77, 102], [14, 39], [38, 63], [153, 178], [64, 89], [49, 74], [208, 233], [8, 33], [84, 109], [29, 54], [3, 28]]; loss = 0.059369
+Epoch 0: |                                                                                                                                                                                      | 579/? [28:52<00:00,  0.33it/s, v_num=edu3]train step 580; scene = ['503b736751827a0b', 'a5b3faa684a1553a', 'cbae7b8786bec1e4', '31c1855b8ad30220', 'a772b5f2e834d70d', '162daefc4ddae544', '363ee2c7afc81e84', '7dcdecb79cbb1334', 'e4d86068c95e0922', '4ec29bd9dfd30c2e', 'b0f5b20c5724e592', '31411305ea3662c6', 'fe8c395dd70d4844', '3bee0daad1cc6c0b', '9a89163b62f0a058', 'b9277bf4125499a0']; context = [[10, 35], [35, 60], [185, 210], [91, 116], [231, 256], [20, 45], [100, 125], [63, 88], [176, 201], [26, 51], [1, 26], [144, 169], [148, 173], [153, 178], [166, 191], [138, 163]]; loss = 0.055185
+Epoch 0: |                                                                                                                                                                                      | 589/? [29:22<00:00,  0.33it/s, v_num=edu3]train step 590; scene = ['bd006cf0a751d205', '4d48befa72535f0a', '2147da0f465a3566', '8b12ac02ddbe40cb', '8481bc2172478625', 'e454026f5348630e', '201e44ad015728c7', '9029d8d8f6f0e98a', '307049b9515d7ee6', '95b1deb2d645f275', '3c418948bc7076cb', '28cab3659a1510a8', '6666ae9375b1656e', 'fd4d668123665d1f', '46d11fdef3f136d3', '3323fc8b3b6bccbc']; context = [[70, 95], [131, 156], [17, 42], [14, 39], [82, 107], [46, 71], [132, 157], [112, 137], [121, 146], [89, 114], [27, 52], [42, 67], [39, 64], [233, 258], [112, 137], [25, 50]]; loss = 0.070673
+Epoch 0: |                                                                                                                                                                                      | 599/? [29:52<00:00,  0.33it/s, v_num=edu3]train step 600; scene = ['80f5da17b4119bbf', '93416bc86e84f4c5', '91722542cd063407', '93e09aeb0a7c54d4', '63f6e847b2e2a2e7', 'a7957a1e2b7ad909', '1045f4f8fd6bdd09', '982da008194b287c', '8d758914077e5926', '6cee4547c99f248b', '4a0f95a3db913b56', 'de8ef03bc3c49125', 'be6ab5e68b93a77e', '4b2e9dd96e4d8542', '8cbb325cdb64a928', '8e129a4801a9f632']; context = [[247, 272], [152, 177], [71, 96], [6, 31], [112, 137], [47, 72], [54, 79], [3, 28], [105, 130], [15, 40], [171, 196], [11, 36], [21, 46], [73, 98], [0, 25], [54, 79]]; loss = 0.067032
+Epoch 0: |                                                                                                                                                                                      | 609/? [30:22<00:00,  0.33it/s, v_num=edu3]train step 610; scene = ['e53ec34eab835d44', '180608332f0b1b2a', '2225123ef31a93e4', 'accfed2fa6ff849d', '75e5ba7e7fb64fc0', 'd0aafe6b7593a8c6', '89b54f912927a60b', 'ac5df5f432218bc1', 'e0b48fb2ddbbe96b', '430ed582e79c5bab', 'dbc96a6d523c159f', '61d8e6311f28fae8', '36df4bc97cbd94f0', '179fbbbcc3e6e195', '448e12a47b369490', '57a522ee28c2896f']; context = [[9, 34], [100, 125], [124, 149], [17, 42], [90, 115], [45, 70], [79, 104], [22, 47], [59, 84], [94, 119], [63, 88], [90, 115], [27, 52], [30, 55], [94, 119], [65, 90]]; loss = 0.074103
+Epoch 0: |                                                                                                                                                                                      | 619/? [30:52<00:00,  0.33it/s, v_num=edu3]train step 620; scene = ['478386221245bac9', '04c740af3de5ae4c', 'bea0e295ee56d42c', '986d6bdd6646acbf', 'bd0f82f93d5693ca', '52aa7676145d5273', '56d0236a2c56b534', 'd1300efa0f9232ac', 'f6a68923ac68bc4a', 'f8f0491a2268ee5e', '6e637d0641b0d845', 'c956b44c983826de', '0abbdaad60ccd753', '27cf7ba27eed1eae', '14b48961c5e1364c', '1fe3e0dfb0bebffe']; context = [[61, 86], [28, 53], [59, 84], [44, 69], [97, 122], [47, 72], [10, 35], [18, 43], [49, 74], [44, 69], [127, 152], [146, 171], [75, 100], [43, 68], [60, 85], [1, 26]]; loss = 0.062135
+Epoch 0: |                                                                                                                                                                                      | 629/? [31:22<00:00,  0.33it/s, v_num=edu3]train step 630; scene = ['53e532801bd44085', '69fc02421193302a', 'a5822c29e761ba5a', 'af7f730cb84dc631', 'f344839ca7c971b7', 'd3dddf816450d1d0', '59a2ea0122aa4591', '030d035ba47fecc3', '41c2debbaa89e15f', '0e9005332bd00350', 'f23d671496254ef5', 'f12268aa63c53837', 'd38fcceec78870bb', '458d2a1fcd766d41', 'd2753ca4cb61778d', '33c23461406aadc8']; context = [[39, 64], [91, 116], [7, 32], [134, 159], [146, 171], [81, 106], [99, 124], [30, 55], [51, 76], [21, 46], [106, 131], [44, 69], [22, 47], [3, 28], [4, 29], [30, 55]]; loss = 0.074597
+Epoch 0: |                                                                                                                                                                                      | 639/? [31:52<00:00,  0.33it/s, v_num=edu3]train step 640; scene = ['7faf1c350d3f1f12', '5746d03325bf70d3', 'a6dbaee66b58a994', '2289780f7fdb5091', '35a67ce0f0a56dea', '4e11b2499f900054', 'b97d670a564c4f01', '5704ff301c085eb6', 'bbb0e9da2dddcfce', '33847bd0ad0529bd', '2ff050f5f09529d0', 'bd70de468feeaf26', '7b3ed343513dc1ba', 'ad1b9085098baa13', '8fa1f01dcf1e67ec', '874157e6587dd60c']; context = [[70, 95], [188, 213], [6, 31], [82, 107], [2, 27], [130, 155], [60, 85], [144, 169], [29, 54], [2, 27], [1, 26], [16, 41], [0, 25], [15, 40], [1, 26], [13, 38]]; loss = 0.054393
+Epoch 0: |                                                                                                                                                                                      | 649/? [32:21<00:00,  0.33it/s, v_num=edu3]train step 650; scene = ['2c0f63521c7e23c6', 'd11dfc2d4c7f412c', '26efd6cf15fcdd57', 'ed5029418049d85e', '600470a337c98190', 'd8eb3a556fde4131', '65134afb11a85252', 'a00a4bace03ba9b9', 'fd3490bd670b5b80', '95281743bb229945', '54f68167ea012743', 'aadfba62f23cd2d5', '67d059492bf13057', 'bebcc76384ce0054', '5f2d539c6dcc0a82', '7abd2cf015e33118']; context = [[7, 32], [16, 41], [27, 52], [73, 98], [80, 105], [14, 39], [13, 38], [155, 180], [9, 34], [24, 49], [98, 123], [24, 49], [159, 184], [25, 50], [158, 183], [48, 73]]; loss = 0.061992
+Epoch 0: |                                                                                                                                                                                      | 659/? [32:52<00:00,  0.33it/s, v_num=edu3]train step 660; scene = ['a9272405ceec0be2', 'd77284dc9b9d1031', 'c270572a7f5ea828', 'ee28455c2e76ef79', 'bdb2281a7c9f1ce5', 'f14721fab585c585', 'd011c2b2ac6ccc0d', '4412131879034bc2', '460d2009e8c97d4e', 'afd2942e07cd0a89', '3281beb44b21ae80', '874e813188a281e4', '175b05b368c133e7', '5c399d154f879440', '29ad06f3162b2bc1', '35fe94a6f4b752b3']; context = [[26, 51], [14, 39], [102, 127], [37, 62], [16, 41], [43, 68], [42, 67], [29, 54], [103, 128], [22, 47], [13, 38], [75, 100], [26, 51], [21, 46], [27, 52], [40, 65]]; loss = 0.055831
+Epoch 0: |                                                                                                                                                                                      | 669/? [33:22<00:00,  0.33it/s, v_num=edu3]train step 670; scene = ['9a94e71cea472790', 'bea3977d3073b2d2', '276de289278edacc', '0304b78786c9cc4f', '56f957c4bc88fa94', '78ca16a59dd41338', 'bedc784fb1794f8c', 'a61c8d77fdac60ca', 'f00a77b3b162ac4b', '4f0b9b05ed188565', '1fe4a6141969e60a', 'badfd7edf4b25b37', '856ff8cdb67ce060', '771c99720c383982', '42b2dabdc5ea4d93', 'cf0f25dfe419c0e5']; context = [[40, 65], [82, 107], [11, 36], [45, 70], [26, 51], [90, 115], [219, 244], [228, 253], [16, 41], [46, 71], [13, 38], [88, 113], [151, 176], [61, 86], [82, 107], [74, 99]]; loss = 0.056527
+Epoch 0: |                                                                                                                                                                                      | 679/? [33:52<00:00,  0.33it/s, v_num=edu3]train step 680; scene = ['dbd909b37336a7d4', 'a5a9c7400a09168b', '91ea7e8b11190fdc', '575979a8750fbd67', 'b09331320276c019', '4a3fd9a21123f197', '3ba6e1145c8b37a9', '839b542ef0c0696e', '02a36358f7cff958', 'c0e29365bc5f76fa', '5b4ae8c5f7349c17', 'df607878c3beffe9', 'f62a48672f8acac7', '97568a7f147a654f', 'f63942420154ff92', '632384f4c6834aea']; context = [[41, 66], [96, 121], [151, 176], [38, 63], [28, 53], [132, 157], [40, 65], [18, 43], [33, 58], [30, 55], [109, 134], [2, 27], [17, 42], [20, 45], [224, 249], [41, 66]]; loss = 0.049795
+Epoch 0: |                                                                                                                                                                                      | 689/? [34:22<00:00,  0.33it/s, v_num=edu3]train step 690; scene = ['668011916dfc9762', '6e23ea0adefdcdfe', '0fef731ee5f47c37', '8d45251c07bebb7b', 'ef8353177113530f', '400098a7ab312bdc', '4f2295416524f95e', 'dc48281d021ea939', 'c61d93b01fd6e377', '17f8f8682a86f190', '6a089a1af0cd47d3', '8044c8eaddd8149d', '7665ff641f430aa5', '882c2c3ccb2f06c3', 'e41c98366fe40b7b', '21e7ab7fe9d8753b']; context = [[40, 65], [53, 78], [85, 110], [11, 36], [52, 77], [48, 73], [19, 44], [69, 94], [27, 52], [128, 153], [6, 31], [91, 116], [124, 149], [33, 58], [70, 95], [48, 73]]; loss = 0.094748
+Epoch 0: |                                                                                                                                                                                      | 699/? [34:53<00:00,  0.33it/s, v_num=edu3]train step 700; scene = ['e9f0dce281fc1ef2', '7eba39d96769f1f3', '92cffae82bfb22ce', 'e1eaaeb084992db2', 'a987dc34de0f7d8b', '8ca0262e75fd9f8b', 'dd0418fa73c25b1e', 'fbfb9866efe5bf64', 'dd409b33f3f5c29b', '644baf494fab7ff7', 'd81f183e644ae7fe', '88b43de931cc94ac', 'fad05546fba5dace', 'bbf7acd688e96278', 'dca4d9700ae80a91', '6ac14f7de01b3021']; context = [[0, 25], [158, 183], [37, 62], [89, 114], [42, 67], [20, 45], [31, 56], [45, 70], [134, 159], [34, 59], [3, 28], [43, 68], [24, 49], [3, 28], [91, 116], [21, 46]]; loss = 0.098750
+Epoch 0: |                                                                                                                                                                                      | 709/? [35:23<00:00,  0.33it/s, v_num=edu3]train step 710; scene = ['7a1a5b280aeed1d1', '3ca192a6f7bd3055', 'b87f3bd2e5b947d9', 'b08fff272e1c806e', 'e40db329b18cfe34', '96405403953c2a44', '9d9e27f41396c20c', 'cb7ba36fbae70c01', '721707ada01bdffe', 'aa9daba10b590900', '0209fa7ef8a06022', '2f0a6c64c44f9979', '89ad52b52eb4ba60', '885587f42ee4e11e', '88b8bb15bd256558', '3d928d1613742585']; context = [[83, 108], [43, 68], [93, 118], [132, 157], [90, 115], [3, 28], [7, 32], [37, 62], [99, 124], [8, 33], [4, 29], [121, 146], [10, 35], [13, 38], [9, 34], [9, 34]]; loss = 0.063785
+Epoch 0: |                                                                                                                                                                                      | 719/? [35:52<00:00,  0.33it/s, v_num=edu3]train step 720; scene = ['ee3c9edbf6eedea9', '8a39c6b779b31dd4', '68320cab3a271ea3', '43544bc50a0d09ee', '494a0bebf7e6139e', '8a37947c1ad8c097', 'b20f139496853291', '14a700b34bfafc6f', 'fe1fe887ee20d877', '10491baf9035e557', '651e203d4e42216f', 'a2b08d5c241d54ec', '3c1022cc42279d6b', '218723a8059f18ac', '9f4b04caeaccc100', '26197c5e7d9e117e']; context = [[178, 203], [6, 31], [112, 137], [30, 55], [47, 72], [82, 107], [32, 57], [15, 40], [60, 85], [22, 47], [58, 83], [5, 30], [0, 25], [6, 31], [1, 26], [53, 78]]; loss = 0.296285
+Epoch 0: |                                                                                                                                                                                      | 729/? [36:23<00:00,  0.33it/s, v_num=edu3]train step 730; scene = ['a57232a4ef2ec923', '564a9a401e37c50b', '53e05632cb9141bd', 'df869fb963216261', '6744c1cf4e4dee91', '509496bad5a12f79', '06917bbe38912ea3', 'e92e4346a6224492', '9a725436379352e6', '5e55a38946593552', 'c5603a5e8498a827', 'f5338d00f9c021a3', '2cc2af36e6bb9fed', '547ebf48151eef4d', 'f603fb7c184a9103', '31304e5e03a0a335']; context = [[54, 79], [63, 88], [3, 28], [8, 33], [186, 211], [20, 45], [72, 97], [20, 45], [21, 46], [69, 94], [29, 54], [179, 204], [7, 32], [23, 48], [73, 98], [123, 148]]; loss = 0.957525
+Epoch 0: |                                                                                                                                                                                      | 739/? [36:52<00:00,  0.33it/s, v_num=edu3]train step 740; scene = ['f8ff67683c2323b8', '070e03b0709a0c7a', 'dfb585600cfa6d6e', '8062e068d364879e', 'bbb9d583807930d7', 'e68962be9d5fa78b', 'c098a117d37c1b11', '75d62d0707d1240e', 'c8b2e4d3f9fe541a', '67353f28c5452f17', 'd4aac0088971ad5d', 'a39b07c625f4fbaf', '086fc9b0951ac9ff', '42e11bb83f385317', 'adaa895408584f36', 'f52ea4df10c6472b']; context = [[16, 41], [162, 187], [116, 141], [92, 117], [25, 50], [46, 71], [224, 249], [157, 182], [18, 43], [76, 101], [67, 92], [80, 105], [51, 76], [56, 81], [150, 175], [6, 31]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 749/? [37:21<00:00,  0.33it/s, v_num=edu3]train step 750; scene = ['3ca9f2ea058b7220', '032961ac14c9b53a', '90a599cbae3f7f49', '4be99782bb201a87', 'd5a3940bb6a3363e', '31f802c4d94b2a8c', 'fb46a820975c2449', '17305b0b98053dee', '9c437a1ff26fc63d', '4bf92848b7ed1bcd', 'df21bcc5b49bc2bd', '3de485eddabc0ae8', '39d39ebdd5959a48', 'f7fd59b61fd90975', '77cba126c83354bb', '23997b17e26ba688']; context = [[7, 32], [15, 40], [168, 193], [24, 49], [134, 159], [84, 109], [120, 145], [227, 252], [202, 227], [42, 67], [1, 26], [11, 36], [142, 167], [28, 53], [3, 28], [17, 42]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 759/? [37:50<00:00,  0.33it/s, v_num=edu3]train step 760; scene = ['f00a0701904b2128', 'c4bfdf6f3a6c774d', 'baaba9c7644ef952', '55671e2db4657b00', 'b4bbab891a72ff1f', '390c8df752d7b22e', '756790007006f209', '05846e2bbc1ea5e0', '60f92bcb07109c4a', '0cd0baf1a9e3847a', 'b3e02abe2e692d4f', 'f35b530672b42ba5', 'f8e32278bb006569', '693db81439adb0d3', '65410d22f0d980f7', 'fd8dec4cd31f64b8']; context = [[1, 26], [35, 60], [43, 68], [158, 183], [0, 25], [70, 95], [26, 51], [3, 28], [6, 31], [9, 34], [0, 25], [42, 67], [96, 121], [24, 49], [32, 57], [6, 31]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 769/? [38:19<00:00,  0.33it/s, v_num=edu3]train step 770; scene = ['ab6103c00d052f04', '7f4c7bd5a9e3cddd', '331af0d1363e80b3', '4d5af03e77b032ba', '95307cda066e2664', '1dbd139488ac84b3', '8b7378a3996ee778', '26b8065adc153c13', '2a4fe9d5f507d831', '548ad1b923e633d8', '92a1f3ab5dbe634e', '4e437d66420067c2', 'c3165a339b9a09e5', '4e4985c5a76ac0e5', '5ac80497fad02a9a', 'a0f148d59958f742']; context = [[5, 30], [200, 225], [46, 71], [73, 98], [150, 175], [21, 46], [8, 33], [0, 25], [67, 92], [171, 196], [0, 25], [30, 55], [101, 126], [15, 40], [85, 110], [38, 63]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 779/? [38:49<00:00,  0.33it/s, v_num=edu3]train step 780; scene = ['1d80884a4b201f28', '1fba7a1a16edf7db', '2bf2121510674b04', '04c4c5a25b4ccb21', '17d74baebc458fb9', 'edf8fdfe10fb893c', '77a0fdc6389de36b', '90686fbf5ca9bd0a', '232da3c3c432bcf5', '2804817967c9ab8b', '34769f82ff4b286c', '4df2a98cf2678d8c', '2eacf9db1843546b', 'c1f32a7a7ec37d39', '6187332401a328eb', 'c495f01f294333ee']; context = [[23, 48], [107, 132], [107, 132], [22, 47], [53, 78], [186, 211], [46, 71], [75, 100], [140, 165], [166, 191], [49, 74], [106, 131], [14, 39], [28, 53], [164, 189], [23, 48]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 789/? [39:18<00:00,  0.33it/s, v_num=edu3]train step 790; scene = ['0cbf1454a4391523', 'f3b24cf238154fc0', '9bb0448e7cb2f733', 'dff181c503e5c202', 'f0b0c2b2fa790b94', '4b30f3b820875a1a', '4a218ddfc3698ca8', 'bc34ecfb3c809d97', 'bca757afa6825a39', '75f7df7e153bb467', '51ba892c38152086', '84335596c296225a', 'c121bf6c2929cfe1', '1e796863c062efaa', '995f249ace489797', 'edc39c2e54f4dd24']; context = [[124, 149], [22, 47], [28, 53], [108, 133], [12, 37], [28, 53], [43, 68], [67, 92], [13, 38], [45, 70], [69, 94], [126, 151], [30, 55], [55, 80], [7, 32], [5, 30]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 799/? [39:47<00:00,  0.33it/s, v_num=edu3]train step 800; scene = ['b1818f89f71d7ede', '1dc0f9200778b6ff', '8bb9dfb202abb4c0', 'd8cd8e74983c2ed5', '9c8a304979cbc7da', 'f00e57d3e757b12e', '96ba226de3006d1b', '49cdcdc8c9fe71f2', 'b7e287b139583003', '04221c6266b4fbb5', '11db2e14d64c1b42', '51f80353f6008c4f', '564437b001139667', '5b7e5ce68733bcc4', '7850797441a57de9', '12048355d2052d78']; context = [[58, 83], [31, 56], [44, 69], [97, 122], [1, 26], [159, 184], [101, 126], [26, 51], [63, 88], [91, 116], [114, 139], [110, 135], [118, 143], [64, 89], [69, 94], [46, 71]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 809/? [40:16<00:00,  0.33it/s, v_num=edu3]train step 810; scene = ['f841958b4f3ea9b6', 'b5a711e9d6e9f7d2', '81239a4d26b89db3', '0c55d99325327032', '0fa3084a8ad67f81', '1766b288f284c234', '792c3b369bcd98f6', '600aa1de0c963a27', '7d81b28b7afe5738', '6b93a9fa1ce08986', '71843154fd778989', '6e024fbe00be60cb', '1cacfe69c509febf', '2765f5d2bd45f599', '591324b1a786d91e', '04740510b6d96720']; context = [[36, 61], [71, 96], [74, 99], [25, 50], [123, 148], [137, 162], [45, 70], [1, 26], [243, 268], [32, 57], [23, 48], [55, 80], [87, 112], [36, 61], [67, 92], [16, 41]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 819/? [40:45<00:00,  0.33it/s, v_num=edu3]train step 820; scene = ['a0cd80f2815b8716', 'e649e3010fab6d40', '3f4cacc2a74df04e', 'b887116c5ac0c3ae', '3bdacd796a82e581', '1631053dcc53cb87', 'b555afebfcce407c', '2405f8024d2d8efa', 'b31dae3a8edd6af8', 'b12f99acac988ca4', '6df08dcd841c4b9b', 'fadf8681f71b6225', '2a16f86d8c47f471', '082c57765925eb2b', '02c39509dcd964e8', '9de33e466a02ea23']; context = [[92, 117], [18, 43], [5, 30], [3, 28], [108, 133], [53, 78], [145, 170], [55, 80], [38, 63], [48, 73], [38, 63], [21, 46], [6, 31], [7, 32], [105, 130], [143, 168]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 829/? [41:15<00:00,  0.33it/s, v_num=edu3]train step 830; scene = ['0841dc352b358629', '9308b0309e860d89', '804c26118696da0b', '6f29d2f4950be336', 'de40426a00dab49a', '546bf84eac7404e8', '11730d25b6f6361b', 'f1f5e26cb49fd2a5', '15cd8f43f173d0e3', 'e1fbf58a49076d36', '2d4b0725e43dc211', 'c2923fa433879e60', 'cae0139a521aa052', '08127a499408b79e', '591bf6ac3800ca21', '811e5be9bf8e26ca']; context = [[110, 135], [51, 76], [6, 31], [2, 27], [36, 61], [11, 36], [33, 58], [0, 25], [10, 35], [9, 34], [240, 265], [175, 200], [15, 40], [21, 46], [5, 30], [43, 68]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 839/? [41:44<00:00,  0.34it/s, v_num=edu3]train step 840; scene = ['f7d5f1a8df9d773f', 'e426679594077686', '1d954e607bc2a8f6', '2290ecbf83cd253e', '9c7c42260596ac3d', '0482811bdc4713a5', '5e78458cfb6b8150', '61327ef41f3b72f1', '74e1a98290b34a81', 'ecc15d661d8dbe9f', 'cfec50872cec53bc', '6a4fe529a00082f2', '6a6beb90616e4f37', '42c7147c591f47cb', 'b05e5158361732c9', '5f22fdf235746488']; context = [[32, 57], [103, 128], [87, 112], [42, 67], [47, 72], [143, 168], [37, 62], [2, 27], [53, 78], [71, 96], [4, 29], [48, 73], [9, 34], [3, 28], [177, 202], [10, 35]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 849/? [42:13<00:00,  0.34it/s, v_num=edu3]train step 850; scene = ['ceba0c9064ef2075', 'f249fdd072d2d35d', '93f0ba6797c67f20', '9122be71240bb162', '9168f179a84dcc90', '2d5886bd890632c6', '7e72a2e96b39c149', '2ca7a809dda33d8e', 'c0ee12e05d7032af', '01d4d24e9f33aa8f', '0cf6039feb87cb09', '825d0b6a944ea867', 'b811e125c84adff1', '3ffc4b7822281c60', '18711838050bef0d', 'fcfd650da12e293f']; context = [[179, 204], [28, 53], [58, 83], [153, 178], [37, 62], [244, 269], [33, 58], [179, 204], [25, 50], [52, 77], [4, 29], [14, 39], [22, 47], [25, 50], [107, 132], [150, 175]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 859/? [42:42<00:00,  0.34it/s, v_num=edu3]train step 860; scene = ['a221a85eb7d833b1', '8985d94d0dd98a65', 'dbf748364163182e', 'd5c1933c67b49875', 'f73708f8f2c1fcc0', '97133598e9144794', '4c4fce2df53535d6', 'ac1e985cf3a2df12', 'ca90cf157ecaadfe', 'c8b8300236a7220a', 'faaa1fa7767acf25', '0ab79d1b1d783e01', 'a7621aea5c101536', '7f11c968deb89da5', 'cc039d2dde109d6f', 'a06b2802ca08da31']; context = [[32, 57], [41, 66], [101, 126], [53, 78], [36, 61], [97, 122], [149, 174], [17, 42], [40, 65], [32, 57], [2, 27], [5, 30], [32, 57], [127, 152], [175, 200], [67, 92]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 869/? [43:11<00:00,  0.34it/s, v_num=edu3]train step 870; scene = ['96fcce02a7674e02', '9a9367ff87294587', 'aaa8dfa7e8498bd9', '5019925808a59723', 'f5934f621b0a002a', 'db77e4c30e954d3f', '23c177f84829d2a6', 'ecc88f03e2e5ef81', 'fb331a491ca2e5e0', '1a2ea561c9c8f419', '3a13e56691c8396e', 'bc67854fca5cdac2', '59e04cb072686a97', '31ba0028326b8b17', '82b8b4c5e983729f', '1b8abd07e10f4e79']; context = [[231, 256], [2, 27], [99, 124], [115, 140], [126, 151], [122, 147], [188, 213], [45, 70], [3, 28], [9, 34], [31, 56], [34, 59], [167, 192], [14, 39], [251, 276], [17, 42]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 879/? [43:41<00:00,  0.34it/s, v_num=edu3]train step 880; scene = ['5e70f8dc0da2945f', '4b7302601da9f913', '6f8ab470e569eced', '4886ce15c821b292', 'a30a83cc257714b2', '774d16d7b7d3ab87', 'c279649ebcdbd4ac', 'e1b0771ca0e3df03', '191f33c96812c34d', '43668ae8f0e517be', '176f7bab982f3aa4', 'e41818c1a640f355', '3670ced044c09f1c', '16e752a0b2952804', '37311e860b1ef72c', 'de5195b1d4b5e650']; context = [[24, 49], [187, 212], [20, 45], [15, 40], [52, 77], [55, 80], [22, 47], [53, 78], [46, 71], [41, 66], [99, 124], [189, 214], [10, 35], [97, 122], [197, 222], [38, 63]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 889/? [44:10<00:00,  0.34it/s, v_num=edu3]train step 890; scene = ['b41d3898f57085e3', '1182579e7ca4ef88', 'e7b0248022eae778', 'b8e91cbe57d886af', 'a308376472392fc6', '011fbca4c333495f', 'c38816ffe6b2c633', '2598b88b7b7d2119', '683c969b32f1b4b6', '1fefe15b477db1e5', '35d9cc4d7e5487a1', 'e5537547dd598187', '32261dd2941cfcae', 'c0e387d3a6285a32', '0cb47bf3e9ed3aa5', 'e55496b6c4f860e5']; context = [[77, 102], [39, 64], [32, 57], [3, 28], [9, 34], [4, 29], [182, 207], [35, 60], [86, 111], [81, 106], [236, 261], [13, 38], [148, 173], [102, 127], [36, 61], [25, 50]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 899/? [44:39<00:00,  0.34it/s, v_num=edu3]train step 900; scene = ['d44eb5d03836bfa8', 'd312027b19525a5a', 'ddabc74cbdf2849b', '72040346a65ca209', '06f384602bc7e787', 'de2ae796f06043da', '51b04fd9435b6853', '74b69334da4939e2', '7257497d7ddefa3f', 'da0f564e5210d09f', 'fb862259dcc8a0aa', '8eb77501b906faf0', '047b9960d19e3c60', '0492445f09a9bb4b', 'd7699c99c299376f', '87dcad4e125ff354']; context = [[0, 25], [0, 25], [0, 25], [242, 267], [1, 26], [126, 151], [54, 79], [72, 97], [61, 86], [69, 94], [23, 48], [74, 99], [153, 178], [213, 238], [90, 115], [160, 185]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 909/? [45:08<00:00,  0.34it/s, v_num=edu3]train step 910; scene = ['af5bc2b9cb5f4dd5', '70e06d0ca9ba12d7', '7cae2e82f768bd58', '75d09be60110e2cd', '8c6613326c9bb024', 'd20cec74d90a31b2', '28cc23ff81e8c7cf', '3c8b0bb757a118b9', '1ffe77b1a0efd127', '7fa1e8ca5ff1aa0a', 'c201594babef23f6', '4e5a24edeab27bde', '0081cfd790d7ad74', 'c8c1e40f50b1ef9d', 'e73b93ff29fb6e8c', '8d19f2c27abf126c']; context = [[1, 26], [145, 170], [133, 158], [21, 46], [79, 104], [45, 70], [181, 206], [37, 62], [0, 25], [66, 91], [6, 31], [121, 146], [16, 41], [158, 183], [18, 43], [68, 93]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 919/? [45:38<00:00,  0.34it/s, v_num=edu3]train step 920; scene = ['d5788ecb38219348', '75840717ecdc2325', 'afb5096c52233388', '706571cd6a9c16a4', 'd7798982569c2714', '334f192d01193f36', '6f3b47cc1831979d', '81124fd6dfc66253', '6131f8c669e8060f', '2b91070adf840bcb', '89d2c2d9e3aee457', '21691784e9e84e35', 'ded38d3ae876e8dc', '0a1aa875aca3711d', '9005d1006b63b2ab', 'b4c2e12ad3d75efd']; context = [[209, 234], [47, 72], [31, 56], [17, 42], [72, 97], [49, 74], [51, 76], [55, 80], [97, 122], [27, 52], [21, 46], [49, 74], [3, 28], [11, 36], [113, 138], [7, 32]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 929/? [46:07<00:00,  0.34it/s, v_num=edu3]train step 930; scene = ['51832326c2486521', 'ae91ec6e3aec00c2', 'e80172c489f74813', 'e034bdb73350efdb', 'b2e5932f5b358c92', 'f80af1dc6cf2854c', 'f8e3b85a21b25764', 'cac8ff0742e1ef7e', '6e6ce6a006f044a6', '9aba38d959e85c32', 'd68ad9834919913a', 'de11e8525444b3e6', '7770c9fd63ae19d3', '929ac3c28edde369', 'fb807a4a935011f1', 'bc920b78aab35655']; context = [[56, 81], [59, 84], [176, 201], [3, 28], [63, 88], [27, 52], [30, 55], [187, 212], [67, 92], [63, 88], [94, 119], [39, 64], [49, 74], [3, 28], [12, 37], [53, 78]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 939/? [46:37<00:00,  0.34it/s, v_num=edu3]train step 940; scene = ['6792d6a4f4e87e24', '06f60580eba61d9d', '42c7147c591f47cb', '5d167006936398f2', '5f22fdf235746488', '337c66222a513af2', 'ecc15d661d8dbe9f', '2d417198a5ebd20e', '6a6beb90616e4f37', 'cfec50872cec53bc', '8b3765816a0cb218', 'fa3c7a9b31e5ab10', '6ad2c6f0c0c903e8', '96fbaabf762077c5', 'af37a973266b99fc', '00d80d0e284dac72']; context = [[41, 66], [25, 50], [13, 38], [15, 40], [37, 62], [0, 25], [45, 70], [66, 91], [89, 114], [30, 55], [24, 49], [22, 47], [53, 78], [16, 41], [37, 62], [13, 38]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 949/? [47:06<00:00,  0.34it/s, v_num=edu3]train step 950; scene = ['d1a496112cee9a32', 'd069c35354465fa9', '1db855eb2be5b983', '09a1e9bbded02341', '3918e4998619b227', '2a23966bfac99977', 'b6f65429632f92fa', '14270bac7ebd4ce4', 'f372609a6c75bced', 'b4a46c5d2d8d0415', '30e1d420b926d6f0', '8902e97c5814433e', '161cc1c3c7d7f131', '52ba723e1b5e31da', '7bf71b62cccdf2fa', 'b23cbee05407661d']; context = [[59, 84], [0, 25], [129, 154], [59, 84], [127, 152], [84, 109], [45, 70], [30, 55], [79, 104], [29, 54], [30, 55], [14, 39], [8, 33], [15, 40], [12, 37], [131, 156]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 959/? [47:35<00:00,  0.34it/s, v_num=edu3]train step 960; scene = ['3ba287815c2fa8bd', '4d9de907289bb1b6', '90b3a044d5c4522e', 'da500b24da7a495b', '5205c4ab691fab79', '979583f22bb73aef', '8c42a0b02e4b80e7', '643a17e2e87792a1', '6af3dac3387f0c7c', '2b85a6b46f2ea8de', 'fd7e46c14085c76d', '789e60a632f1c397', '4bb8768dea446ee1', 'ae846b6a792cadc1', 'e72abbc2e978475b', '6a11871c9643bd79']; context = [[29, 54], [134, 159], [26, 51], [17, 42], [52, 77], [55, 80], [211, 236], [11, 36], [224, 249], [1, 26], [84, 109], [18, 43], [44, 69], [13, 38], [44, 69], [28, 53]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 969/? [48:05<00:00,  0.34it/s, v_num=edu3]train step 970; scene = ['da7b5b8c48f2b345', '800692cf789b69bb', 'eb8e1a343a57f246', 'ef63ce1150b93a2f', '6f8cd3c25e7ce532', '430828a73b0473c1', 'fae232a4edb86602', 'ab533c63e6b7ef78', '6f2daf5cb8151b38', 'cd2bc85e8f87f404', '363163aa8eb9a2dd', 'f6d48213ef0665a5', 'cd7a270da6f6da4a', '2018a20c3c41f278', '0274ae92b3b40648', '56f96532473be1b0']; context = [[4, 29], [83, 108], [63, 88], [22, 47], [1, 26], [2, 27], [109, 134], [36, 61], [133, 158], [18, 43], [62, 87], [56, 81], [22, 47], [95, 120], [208, 233], [74, 99]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 979/? [48:34<00:00,  0.34it/s, v_num=edu3]train step 980; scene = ['0f8c770f3a37cacf', '4e9bc83dc19ab35b', '02b4de9af6e29e8a', 'ff7c5e76dbb32bf8', 'fc0eb949f7e3495d', '809ac9d0df881cdc', '9b1748955fdd2d93', 'c62d89b4898ca134', '8fe0153671265cd8', 'cac682fd737ac05a', 'd5c8690ebede654c', '68f5a9c218ae58de', '6f734cb7648a8dc0', '3a06c662b556f35d', '2ab59b3cd18cf2a6', '7575c3ff61344cf7']; context = [[55, 80], [0, 25], [75, 100], [18, 43], [56, 81], [21, 46], [3, 28], [51, 76], [11, 36], [0, 25], [119, 144], [163, 188], [112, 137], [53, 78], [0, 25], [75, 100]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 989/? [49:04<00:00,  0.34it/s, v_num=edu3]train step 990; scene = ['065d41d6e7cd8ad8', 'fbc4bbe6225528ec', '3baeb8a657696aec', '0224ae3b7123367e', '61327b567aa8d6f6', '0856eb1e161c6cb8', 'e6406106735f8b34', '12108a3fd93471fb', '6080fc9345448eec', '39ee4bf580cf2bbe', '5a1ccf5da976f592', 'c2de98e8401d1883', '877e982c2aeacb3a', '9f7fa8a32d5eaa0f', 'd3b5f1786c3ae039', '28478600e3b81375']; context = [[38, 63], [125, 150], [40, 65], [231, 256], [11, 36], [44, 69], [226, 251], [29, 54], [48, 73], [119, 144], [25, 50], [3, 28], [40, 65], [8, 33], [0, 25], [1, 26]]; loss = nan
+Epoch 0: |                                                                                                                                                                                      | 999/? [49:33<00:00,  0.34it/s, v_num=edu3]train step 1000; scene = ['d31dab63cc54c59e', '767c62f619b15386', '95c9d9e0f42b1a15', '7b698e19de628b10', 'dca0f0eacbccd9c8', '6a98a5412a5aee02', '58862a8fe2c91894', '6bb1fe38f50ee2f6', '5e8552f9cdf1e078', '344337509a48a21c', '3220f882c1f72cca', '272f7a997c94d84f', '6af18067925fa678', 'b12b1204a0c7c197', '5db240a3c54df22b', '28ada2a08238d1c8']; context = [[3, 28], [43, 68], [3, 28], [10, 35], [13, 38], [40, 65], [17, 42], [5, 30], [67, 92], [15, 40], [77, 102], [151, 176], [33, 58], [90, 115], [41, 66], [24, 49]]; loss = nan
+Epoch 0: |                                                                                                                                                                                     | 1000/? [49:36<00:00,  0.34it/s, v_num=edu3]Validation epoch start on rank 0
+Validation: |                                                                                                                                                                                                         | 0/? [00:00<?, ?it/s]validation step 1000; scene = ['73d6f935f31b3fd4']; context = [[41, 66]]
+target intrinsic:  tensor(0.8576, device='cuda:0') tensor(0.8579, device='cuda:0')                                                                                                                                    | 0/1 [00:00<?, ?it/s]
+pred intrinsic:  tensor(nan, device='cuda:0') tensor(nan, device='cuda:0')
+No valid depth values found.
+Epoch 0: |                                                                                                                                                                                     | 1009/? [50:03<00:00,  0.34it/s, v_num=edu3]train step 1010; scene = ['917a89fdc7272009', '5eee3bba5d8a0af0', '09857c20a9b14df3', '184fb534f9aca4da', '51e6b1d7cd013f21', 'aa85d24bbd87a8a6', '10c6540f2e3af25b', '9159925b118937a4', '13c85315b9e987d9', '3075c400813b4fd5', 'e0fc35891bf29a7a', '8b5c24358ec92b0c', '8376f898eabbd0e1', '8da41667e8f33d92', '0ddb1fa8f7729c9f', '9d714edfee008fc2']; context = [[48, 73], [76, 101], [100, 125], [36, 61], [193, 218], [190, 215], [49, 74], [200, 225], [204, 229], [56, 81], [12, 37], [138, 163], [47, 72], [101, 126], [8, 33], [25, 50]]; loss = nan
+Epoch 0: |                                                                                                                                                                                     | 1019/? [50:33<00:00,  0.34it/s, v_num=edu3]train step 1020; scene = ['06c7cb360e434de5', '68d0e78200de8dbf', '7567fe94d4fba9d7', 'a0493b972780d9ae', 'dbbd2d62968c2aab', '7eb1c8969766243e', 'abd353eabd0dea01', 'b6cacdcdab5755b8', '7e731568d0288a55', '085ff8ed78f58252', 'b6f08954c7c64f59', '700d4f6b814c5dd7', '770ffcd0c79519aa', 'aa3520c25c879a9f', '9530c8a093a39160', '3195acecb19b0955']; context = [[39, 64], [129, 154], [28, 53], [65, 90], [6, 31], [0, 25], [112, 137], [3, 28], [214, 239], [156, 181], [6, 31], [18, 43], [40, 65], [53, 78], [10, 35], [215, 240]]; loss = nan
+Epoch 0: |                                                                                                                                                                                     | 1029/? [51:01<00:00,  0.34it/s, v_num=edu3]train step 1030; scene = ['1ad7adae6a042ce1', 'df0ac86df48f6d06', 'cc6c8d74122beb91', 'cbb91c77a7526d61', '4edcc58025a77735', 'b1dd1726f1f46b42', '15f14a9a8d3f7769', 'dacf77dd43f1d0f1', 'e53a82bfe84a2251', '1018776739e1250b', '0c52454885189188', '7e25e7669443bd73', 'cabe1e553676170f', '29fabf56364e9d1b', 'e58feef82356fb26', 'e2202f60a31710bc']; context = [[40, 65], [207, 232], [70, 95], [25, 50], [18, 43], [63, 88], [11, 36], [53, 78], [0, 25], [34, 59], [79, 104], [111, 136], [95, 120], [49, 74], [29, 54], [180, 205]]; loss = nan
+Epoch 0: |                                                                                                                                                                                     | 1039/? [51:31<00:00,  0.34it/s, v_num=edu3]train step 1040; scene = ['e28ac2de4b57c0fd', '71499e389bdaea92', 'd7c0ebc89793e158', 'b428645647d53e65', '1c1f2b158b8527cc', '5a23762e2edbeba0', '581d57e13e40094d', 'cb64aaa680fba010', '3f7c40c559810888', '2edb7ac3009c5f9e', 'f7887b5d47d53d20', 'a79888bb552c7862', 'cb7487a5189f6b2b', 'dd71da9a017228df', 'fdbfe35f5940d3ad', '4a7a158bc511d737']; context = [[5, 30], [54, 79], [64, 89], [69, 94], [25, 50], [14, 39], [32, 57], [101, 126], [80, 105], [17, 42], [46, 71], [204, 229], [10, 35], [104, 129], [101, 126], [119, 144]]; loss = nan
+Epoch 0: |                                                                                                                                                                                     | 1049/? [52:01<00:00,  0.34it/s, v_num=edu3]train step 1050; scene = ['200afae0928a1372', '88402eba1dca60a2', '7fbbee2db2e6cd4d', 'd23843ee6485a4ca', 'a2c884e396d7f501', 'ebaeb44a88de9332', '6af98c375abdb63d', '38c220ff58e572ac', '527265924b9b6375', '2962175a510cf868', '94418a5db600cc33', 'ff476ba779fd6021', '518200a52559d895', 'ca342b60ac94b65e', '0d4ebded1cdfa14c', '7f9be765fc88ada1']; context = [[140, 165], [72, 97], [48, 73], [24, 49], [10, 35], [18, 43], [61, 86], [106, 131], [14, 39], [4, 29], [16, 41], [86, 111], [7, 32], [199, 224], [39, 64], [23, 48]]; loss = nan
+Epoch 0: |                                                                                                                                                                                     | 1059/? [52:29<00:00,  0.34it/s, v_num=edu3]train step 1060; scene = ['eb5a6e0843067cdc', '3ebbf175cda595a2', '1cf5589d5ecddb95', '74eb411688ba5e15', 'b071ad8689db9c26', '5c50277359a0843c', '26ea8cc2791328c2', 'ac64989dd2b2a22a', '6955cf41eb23fe3b', '39f7c76ef42ac1d4', '0f04f8ee8b2396c7', 'aaece70a39dd3257', '0d8f65989586d136', '7fde9a464e6b2a7b', '55f0fab213db4673', '104d225e8ec52733']; context = [[27, 52], [21, 46], [109, 134], [53, 78], [205, 230], [27, 52], [23, 48], [77, 102], [1, 26], [125, 150], [89, 114], [138, 163], [9, 34], [98, 123], [25, 50], [17, 42]]; loss = nan
+Epoch 0: |                                                                                                                                                                                     | 1069/? [52:59<00:00,  0.34it/s, v_num=edu3]train step 1070; scene = ['e6d924f92f44c985', 'c96388f9522a65b9', '3193e8bf21491023', 'b86dc6a2be5101ab', '42df8c69e0daeec6', '95908390cfcab857', 'ebf1a7e113d1cfd2', 'c8a89452aa4cf54f', 'c5f6f1a662e28071', '3678468a9d6d42cc', 'b950cd4348d01e2a', '623d5619e4702864', '804998e7ec39d253', '0d75be966f561519', '0d32af0aa9eaae9a', '68528e468bf52aca']; context = [[83, 108], [123, 148], [72, 97], [18, 43], [43, 68], [24, 49], [217, 242], [44, 69], [180, 205], [114, 139], [46, 71], [90, 115], [50, 75], [34, 59], [15, 40], [53, 78]]; loss = nan
+Epoch 0: |                                                                                                                                                                                     | 1079/? [53:28<00:00,  0.34it/s, v_num=edu3]train step 1080; scene = ['5fe123b3d2013cfc', '05c25202589a6f81', 'c6cdc4c9282bd491', '8b3e50b64cdb9a4a', '34a9201614a19005', 'b765858262583af6', '7f20b3df52cf4e98', '43c619a8b35c357f', '32ed60ed023b59cf', '0560b7d07f6e6899', '262a536e25e2ae43', '572d67c3862dd6e1', 'd59ab97b03d1164a', '46480458ef51b393', '03d3092e230ed7f0', '79f4ef188c1f0a3d']; context = [[10, 35], [8, 33], [83, 108], [21, 46], [107, 132], [205, 230], [25, 50], [43, 68], [15, 40], [0, 25], [252, 277], [212, 237], [1, 26], [27, 52], [0, 25], [42, 67]]; loss = nan
+Epoch 0: |                                                                                                                                                                                     | 1089/? [53:58<00:00,  0.34it/s, v_num=edu3]train step 1090; scene = ['b333ae137d9b364d', '111edff73abfa215', '03150a5a24b88cf4', 'fa3c714f9feb0ca1', 'b4a5824300555fe9', '427dbba2f4ddb5ec', '423fb23a46b7ad6e', '9589bf6c34437171', 'ed90fc76c5f364e8', 'e81375f276552936', 'ec9ce5f9d9512215', 'fbf467b7610b569c', '49e4f5365be165ee', '34580d00589eee64', '3205bb1ec6c5e8a4', '24a2c0442c06f4ec']; context = [[126, 151], [8, 33], [183, 208], [109, 134], [41, 66], [20, 45], [18, 43], [57, 82], [0, 25], [5, 30], [61, 86], [10, 35], [5, 30], [70, 95], [20, 45], [175, 200]]; loss = nan
+Epoch 0: |                                                                                                                                                                                     | 1099/? [54:26<00:00,  0.34it/s, v_num=edu3]train step 1100; scene = ['d8c0b4ca0defd965', '73a06272f1314473', 'd5ae945909789ef7', 'da86c1e65642aa38', '37c5b704883ea1b7', '6d49fec0fd3dbea0', 'bcede31352c7bd4d', 'bb5bb772c9c01b1a', '309060e859c57cc7', 'bf5caf81dc58798c', '10e11b3eb5bba7c9', 'a23531e747abeb94', 'd40a51a67fb79544', '0880f556a19d07f5', '577c560e66207737', '9b6a57a6ff02dd12']; context = [[245, 270], [19, 44], [67, 92], [48, 73], [8, 33], [128, 153], [40, 65], [28, 53], [67, 92], [51, 76], [17, 42], [41, 66], [207, 232], [114, 139], [6, 31], [20, 45]]; loss = nan
+Epoch 0: |                                                                                                                                                                                     | 1107/? [54:50<00:00,  0.34it/s, v_num=edu3]

wandb/run-20250916_181645-hudaedu3/files/requirements.txt

@@ -0,0 +1,129 @@
+ruff==0.11.4
+PyYAML==6.0.2
+sympy==1.13.3
+fsspec==2024.6.1
+imageio-ffmpeg==0.6.0
+setuptools==78.1.1
+timm==1.0.15
+fonttools==4.57.0
+nvidia-cublas-cu12==12.8.3.14
+nvidia-cuda-nvrtc-cu12==12.8.61
+dacite==1.9.2
+hydra-core==1.3.2
+certifi==2022.12.7
+rich==14.0.0
+filelock==3.13.1
+idna==3.4
+nvidia-cuda-cupti-cu12==12.8.57
+pip==25.2
+wandb==0.19.9
+ninja==1.11.1.4
+networkx==3.3
+GitPython==3.1.44
+moviepy==1.0.3
+markdown-it-py==3.0.0
+lightning==2.5.1
+Jinja2==3.1.4
+nvidia-cusparse-cu12==12.5.7.53
+tabulate==0.9.0
+tifffile==2025.3.30
+nvidia-nvtx-cu12==12.8.55
+jaxtyping==0.2.37
+aiohappyeyeballs==2.6.1
+MarkupSafe==2.1.5
+pydantic_core==2.33.1
+mypy-extensions==1.0.0
+lazy_loader==0.4
+pydantic==2.11.2
+typing-inspection==0.4.0
+pillow==10.4.0
+nvidia-nvjitlink-cu12==12.8.61
+omegaconf==2.3.0
+scikit-image==0.25.2
+jaraco.context==5.3.0
+typing_extensions==4.12.2
+mpmath==1.3.0
+opencv-python==4.11.0.86
+backports.tarfile==1.2.0
+async-timeout==5.0.1
+opt_einsum==3.4.0
+nvidia-cuda-runtime-cu12==12.8.57
+numpy==1.26.4
+torch==2.7.1+cu128
+psutil==7.0.0
+pytorch-lightning==2.5.1
+protobuf==5.29.4
+imageio==2.37.0
+colorama==0.4.6
+attrs==25.3.0
+importlib_metadata==8.0.0
+frozenlist==1.5.0
+beartype==0.19.0
+propcache==0.3.1
+typeguard==4.3.0
+torchaudio==2.7.1+cu128
+decorator==4.4.2
+nvidia-cudnn-cu12==9.7.1.26
+torchvision==0.22.1+cu128
+sentry-sdk==2.25.1
+mdurl==0.1.2
+more-itertools==10.3.0
+einops==0.8.1
+zipp==3.19.2
+antlr4-python3-runtime==4.9.3
+cycler==0.12.1
+nvidia-cufile-cu12==1.13.0.11
+jaraco.collections==5.1.0
+jaraco.text==3.12.1
+torchmetrics==1.8.2
+packaging==24.2
+black==25.1.0
+setproctitle==1.3.5
+Pygments==2.19.2
+tomli==2.2.1
+six==1.17.0
+docker-pycreds==0.4.0
+inflect==7.3.1
+requests==2.28.1
+svg.py==1.6.0
+python-dateutil==2.9.0.post0
+proglog==0.1.11
+gsplat==1.5.3
+e3nn==0.5.6
+pathspec==0.12.1
+aiosignal==1.3.2
+wheel==0.45.1
+multidict==6.3.2
+pyparsing==3.2.3
+plyfile==1.1
+jaraco.functools==4.0.1
+nvidia-nccl-cu12==2.26.2
+triton==3.3.1
+nvidia-cusparselt-cu12==0.6.3
+scikit-video==1.1.11
+huggingface-hub==0.30.1
+lightning-utilities==0.14.3
+wadler_lindig==0.1.4
+nvidia-curand-cu12==10.3.9.55
+matplotlib==3.10.1
+kiwisolver==1.4.8
+tqdm==4.67.1
+safetensors==0.5.3
+scipy==1.15.2
+lpips==0.1.4
+nvidia-cusolver-cu12==11.7.2.55
+click==8.1.8
+charset-normalizer==2.1.1
+platformdirs==4.3.7
+opt-einsum-fx==0.1.4
+annotated-types==0.7.0
+smmap==5.0.2
+gitdb==4.0.12
+nvidia-cufft-cu12==11.3.3.41
+contourpy==1.3.1
+urllib3==1.26.13
+autocommand==2.2.2
+yarl==1.19.0
+aiohttp==3.11.16
+colorspacious==1.1.2
+python-dotenv==1.1.0

wandb/run-20250916_181645-hudaedu3/files/wandb-metadata.json

@@ -0,0 +1,79 @@
+{
+  "os": "Linux-5.15.0-153-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.12.11",
+  "startedAt": "2025-09-16T10:16:46.030205Z",
+  "args": [
+    "+experiment=re10k",
+    "wandb.mode=online",
+    "wandb.name=re10k_hpc"
+  ],
+  "program": "-m src.main",
+  "git": {
+    "remote": "git@github.com:K-nowing/ICLR2026_HCP.git",
+    "commit": "6c0080eae0295d43dd7f18fd71e742d924238f89"
+  },
+  "email": "dna9041@korea.ac.kr",
+  "root": "/root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc",
+  "host": "notebook-72e474dd-92e1-11f0-a6ea-7ab79f1c53cb-0",
+  "executable": "/opt/conda/envs/ours/bin/python",
+  "cpu_count": 144,
+  "cpu_count_logical": 288,
+  "gpu": "NVIDIA B200",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1081101176832",
+      "used": "211822145536"
+    }
+  },
+  "memory": {
+    "total": "2434690400256"
+  },
+  "cpu": {
+    "count": 144,
+    "countLogical": 288
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    },
+    {
+      "name": "NVIDIA B200",
+      "memoryTotal": "192265846784",
+      "cudaCores": 18944
+    }
+  ],
+  "cudaVersion": "12.8"
+}

wandb/run-20250916_181645-hudaedu3/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-09-16T18:16:45.655897223+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpozwaujmt/port-11845.txt","pid":11845,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-09-16T18:16:45.659989532+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":11845}
+{"time":"2025-09-16T18:16:45.659989532+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":33483,"Zone":""}}
+{"time":"2025-09-16T18:16:45.735655232+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:57558"}
+{"time":"2025-09-16T18:16:46.033140177+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"hudaedu3","id":"127.0.0.1:57558"}
+{"time":"2025-09-16T18:16:46.500311533+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"hudaedu3","id":"127.0.0.1:57558"}
+{"time":"2025-09-16T19:11:51.048775677+08:00","level":"INFO","msg":"Parent process exited, terminating service process."}

wandb/run-20250916_181645-hudaedu3/logs/debug-internal.log

@@ -0,0 +1,7 @@
+{"time":"2025-09-16T18:16:46.035459068+08:00","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"/root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc/wandb/run-20250916_181645-hudaedu3/logs/debug-core.log"}
+{"time":"2025-09-16T18:16:46.500077696+08:00","level":"INFO","msg":"created new stream","id":"hudaedu3"}
+{"time":"2025-09-16T18:16:46.500270254+08:00","level":"INFO","msg":"stream: started","id":"hudaedu3"}
+{"time":"2025-09-16T18:16:46.500517891+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"hudaedu3"}
+{"time":"2025-09-16T18:16:46.500710488+08:00","level":"INFO","msg":"sender: started","stream_id":"hudaedu3"}
+{"time":"2025-09-16T18:16:46.500985645+08:00","level":"INFO","msg":"handler: started","stream_id":"hudaedu3"}
+{"time":"2025-09-16T18:16:46.896044377+08:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20250916_181645-hudaedu3/logs/debug.log

@@ -0,0 +1,22 @@
+2025-09-16 18:16:46,003 INFO    MainThread:11845 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9
+2025-09-16 18:16:46,004 INFO    MainThread:11845 [wandb_setup.py:_flush():67] Configure stats pid to 11845
+2025-09-16 18:16:46,004 INFO    MainThread:11845 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings
+2025-09-16 18:16:46,004 INFO    MainThread:11845 [wandb_setup.py:_flush():67] Loading settings from /root/highspeedstorage/injae/code/ICLR2026_HCP/wandb/settings
+2025-09-16 18:16:46,004 INFO    MainThread:11845 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-09-16 18:16:46,004 INFO    MainThread:11845 [wandb_init.py:setup_run_log_directory():662] Logging user logs to /root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc/wandb/run-20250916_181645-hudaedu3/logs/debug.log
+2025-09-16 18:16:46,005 INFO    MainThread:11845 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to /root/highspeedstorage/injae/code/ICLR2026_HCP/outputs/re10k_hpc/wandb/run-20250916_181645-hudaedu3/logs/debug-internal.log
+2025-09-16 18:16:46,005 INFO    MainThread:11845 [wandb_init.py:init():781] calling init triggers
+2025-09-16 18:16:46,005 INFO    MainThread:11845 [wandb_init.py:init():786] wandb.init called with sweep_config: {}
+config: {'model': {'encoder': {'backbone': {'name': 'croco', 'model': 'ViTLarge_BaseDecoder', 'patch_embed_cls': 'PatchEmbedDust3R', 'asymmetry_decoder': True, 'intrinsics_embed_loc': 'encoder', 'intrinsics_embed_degree': 4, 'intrinsics_embed_type': 'token'}, 'name': 'noposplat', 'opacity_mapping': {'initial': 0.0, 'final': 0.0, 'warm_up': 1}, 'num_monocular_samples': 32, 'num_surfaces': 1, 'predict_opacity': False, 'gaussians_per_pixel': 1, 'gaussian_adapter': {'gaussian_scale_min': 0.5, 'gaussian_scale_max': 15.0, 'sh_degree': 4}, 'd_feature': 128, 'visualizer': {'num_samples': 8, 'min_resolution': 256, 'export_ply': False}, 'apply_bounds_shim': True, 'gs_params_head_type': 'dpt_gs', 'pose_free': True, 'pretrained_weights': ''}, 'decoder': {'name': 'splatting_cuda', 'background_color': [0.0, 0.0, 0.0], 'make_scale_invariant': False, 'use_gsplat': True}, 'vggt': {'input_image_shape': [518, 518], 'head_mode': 'pcd', 'same_head': False, 'camera_prompt': False, 'num_visual_prompt': 0, 'use_extrinsic_token': False, 'use_intrinsic_token': False, 'detr': {'name': 'deformable_detr', 'features_num_channel': [128, 32], 'hidden_dim': 32, 'dim_down_factor': 1, 'dim_feedforward_factor': 4, 'enc_layers': 6, 'dec_layers': 3, 'dropout': 0.1, 'num_feature_levels': 2, 'dec_n_points': 4, 'enc_n_points': 4, 'split_threshold': 1.0, 'num_queries': 256, 'base_HW': 64, 'position_embedding': 'sine', 'pre_norm': False, 'self_layer_num': 1, 'num_level': 3, 'debug_mode': 0, 'decoder_type': 'conv', 'split_threshold_range': [0.0, 2.0], 'only_last_level': False, 'grad_mode': 'absgrad', 'same_split_head': False, 'use_mean_features': True}}}, 'loss': {'mse': {'weight': 1.0}, 'lpips': {'weight': 0.05, 'apply_after_step': 0}}, 'wandb': {'project': 'noposplat', 'entity': 'scene-representation-group', 'name': 're10k_hpc', 'mode': 'online', 'tags': ['re10k', '256x256']}, 'mode': 'train', 'data_loader': {'train': {'num_workers': 16, 'persistent_workers': True, 'batch_size': 16, 'seed': 1234}, 'test': {'num_workers': 4, 'persistent_workers': False, 'batch_size': 1, 'seed': 2345}, 'val': {'num_workers': 1, 'persistent_workers': True, 'batch_size': 1, 'seed': 3456}}, 'optimizer': {'lr': 0.0002, 'warm_up_steps': 125, 'backbone_lr_multiplier': 0.1, 'backbone_trainable': 'GB+H', 'accumulate': 1}, 'checkpointing': {'load': None, 'every_n_train_steps': 9375, 'save_top_k': 1, 'save_weights_only': True}, 'train': {'depth_mode': None, 'extended_visualization': False, 'print_log_every_n_steps': 10, 'distiller': '', 'distill_max_steps': 1000000, 'debug_mode': 0, 'camera_consis_loss': 0.0, 'camera_loss': 1.0, 'aux_loss': 0.0, 'one_sample_train': False, 'target_scale_mode': 2, 'extrinsic_drop': 0.1, 'intrinsic_drop': 0.1, 'align_corners': False, 'backbone_checkpoint': 'none', 'intrinsic_scaling': True, 'return_level_gs': True, 'train_context': False, 'use_pred_point_focal': False, 'scaling_detach': True, 'split_score_log_scale': True, 'split_score_grad_scale': 10000.0, 'split_loss': 0.01, 'random_split_strategy': 'score'}, 'test': {'output_path': 'test', 'align_pose': False, 'pose_align_steps': 100, 'rot_opt_lr': 0.005, 'trans_opt_lr': 0.005, 'compute_scores': True, 'save_image': False, 'save_video': False, 'save_compare': False, 'split_threshold_list': [0.0, 0.1, 0.2, 0.3, 0.4], 'pred_intrinsic': False}, 'seed': 111123, 'trainer': {'max_steps': 18751, 'val_check_interval': 500, 'gradient_clip_val': 0.5, 'num_nodes': 1}, 'dataset': {'re10k': {'make_baseline_1': True, 'relative_pose': True, 'augment': True, 'background_color': [0.0, 0.0, 0.0], 'overfit_to_scene': None, 'skip_bad_shape': True, 'view_sampler': {'name': 'bounded', 'num_target_views': 4, 'num_context_views': 2, 'min_distance_between_context_views': 45, 'max_distance_between_context_views': 90, 'min_distance_to_context_views': 0, 'warm_up_steps': 9375, 'initial_min_distance_between_context_views': 25, 'initial_max_distance_between_context_views': 25, 'distill_steps': 0}, 'name': 're10k', 'roots': ['datasets/re10k'], 'input_image_shape': [256, 256], 'original_image_shape': [360, 640], 'cameras_are_circular': False, 'baseline_min': 0.001, 'baseline_max': 10000000000.0, 'max_fov': 100.0}}, '_wandb': {}}
+2025-09-16 18:16:46,005 INFO    MainThread:11845 [wandb_init.py:init():809] starting backend
+2025-09-16 18:16:46,005 INFO    MainThread:11845 [wandb_init.py:init():813] sending inform_init request
+2025-09-16 18:16:46,028 INFO    MainThread:11845 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-09-16 18:16:46,029 INFO    MainThread:11845 [wandb_init.py:init():823] backend started and connected
+2025-09-16 18:16:46,046 INFO    MainThread:11845 [wandb_init.py:init():915] updated telemetry
+2025-09-16 18:16:46,102 INFO    MainThread:11845 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout
+2025-09-16 18:16:46,884 INFO    MainThread:11845 [wandb_init.py:init():1014] starting run threads in backend
+2025-09-16 18:16:47,060 INFO    MainThread:11845 [wandb_run.py:_console_start():2454] atexit reg
+2025-09-16 18:16:47,061 INFO    MainThread:11845 [wandb_run.py:_redirect():2306] redirect: wrap_raw
+2025-09-16 18:16:47,063 INFO    MainThread:11845 [wandb_run.py:_redirect():2371] Wrapping output streams.
+2025-09-16 18:16:47,063 INFO    MainThread:11845 [wandb_run.py:_redirect():2394] Redirects installed.
+2025-09-16 18:16:47,065 INFO    MainThread:11845 [wandb_init.py:init():1056] run started, returning control to user process