backup exp_gopro_defocus_from_rsblur_v6_19072

.gitattributes

@@ -142,3 +142,5 @@ exp_finetune_dav3_scannet_i2slam_v25_pureGT_lr2e5/2026-05-06_18-52-35/wandb/run-
 exp_finetune_dav3_scannet_i2slam_v25_pureGT_lr2e5/2026-05-06_18-52-35/wandb/run-20260506_185501-j2o8bdxw/files/media/images/comparison_891_2bc44a3b6bfde8cec67f.png filter=lfs diff=lfs merge=lfs -text
 exp_finetune_dav3_scannet_i2slam_v25_pureGT_lr2e5/2026-05-06_18-52-35/wandb/run-20260506_185501-j2o8bdxw/files/media/images/projection_16_8f774b0188cb014d8708.png filter=lfs diff=lfs merge=lfs -text
 exp_finetune_dav3_scannet_i2slam_v25_pureGT_lr2e5/2026-05-06_18-52-35/wandb/run-20260506_185501-j2o8bdxw/run-j2o8bdxw.wandb filter=lfs diff=lfs merge=lfs -text
+exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/run-20260505_134952-91bfp1vi/files/media/images/comparison_0_fde3f179d74ba9a3c0df.png filter=lfs diff=lfs merge=lfs -text
+exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/run-20260505_134952-91bfp1vi/files/media/images/projection_1_8a38d1c3c9c41d01f026.png filter=lfs diff=lfs merge=lfs -text

exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/.hydra/config.yaml

@@ -0,0 +1,186 @@
+model:
+  encoder:
+    backbone:
+      name: croco
+      model: ViTLarge_BaseDecoder
+      patch_embed_cls: PatchEmbedDust3R
+      asymmetry_decoder: true
+      intrinsics_embed_loc: encoder
+      intrinsics_embed_degree: 4
+      intrinsics_embed_type: token
+    name: da3_resplat
+    opacity_mapping:
+      initial: 0.0
+      final: 0.0
+      warm_up: 1
+    num_monocular_samples: 32
+    num_surfaces: 1
+    predict_opacity: false
+    gaussians_per_pixel: 1
+    gaussian_adapter:
+      gaussian_scale_min: 0.5
+      gaussian_scale_max: 15.0
+      sh_degree: 3
+    d_feature: 128
+    visualizer:
+      num_samples: 8
+      min_resolution: 256
+      export_ply: false
+    apply_bounds_shim: true
+    gs_params_head_type: dpt_gs
+    pose_free: true
+    pretrained_weights: ''
+    da3_checkpoint: /gpfs/scratch1/shared/qzhang1/da3_pretrained/DA3-GIANT
+    resplat_checkpoint: /gpfs/scratch1/shared/qzhang1/resplat_pretrained/resplat-base-dl3dv-256x448-view8-1934a04c.pth
+    freeze_da3: true
+    freeze_resplat: true
+    use_pred_pose: true
+  decoder:
+    name: splatting_cuda
+    background_color:
+    - 0.0
+    - 0.0
+    - 0.0
+    make_scale_invariant: true
+    low_pass_filter: 0.3
+    decrease_lpf_step: -1
+  refiner: null
+  diffusion_head:
+    enc_k: 256
+    enc_df: 128
+    enc_df_out: 128
+    enc_d_point: 64
+    enc_cross_depth: 4
+    enc_self_depth: 2
+    dec_dim_model: 64
+    dec_cross_depth: 3
+    dec_num_virtual: 256
+    train_num_steps: 3
+    infer_num_steps: 3
+    use_cam_cond: false
+    use_vggt_priors: true
+    use_cam_pred: false
+loss:
+  mse_l1:
+    weight: 0.5
+    empty_render_thresh: 0.02
+  lpips:
+    weight: 0.15
+    apply_after_step: 0
+  ssim:
+    weight: 0.05
+wandb:
+  project: DAV3
+  entity: zswjsw-the-university-of-sydney
+  name: gopro_defocus_from_rsblur_v6_19072
+  mode: online
+  tags:
+  - gopro_defocus
+  - diffusion_head
+  - da3_resplat
+  - finetune
+  - pred_pose
+  - single_view
+  - curriculum
+mode: train
+data_loader:
+  train:
+    num_workers: 4
+    persistent_workers: true
+    batch_size: 1
+    seed: 1234
+  test:
+    num_workers: 2
+    persistent_workers: false
+    batch_size: 1
+    seed: 2345
+  val:
+    num_workers: 1
+    persistent_workers: false
+    batch_size: 1
+    seed: 3456
+optimizer:
+  lr: 2.0e-05
+  warm_up_steps: 200
+  backbone_lr_multiplier: 0.01
+checkpointing:
+  load: /scratch-shared/qzhang1/C3G_outputs/exp_rsblur_v6_bordermask_from_18999/2026-05-05_06-22-33/checkpoints/best-psnr-val/psnr_19.072-info/global_step_2749.ckpt
+  every_n_train_steps: 500
+  save_top_k: 1
+  save_weights_only: false
+  load_weights_only: true
+train:
+  depth_mode: depth
+  extended_visualization: false
+  print_log_every_n_steps: 10
+  context_view_loss: false
+  random_select_context_view: false
+  freeze_encoder: true
+  share_weight: 0.0
+  l_flow_weight: 0.0
+  cam_loss_weight: 0.0
+test:
+  output_path: outputs/test
+  align_pose: true
+  pose_align_steps: 100
+  rot_opt_lr: 0.005
+  trans_opt_lr: 0.005
+  compute_scores: true
+  save_image: true
+  save_video: false
+  save_compare: false
+  use_tto: false
+  tto_steps: 200
+  tto_lr: 0.001
+  tto_geo_lr_scale: 0.1
+seed: 111123
+trainer:
+  max_steps: 20000
+  val_check_interval: 500
+  gradient_clip_val: 1.0
+  num_nodes: 1
+  accumulate_grad_batches: 4
+  devices: 4
+  strategy: ddp
+  enable_progress_bar: true
+  num_sanity_val_steps: 0
+  precision: bf16-mixed
+dataset:
+  gopro_defocus:
+    make_baseline_1: false
+    relative_pose: false
+    augment: false
+    background_color:
+    - 0.0
+    - 0.0
+    - 0.0
+    overfit_to_scene: null
+    skip_bad_shape: false
+    view_sampler:
+      name: bounded
+      num_target_views: 1
+      num_context_views: 2
+      min_distance_between_context_views: 2
+      max_distance_between_context_views: 6
+      min_distance_to_context_views: 0
+      warm_up_steps: 0
+      initial_min_distance_between_context_views: 2
+      initial_max_distance_between_context_views: 6
+    name: gopro_defocus
+    roots:
+    - /scratch-shared/qzhang1/datasets/gopro_defocus/Go_pro_defocus_train
+    near: 0.1
+    far: 100.0
+    input_image_shape:
+    - 336
+    - 336
+    original_image_shape:
+    - 336
+    - 336
+    cameras_are_circular: false
+    baseline_min: 0.0
+    baseline_max: 10000000000.0
+    max_fov: 180.0
+    val_ids_start: 502
+    samples_per_scene: 1
+    val_samples_per_scene: 1

exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/.hydra/hydra.yaml

@@ -0,0 +1,167 @@
+hydra:
+  run:
+    dir: /scratch-shared/qzhang1/C3G_outputs/exp_${wandb.name}/${now:%Y-%m-%d_%H-%M-%S}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - +training=finetune_dav3_gopro_defocus_l1
+    - trainer.num_nodes=1
+    - trainer.devices=4
+    - checkpointing.load=/scratch-shared/qzhang1/C3G_outputs/exp_rsblur_v6_bordermask_from_18999/2026-05-05_06-22-33/checkpoints/best-psnr-val/psnr_19.072-info/global_step_2749.ckpt
+    - checkpointing.load_weights_only=true
+    - wandb.mode=online
+    - wandb.name=gopro_defocus_from_rsblur_v6_19072
+  job:
+    name: main
+    chdir: null
+    override_dirname: +training=finetune_dav3_gopro_defocus_l1,checkpointing.load=/scratch-shared/qzhang1/C3G_outputs/exp_rsblur_v6_bordermask_from_18999/2026-05-05_06-22-33/checkpoints/best-psnr-val/psnr_19.072-info/global_step_2749.ckpt,checkpointing.load_weights_only=true,trainer.devices=4,trainer.num_nodes=1,wandb.mode=online,wandb.name=gopro_defocus_from_rsblur_v6_19072
+    id: ???
+    num: ???
+    config_name: main
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /gpfs/home2/qzhang1/C3G
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /gpfs/home2/qzhang1/C3G/config
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /scratch-shared/qzhang1/C3G_outputs/exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53
+    choices:
+      training: finetune_dav3_gopro_defocus_l1
+      dataset@dataset.gopro_defocus: gopro_defocus
+      dataset/view_sampler@dataset.gopro_defocus.view_sampler: bounded
+      model/decoder: splatting_cuda
+      model/encoder: noposplat
+      model/encoder/backbone: croco
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/.hydra/overrides.yaml

@@ -0,0 +1,7 @@
+- +training=finetune_dav3_gopro_defocus_l1
+- trainer.num_nodes=1
+- trainer.devices=4
+- checkpointing.load=/scratch-shared/qzhang1/C3G_outputs/exp_rsblur_v6_bordermask_from_18999/2026-05-05_06-22-33/checkpoints/best-psnr-val/psnr_19.072-info/global_step_2749.ckpt
+- checkpointing.load_weights_only=true
+- wandb.mode=online
+- wandb.name=gopro_defocus_from_rsblur_v6_19072

exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/main.log

@@ -0,0 +1,9 @@
+[2026-05-05 13:48:17,099][dinov2][INFO] - using MLP layer as FFN
+[2026-05-05 13:48:17,344][dinov2][INFO] - using MLP layer as FFN
+[2026-05-05 13:48:26,853][dinov2][INFO] - using MLP layer as FFN
+[2026-05-05 13:48:27,287][dinov2][INFO] - using MLP layer as FFN
+[2026-05-05 13:49:48,707][pytorch_lightning.utilities.rank_zero][INFO] - ----------------------------------------------------------------------------------------------------
+distributed_backend=nccl
+All distributed processes registered. Starting with 4 processes
+----------------------------------------------------------------------------------------------------
+

exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2026-05-05T13:49:53.115315174+02:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"/scratch-shared/qzhang1/C3G_outputs/exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/run-20260505_134952-91bfp1vi/logs/debug-core.log"}
+{"time":"2026-05-05T13:49:53.473610235+02:00","level":"INFO","msg":"created new stream","id":"91bfp1vi"}
+{"time":"2026-05-05T13:49:53.475335599+02:00","level":"INFO","msg":"stream: started","id":"91bfp1vi"}
+{"time":"2026-05-05T13:49:53.475443838+02:00","level":"INFO","msg":"sender: started","stream_id":"91bfp1vi"}
+{"time":"2026-05-05T13:49:53.475504407+02:00","level":"INFO","msg":"handler: started","stream_id":"91bfp1vi"}
+{"time":"2026-05-05T13:49:53.475451708+02:00","level":"INFO","msg":"writer: Do: started","stream_id":"91bfp1vi"}
+{"time":"2026-05-05T13:49:54.336482342+02:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2026-05-05T13:50:06.775080191+02:00","level":"INFO","msg":"stream: closing","id":"91bfp1vi"}
+{"time":"2026-05-05T13:50:06.77516645+02:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2026-05-05T13:50:06.775294229+02:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2026-05-05T13:50:07.467926865+02:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-05-05T13:50:07.756723943+02:00","level":"INFO","msg":"handler: closed","stream_id":"91bfp1vi"}
+{"time":"2026-05-05T13:50:07.756787683+02:00","level":"INFO","msg":"writer: Close: closed","stream_id":"91bfp1vi"}
+{"time":"2026-05-05T13:50:07.756794253+02:00","level":"INFO","msg":"sender: closed","stream_id":"91bfp1vi"}
+{"time":"2026-05-05T13:50:07.75711223+02:00","level":"INFO","msg":"stream: closed","id":"91bfp1vi"}

exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/debug.log

@@ -0,0 +1,85 @@
+2026-05-05 13:49:52,978 INFO    MainThread:2231583 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2026-05-05 13:49:52,979 INFO    MainThread:2231583 [wandb_setup.py:_flush():70] Configure stats pid to 2231583
+2026-05-05 13:49:52,979 INFO    MainThread:2231583 [wandb_setup.py:_flush():70] Loading settings from /home/qzhang1/.config/wandb/settings
+2026-05-05 13:49:52,979 INFO    MainThread:2231583 [wandb_setup.py:_flush():70] Loading settings from /gpfs/home2/qzhang1/C3G/wandb/settings
+2026-05-05 13:49:52,979 INFO    MainThread:2231583 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2026-05-05 13:49:52,979 INFO    MainThread:2231583 [wandb_init.py:setup_run_log_directory():724] Logging user logs to /scratch-shared/qzhang1/C3G_outputs/exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/run-20260505_134952-91bfp1vi/logs/debug.log
+2026-05-05 13:49:52,979 INFO    MainThread:2231583 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to /scratch-shared/qzhang1/C3G_outputs/exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/run-20260505_134952-91bfp1vi/logs/debug-internal.log
+2026-05-05 13:49:52,979 INFO    MainThread:2231583 [wandb_init.py:init():852] calling init triggers
+2026-05-05 13:49:52,979 INFO    MainThread:2231583 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'model': {'encoder': {'backbone': {'name': 'croco', 'model': 'ViTLarge_BaseDecoder', 'patch_embed_cls': 'PatchEmbedDust3R', 'asymmetry_decoder': True, 'intrinsics_embed_loc': 'encoder', 'intrinsics_embed_degree': 4, 'intrinsics_embed_type': 'token'}, 'name': 'da3_resplat', 'opacity_mapping': {'initial': 0.0, 'final': 0.0, 'warm_up': 1}, 'num_monocular_samples': 32, 'num_surfaces': 1, 'predict_opacity': False, 'gaussians_per_pixel': 1, 'gaussian_adapter': {'gaussian_scale_min': 0.5, 'gaussian_scale_max': 15.0, 'sh_degree': 3}, 'd_feature': 128, 'visualizer': {'num_samples': 8, 'min_resolution': 256, 'export_ply': False}, 'apply_bounds_shim': True, 'gs_params_head_type': 'dpt_gs', 'pose_free': True, 'pretrained_weights': '', 'da3_checkpoint': '/gpfs/scratch1/shared/qzhang1/da3_pretrained/DA3-GIANT', 'resplat_checkpoint': '/gpfs/scratch1/shared/qzhang1/resplat_pretrained/resplat-base-dl3dv-256x448-view8-1934a04c.pth', 'freeze_da3': True, 'freeze_resplat': True, 'use_pred_pose': True}, 'decoder': {'name': 'splatting_cuda', 'background_color': [0.0, 0.0, 0.0], 'make_scale_invariant': True, 'low_pass_filter': 0.3, 'decrease_lpf_step': -1}, 'refiner': None, 'diffusion_head': {'enc_k': 256, 'enc_df': 128, 'enc_df_out': 128, 'enc_d_point': 64, 'enc_cross_depth': 4, 'enc_self_depth': 2, 'dec_dim_model': 64, 'dec_cross_depth': 3, 'dec_num_virtual': 256, 'train_num_steps': 3, 'infer_num_steps': 3, 'use_cam_cond': False, 'use_vggt_priors': True, 'use_cam_pred': False}}, 'loss': {'mse_l1': {'weight': 0.5, 'empty_render_thresh': 0.02}, 'lpips': {'weight': 0.15, 'apply_after_step': 0}, 'ssim': {'weight': 0.05}}, 'wandb': {'project': 'DAV3', 'entity': 'zswjsw-the-university-of-sydney', 'name': 'gopro_defocus_from_rsblur_v6_19072', 'mode': 'online', 'tags': ['gopro_defocus', 'diffusion_head', 'da3_resplat', 'finetune', 'pred_pose', 'single_view', 'curriculum']}, 'mode': 'train', 'data_loader': {'train': {'num_workers': 4, 'persistent_workers': True, 'batch_size': 1, 'seed': 1234}, 'test': {'num_workers': 2, 'persistent_workers': False, 'batch_size': 1, 'seed': 2345}, 'val': {'num_workers': 1, 'persistent_workers': False, 'batch_size': 1, 'seed': 3456}}, 'optimizer': {'lr': 2e-05, 'warm_up_steps': 200, 'backbone_lr_multiplier': 0.01}, 'checkpointing': {'load': '/scratch-shared/qzhang1/C3G_outputs/exp_rsblur_v6_bordermask_from_18999/2026-05-05_06-22-33/checkpoints/best-psnr-val/psnr_19.072-info/global_step_2749.ckpt', 'every_n_train_steps': 500, 'save_top_k': 1, 'save_weights_only': False, 'load_weights_only': True}, 'train': {'depth_mode': 'depth', 'extended_visualization': False, 'print_log_every_n_steps': 10, 'context_view_loss': False, 'random_select_context_view': False, 'freeze_encoder': True, 'share_weight': 0.0, 'l_flow_weight': 0.0, 'cam_loss_weight': 0.0}, 'test': {'output_path': 'outputs/test', 'align_pose': True, 'pose_align_steps': 100, 'rot_opt_lr': 0.005, 'trans_opt_lr': 0.005, 'compute_scores': True, 'save_image': True, 'save_video': False, 'save_compare': False, 'use_tto': False, 'tto_steps': 200, 'tto_lr': 0.001, 'tto_geo_lr_scale': 0.1}, 'seed': 111123, 'trainer': {'max_steps': 20000, 'val_check_interval': 500, 'gradient_clip_val': 1.0, 'num_nodes': 1, 'accumulate_grad_batches': 4, 'devices': 4, 'strategy': 'ddp', 'enable_progress_bar': True, 'num_sanity_val_steps': 0, 'precision': 'bf16-mixed'}, 'dataset': {'gopro_defocus': {'make_baseline_1': False, 'relative_pose': False, 'augment': False, 'background_color': [0.0, 0.0, 0.0], 'overfit_to_scene': None, 'skip_bad_shape': False, 'view_sampler': {'name': 'bounded', 'num_target_views': 1, 'num_context_views': 2, 'min_distance_between_context_views': 2, 'max_distance_between_context_views': 6, 'min_distance_to_context_views': 0, 'warm_up_steps': 0, 'initial_min_distance_between_context_views': 2, 'initial_max_distance_between_context_views': 6}, 'name': 'gopro_defocus', 'roots': ['/scratch-shared/qzhang1/datasets/gopro_defocus/Go_pro_defocus_train'], 'near': 0.1, 'far': 100.0, 'input_image_shape': [336, 336], 'original_image_shape': [336, 336], 'cameras_are_circular': False, 'baseline_min': 0.0, 'baseline_max': 10000000000.0, 'max_fov': 180.0, 'val_ids_start': 502, 'samples_per_scene': 1, 'val_samples_per_scene': 1}}, '_wandb': {}}
+2026-05-05 13:49:53,100 INFO    MainThread:2231583 [wandb_init.py:init():893] starting backend
+2026-05-05 13:49:53,100 INFO    MainThread:2231583 [wandb_init.py:init():897] sending inform_init request
+2026-05-05 13:49:53,111 INFO    MainThread:2231583 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-05-05 13:49:53,112 INFO    MainThread:2231583 [wandb_init.py:init():907] backend started and connected
+2026-05-05 13:49:53,115 INFO    MainThread:2231583 [wandb_init.py:init():1005] updated telemetry
+2026-05-05 13:49:53,134 INFO    MainThread:2231583 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2026-05-05 13:49:54,293 INFO    MainThread:2231583 [wandb_init.py:init():1104] starting run threads in backend
+2026-05-05 13:49:54,770 INFO    MainThread:2231583 [wandb_run.py:_console_start():2573] atexit reg
+2026-05-05 13:49:54,770 INFO    MainThread:2231583 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2026-05-05 13:49:54,771 INFO    MainThread:2231583 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2026-05-05 13:49:54,771 INFO    MainThread:2231583 [wandb_run.py:_redirect():2513] Redirects installed.
+2026-05-05 13:49:54,776 INFO    MainThread:2231583 [wandb_init.py:init():1150] run started, returning control to user process
+2026-05-05 13:50:06,774 INFO    MsgRouterThr:2231583 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.
+2026-05-05 13:50:08,594 ERROR   MainThread:2231583 [redirect.py:_on_write():664] [no run ID] error in stdout callback
+Traceback (most recent call last):
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
+    cb(written_data)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2494, in <lambda>
+    lambda data: self._console_raw_callback("stdout", data),
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 406, in wrapper
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 464, in wrapper_fn
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1639, in _console_raw_callback
+    self._backend.interface.publish_output_raw(name, data)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 762, in publish_output_raw
+    self._publish_output_raw(o)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
+    self._publish(rec)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
+    self._sock_client.send_record_publish(record)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
+    self.send_server_request(server_req)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
+    self._send_message(msg)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
+    self._sendall_with_error_handle(header + data)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
+    sent = self._sock.send(data)
+           ^^^^^^^^^^^^^^^^^^^^^
+BrokenPipeError: [Errno 32] Broken pipe
+2026-05-05 13:50:08,600 ERROR   MainThread:2231583 [redirect.py:_on_write():664] [no run ID] error in stdout callback
+Traceback (most recent call last):
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
+    cb(written_data)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2494, in <lambda>
+    lambda data: self._console_raw_callback("stdout", data),
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 406, in wrapper
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 464, in wrapper_fn
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1639, in _console_raw_callback
+    self._backend.interface.publish_output_raw(name, data)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 762, in publish_output_raw
+    self._publish_output_raw(o)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
+    self._publish(rec)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
+    self._sock_client.send_record_publish(record)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
+    self.send_server_request(server_req)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
+    self._send_message(msg)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
+    self._sendall_with_error_handle(header + data)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
+    sent = self._sock.send(data)
+           ^^^^^^^^^^^^^^^^^^^^^
+BrokenPipeError: [Errno 32] Broken pipe

exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/run-20260505_134952-91bfp1vi/files/config.yaml

@@ -0,0 +1,438 @@
+_wandb:
+    value:
+        cli_version: 0.19.11
+        m:
+            - "1": cameras._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+            - "1": cameras.width
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": cameras.count
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": info/n_gaussians_coarse_step
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": diag/aniso_mean
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": diag/log_aniso_p99
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.width
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.format
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": projection.format
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": epoch
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": diag/log_aniso_mean
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.captions
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": cameras.height
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": projection.count
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": projection.height
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": diag/scale_max_mean
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.height
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.count
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": projection.filenames
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": diag/aniso_max
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": val/lpips
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": cameras.filenames
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": info/n_gaussians_coarse_epoch
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": projection.width
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": cameras.format
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": val/ssim
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": comparison.filenames
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": projection._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": diag/scale_min_mean
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": val/psnr
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+        python_version: 3.11.15
+        t:
+            "1":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 41
+                - 49
+                - 50
+                - 53
+                - 55
+                - 71
+                - 103
+                - 105
+                - 106
+            "2":
+                - 1
+                - 5
+                - 9
+                - 11
+                - 41
+                - 49
+                - 50
+                - 53
+                - 55
+                - 71
+                - 103
+                - 105
+                - 106
+            "3":
+                - 7
+                - 13
+                - 15
+                - 16
+                - 23
+                - 55
+                - 66
+            "4": 3.11.15
+            "5": 0.19.11
+            "6": 4.57.1
+            "8":
+                - 2
+                - 5
+            "12": 0.19.11
+            "13": linux-x86_64
+checkpointing:
+    value:
+        every_n_train_steps: 500
+        load: /scratch-shared/qzhang1/C3G_outputs/exp_rsblur_v6_bordermask_from_18999/2026-05-05_06-22-33/checkpoints/best-psnr-val/psnr_19.072-info/global_step_2749.ckpt
+        load_weights_only: true
+        save_top_k: 1
+        save_weights_only: false
+data_loader:
+    value:
+        test:
+            batch_size: 1
+            num_workers: 2
+            persistent_workers: false
+            seed: 2345
+        train:
+            batch_size: 1
+            num_workers: 4
+            persistent_workers: true
+            seed: 1234
+        val:
+            batch_size: 1
+            num_workers: 1
+            persistent_workers: false
+            seed: 3456
+dataset:
+    value:
+        gopro_defocus:
+            augment: false
+            background_color:
+                - 0
+                - 0
+                - 0
+            baseline_max: 1e+10
+            baseline_min: 0
+            cameras_are_circular: false
+            far: 100
+            input_image_shape:
+                - 336
+                - 336
+            make_baseline_1: false
+            max_fov: 180
+            name: gopro_defocus
+            near: 0.1
+            original_image_shape:
+                - 336
+                - 336
+            overfit_to_scene: null
+            relative_pose: false
+            roots:
+                - /scratch-shared/qzhang1/datasets/gopro_defocus/Go_pro_defocus_train
+            samples_per_scene: 1
+            skip_bad_shape: false
+            val_ids_start: 502
+            val_samples_per_scene: 1
+            view_sampler:
+                initial_max_distance_between_context_views: 6
+                initial_min_distance_between_context_views: 2
+                max_distance_between_context_views: 6
+                min_distance_between_context_views: 2
+                min_distance_to_context_views: 0
+                name: bounded
+                num_context_views: 2
+                num_target_views: 1
+                warm_up_steps: 0
+loss:
+    value:
+        lpips:
+            apply_after_step: 0
+            weight: 0.15
+        mse_l1:
+            empty_render_thresh: 0.02
+            weight: 0.5
+        ssim:
+            weight: 0.05
+mode:
+    value: train
+model:
+    value:
+        decoder:
+            background_color:
+                - 0
+                - 0
+                - 0
+            decrease_lpf_step: -1
+            low_pass_filter: 0.3
+            make_scale_invariant: true
+            name: splatting_cuda
+        diffusion_head:
+            dec_cross_depth: 3
+            dec_dim_model: 64
+            dec_num_virtual: 256
+            enc_cross_depth: 4
+            enc_d_point: 64
+            enc_df: 128
+            enc_df_out: 128
+            enc_k: 256
+            enc_self_depth: 2
+            infer_num_steps: 3
+            train_num_steps: 3
+            use_cam_cond: false
+            use_cam_pred: false
+            use_vggt_priors: true
+        encoder:
+            apply_bounds_shim: true
+            backbone:
+                asymmetry_decoder: true
+                intrinsics_embed_degree: 4
+                intrinsics_embed_loc: encoder
+                intrinsics_embed_type: token
+                model: ViTLarge_BaseDecoder
+                name: croco
+                patch_embed_cls: PatchEmbedDust3R
+            d_feature: 128
+            da3_checkpoint: /gpfs/scratch1/shared/qzhang1/da3_pretrained/DA3-GIANT
+            freeze_da3: true
+            freeze_resplat: true
+            gaussian_adapter:
+                gaussian_scale_max: 15
+                gaussian_scale_min: 0.5
+                sh_degree: 3
+            gaussians_per_pixel: 1
+            gs_params_head_type: dpt_gs
+            name: da3_resplat
+            num_monocular_samples: 32
+            num_surfaces: 1
+            opacity_mapping:
+                final: 0
+                initial: 0
+                warm_up: 1
+            pose_free: true
+            predict_opacity: false
+            pretrained_weights: ""
+            resplat_checkpoint: /gpfs/scratch1/shared/qzhang1/resplat_pretrained/resplat-base-dl3dv-256x448-view8-1934a04c.pth
+            use_pred_pose: true
+            visualizer:
+                export_ply: false
+                min_resolution: 256
+                num_samples: 8
+        refiner: null
+optimizer:
+    value:
+        backbone_lr_multiplier: 0.01
+        lr: 2e-05
+        warm_up_steps: 200
+seed:
+    value: 111123
+test:
+    value:
+        align_pose: true
+        compute_scores: true
+        output_path: outputs/test
+        pose_align_steps: 100
+        rot_opt_lr: 0.005
+        save_compare: false
+        save_image: true
+        save_video: false
+        trans_opt_lr: 0.005
+        tto_geo_lr_scale: 0.1
+        tto_lr: 0.001
+        tto_steps: 200
+        use_tto: false
+train:
+    value:
+        cam_loss_weight: 0
+        context_view_loss: false
+        depth_mode: depth
+        extended_visualization: false
+        freeze_encoder: true
+        l_flow_weight: 0
+        print_log_every_n_steps: 10
+        random_select_context_view: false
+        share_weight: 0
+trainer:
+    value:
+        accumulate_grad_batches: 4
+        devices: 4
+        enable_progress_bar: true
+        gradient_clip_val: 1
+        max_steps: 20000
+        num_nodes: 1
+        num_sanity_val_steps: 0
+        precision: bf16-mixed
+        strategy: ddp
+        val_check_interval: 500
+wandb:
+    value:
+        entity: zswjsw-the-university-of-sydney
+        mode: online
+        name: gopro_defocus_from_rsblur_v6_19072
+        project: DAV3
+        tags:
+            - gopro_defocus
+            - diffusion_head
+            - da3_resplat
+            - finetune
+            - pred_pose
+            - single_view
+            - curriculum

exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/run-20260505_134952-91bfp1vi/files/output.log

@@ -0,0 +1,156 @@
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3]
+/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:216: Using `DistributedSampler` with the dataloaders. During `trainer.validate()`, it is recommended to use `Trainer(devices=1, num_nodes=1)` to ensure each sample/batch gets evaluated exactly once. Otherwise, multi-device settings use `DistributedSampler` that replicates some samples to make sure all devices have same batch size in case of uneven inputs.
+/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:425: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=15` in the `DataLoader` to improve performance.
+Validation DataLoader 0:   0%|          | 0/1 [00:00<?, ?it/s]validation step 0; scene = ['502']; context = [[3, 4]]
+/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/torch/functional.py:534: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3595.)
+  return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]
+/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/pointops/query.py:19: UserWarning: The torch.cuda.*DtypeTensor constructors are no longer recommended. It's best to use methods such as torch.tensor(data, dtype=*, device='cuda') to create tensors. (Triggered internally at ../torch/csrc/tensor/python_tensor.cpp:78.)
+  idx = torch.cuda.IntTensor(m, nsample).zero_()
+[DIAG A] ctx_idx=[3, 4]
+[DIAG A] tgt_idx=[3]  (subset of ctx)
+[DIAG A] ctx_pos_for_tgt=[0]  (positions in ctx list)
+[DIAG A] ctx_t (translations) per view:
+           ctx[ 0] t=[ -0.084,  +0.007,  -0.004]
+           ctx[ 1] t=[+14.329,  +0.257,  +2.128]
+[DIAG A] render_t (target extrinsics used by decoder):
+           render[ 0] t=[ -0.084,  +0.007,  -0.004]  ←supposedly ctx[0]
+[DIAG A] render==ctx[ctx_pos_for_tgt] check: PASS
+[DIAG B] translation std  per axis (xyz): [10.1909, 0.1765, 1.5076]
+[DIAG B] translation range per axis (xyz): [14.4121, 0.2497, 2.1321]
+[DIAG B] y_range = 0.2497  (>0.05 = catastrophic for static rig)
+[DIAG C] log_aniso mean=12.612  p50=12.931  p95=13.108  p99=13.186  max=13.214
+[DIAG C] (recall: log_aniso=ln(max_scale/min_scale); 0=spherical, 5+=needle)
+[INTR] fx=2.028 fy=2.199 cx=0.500 cy=0.500
+/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/utilities/data.py:79: Trying to infer the `batch_size` from an ambiguous collection. The batch size we found is 1. To avoid any miscalculations, use `self.log(..., batch_size=batch_size)`.
+[DIAG step=0] aniso mean=8060755.00 max=31616148.00 | log10_aniso mean=5.705 p99=7.399
+[COVER] near-bg fraction: full=0.0174 bottom20%=0.0425 top20%=0.0003
+[COVER] bottom/full ratio = 2.44  (>>1 = bottom coverage collapse, ≈1 = uniform coverage shift)
+Setting up [LPIPS] perceptual loss: trunk [vgg], v[0.1], spatial [off]
+/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.
+  warnings.warn(
+/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=VGG16_Weights.IMAGENET1K_V1`. You can also use `weights=VGG16_Weights.DEFAULT` to get the most up-to-date weights.
+  warnings.warn(msg)
+Loading model from: /home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lpips/weights/v0.1/vgg.pth
+/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lpips/lpips.py:107: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.
+  self.load_state_dict(torch.load(model_path, map_location='cpu'), strict=False)
+Validation DataLoader 0: 100%|██████████| 1/1 [00:04<00:00,  0.24it/s]
+/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('info/n_gaussians_coarse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+┃        Validate metric        ┃         DataLoader 0          ┃
+┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
+│        diag/aniso_max         │          38765508.0           │
+│        diag/aniso_mean        │           9784027.0           │
+│      diag/log_aniso_mean      │       5.756680488586426       │
+│      diag/log_aniso_p99       │       7.483924388885498       │
+│      diag/scale_max_mean      │       18.98053741455078       │
+│      diag/scale_min_mean      │     0.0008003668626770377     │
+│ info/n_gaussians_coarse_epoch │            12800.0            │
+│           val/lpips           │      0.7240225076675415       │
+│           val/psnr            │      15.567266464233398       │
+│           val/ssim            │      0.33824753761291504      │
+└───────────────────────────────┴───────────────────────────────┘
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3]
+
+  | Name           | Type                   | Params | Mode
+------------------------------------------------------------------
+0 | encoder        | EncoderDA3ReSplat      | 1.6 B  | eval
+1 | decoder        | DecoderSplattingCUDA   | 0      | train
+2 | diffusion_head | DiffusionHead          | 2.3 M  | train
+3 | losses         | ModuleList             | 0      | train
+4 | miou           | MulticlassJaccardIndex | 0      | train
+5 | acc            | MulticlassAccuracy     | 0      | train
+------------------------------------------------------------------
+2.2 M     Trainable params
+1.6 B     Non-trainable params
+1.6 B     Total params
+6,269.182 Total estimated model params size (MB)
+230       Modules in train mode
+1954      Modules in eval mode
+Epoch 0: |          | 0/? [00:00<?, ?it/s] 
+Error executing job with overrides: ['+training=finetune_dav3_gopro_defocus_l1', 'trainer.num_nodes=1', 'trainer.devices=4', 'checkpointing.load=/scratch-shared/qzhang1/C3G_outputs/exp_rsblur_v6_bordermask_from_18999/2026-05-05_06-22-33/checkpoints/best-psnr-val/psnr_19.072-info/global_step_2749.ckpt', 'checkpointing.load_weights_only=true', 'wandb.mode=online', 'wandb.name=gopro_defocus_from_rsblur_v6_19072']
+Traceback (most recent call last):
+  File "/gpfs/home2/qzhang1/C3G/src/main.py", line 314, in train
+    trainer.fit(model_wrapper, datamodule=data_module, ckpt_path=checkpoint_path)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/trainer/trainer.py", line 561, in fit
+    call._call_and_handle_interrupt(
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/trainer/call.py", line 47, in _call_and_handle_interrupt
+    return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/strategies/launchers/subprocess_script.py", line 105, in launch
+    return function(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/trainer/trainer.py", line 599, in _fit_impl
+    self._run(model, ckpt_path=ckpt_path)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/trainer/trainer.py", line 1012, in _run
+    results = self._run_stage()
+              ^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/trainer/trainer.py", line 1056, in _run_stage
+    self.fit_loop.run()
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/loops/fit_loop.py", line 216, in run
+    self.advance()
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/loops/fit_loop.py", line 455, in advance
+    self.epoch_loop.run(self._data_fetcher)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/loops/training_epoch_loop.py", line 150, in run
+    self.advance(data_fetcher)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/loops/training_epoch_loop.py", line 320, in advance
+    batch_output = self.automatic_optimization.run(trainer.optimizers[0], batch_idx, kwargs)
+                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/loops/optimization/automatic.py", line 185, in run
+    closure()
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/loops/optimization/automatic.py", line 146, in __call__
+    self._result = self.closure(*args, **kwargs)
+                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
+    return func(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/loops/optimization/automatic.py", line 131, in closure
+    step_output = self._step_fn()
+                  ^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/loops/optimization/automatic.py", line 319, in _training_step
+    training_step_output = call._call_strategy_hook(trainer, "training_step", *kwargs.values())
+                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/trainer/call.py", line 328, in _call_strategy_hook
+    output = fn(*args, **kwargs)
+             ^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/strategies/strategy.py", line 390, in training_step
+    return self._forward_redirection(self.model, self.lightning_module, "training_step", *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/strategies/strategy.py", line 641, in __call__
+    wrapper_output = wrapper_module(*args, **kwargs)
+                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1643, in forward
+    else self._run_ddp_forward(*inputs, **kwargs)
+         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/torch/nn/parallel/distributed.py", line 1459, in _run_ddp_forward
+    return self.module(*inputs, **kwargs)  # type: ignore[index]
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1736, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1747, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/lightning/pytorch/strategies/strategy.py", line 634, in wrapped_forward
+    out = method(*_args, **_kwargs)
+          ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/qzhang1/C3G/src/model/model_wrapper.py", line 559, in training_step
+    output = self.decoder.forward(
+             ^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/qzhang1/C3G/src/model/decoder/decoder_splatting_cuda.py", line 60, in forward
+    color, depth, feature = render_cuda(
+                            ^^^^^^^^^^^^
+  File "/gpfs/home2/qzhang1/C3G/src/model/decoder/cuda_splatting.py", line 94, in render_cuda
+    projection_matrix = get_projection_matrix(near, far, fov_x, fov_y)
+                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/gpfs/home2/qzhang1/C3G/src/model/decoder/cuda_splatting.py", line 42, in get_projection_matrix
+    result[:, 0, 0] = 2 * near / (right - left)
+    ~~~~~~^^^^^^^^^
+RuntimeError: The expanded size of the tensor (1) must match the existing size (2) at non-singleton dimension 0.  Target sizes: [1].  Tensor sizes: [2]
+
+Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.

exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/run-20260505_134952-91bfp1vi/files/requirements.txt

@@ -0,0 +1,340 @@
+joblib==1.5.3
+pillow_heif==1.3.0
+pccm==0.4.16
+decorator==4.4.2
+coloredlogs==15.0.1
+opt_einsum==3.4.0
+nvidia-cublas-cu12==12.1.3.1
+tqdm==4.67.3
+wavedrom==2.0.3.post3
+py-cpuinfo==9.0.0
+tifffile==2026.3.3
+jaxtyping==0.3.2
+yourdfpy==0.0.57
+safehttpx==0.1.6
+h11==0.14.0
+narwhals==2.6.0
+aiofiles==24.1.0
+ffmpy==0.6.1
+facexlib==0.3.0
+charset-normalizer==3.4.2
+plotly==6.3.0
+pluggy==1.6.0
+svgwrite==1.4.3
+svg.py==1.6.0
+ptyprocess==0.7.0
+webencodings==0.5.1
+nvidia-cuda-runtime==13.0.96
+protobuf==6.33.6
+pfzy==0.3.4
+black==25.1.0
+ImageIO==2.37.3
+lightning-utilities==0.15.3
+ftfy==6.3.1
+sentencepiece==0.1.99
+Jinja2==3.1.6
+peft==0.4.0
+nvidia-nvjitlink-cu12==12.9.86
+jsonschema-specifications==2025.4.1
+pydantic_core==2.33.2
+virtualenv==21.2.4
+brotli==1.2.0
+pyarrow-hotfix==0.7
+pytorch-lightning==2.5.5
+gitdb==4.0.12
+aiosignal==1.4.0
+pycollada==0.9
+widgetsnbextension==4.0.14
+python-discovery==1.2.2
+yarl==1.20.1
+ruff==0.13.2
+diff_gaussian_rasterization==0.0.0
+kornia_rs==0.1.9
+tensorboard==2.20.0
+platformdirs==4.3.8
+docstring_parser==0.16
+transformers==4.57.1
+nvidia-cusolver-cu12==11.4.5.107
+portalocker==3.2.0
+cuda-bindings==13.2.0
+fonttools==4.62.1
+iniconfig==2.3.0
+starlette==0.48.0
+inquirerpy==0.3.4
+manifold3d==3.1.0
+opencv-python==4.11.0.86
+mediapy==1.2.4
+nvidia-cudnn-cu12==9.1.0.70
+pyparsing==3.3.2
+e3nn==0.5.6
+identify==2.6.19
+httpx==0.24.1
+mamba-ssm==2.2.4
+annotated-types==0.7.0
+contourpy==1.3.3
+antlr4-python3-runtime==4.9.3
+filelock==3.25.2
+azure-cognitiveservices-speech==1.46.0
+nvidia-nvtx-cu12==12.1.105
+cfgv==3.5.0
+PySocks==1.7.1
+scipy==1.17.1
+future==1.0.0
+filterpy==1.4.5
+gradio_client==1.13.3
+hjson==3.1.0
+tensorboard-data-server==0.7.2
+clip==1.0
+wcwidth==0.2.13
+av==16.0.1
+nvidia-cufile==1.15.1.6
+nvidia-cuda-nvrtc==13.0.88
+hydra-core==1.3.2
+six==1.17.0
+python-multipart==0.0.20
+triton==3.1.0
+argcomplete==3.6.2
+addict==2.4.0
+zipp==3.23.0
+pointops==1.0
+pointops==1.0
+nvidia-cuda-cupti==13.0.85
+nvidia-cusparse==12.6.3.3
+nvidia-cusparse-cu12==12.1.0.106
+rtree==1.4.0
+huggingface-hub==0.34.1
+tzdata==2025.2
+markdown2==2.5.4
+soupsieve==2.8
+nvidia-ml-py==13.580.82
+nvidia-cufft==12.0.0.61
+pexpect==4.9.0
+deepspeed==0.14.4
+latex2mathml==3.78.1
+sentry-sdk==2.29.1
+svg.path==6.3
+azure-core==1.35.1
+tb-nightly==2.21.0a20251023
+referencing==0.36.2
+typing_extensions==4.15.0
+multidict==6.6.4
+lpips==0.1.4
+viser==0.2.23
+natsort==8.4.0
+mdurl==0.1.2
+wadler_lindig==0.1.6
+causal-conv1d==1.5.0.post8
+proglog==0.1.12
+docker-pycreds==0.4.0
+munch==4.0.0
+evo==1.31.1
+bitsandbytes==0.43.3
+humanfriendly==10.0
+typing-inspection==0.4.1
+fastapi==0.118.0
+lightning==2.5.1.post0
+pydantic==2.11.4
+torch-geometric==2.6.1
+prompt_toolkit==3.0.51
+lxml==5.4.0
+Werkzeug==3.1.8
+jupyterlab_widgets==3.0.15
+nvidia-nvtx==13.0.85
+pyglet==1.5.31
+pyceres==2.3
+msgspec==0.19.0
+dacite==1.9.2
+setuptools==81.0.0
+setuptools==79.0.1
+vhacdx==0.0.8.post2
+tensorboardX==2.6.4
+nvidia-nvshmem-cu13==3.4.5
+lz4==4.4.4
+click==8.1.8
+shapely==2.1.1
+torchmetrics==1.9.0
+typeguard==4.4.2
+numpy==2.4.3
+einops==0.6.1
+embreex==2.17.7.post6
+nvidia-nvjitlink==13.0.88
+nodeenv==1.9.1
+packaging==26.1
+dill==0.3.7
+semantic-version==2.10.0
+cycler==0.12.1
+termcolor==3.1.0
+fsspec==2023.10.0
+fsspec==2026.2.0
+safetensors==0.7.0
+colorspacious==1.1.2
+nvidia-cufft-cu12==11.0.2.54
+matplotlib==3.10.8
+lazy-loader==0.5
+mypy_extensions==1.1.0
+pip==26.0.1
+groovy==0.1.2
+bleach==6.3.0
+cuda-pathfinder==1.5.3
+idna==3.10
+scikit-video==1.1.11
+h5py==3.14.0
+plyfile==1.1.2
+frozenlist==1.7.0
+nvidia-cusolver==12.0.4.66
+regex==2025.7.34
+openai-clip==1.0.1
+basicsr==1.4.2
+nvidia-curand==10.4.0.35
+nvidia-cuda-cupti-cu12==12.1.105
+tomlkit==0.13.3
+nvidia-cublas==13.1.0.3
+scikit-image==0.26.0
+omegaconf==2.3.0
+shortuuid==1.0.13
+imageio-ffmpeg==0.6.0
+einops-exts==0.0.4
+rich==14.1.0
+pandas==3.0.2
+grpcio==1.80.0
+importlib_metadata==8.7.0
+droid_backends==0.0.0
+pybind11==3.0.1
+psutil==7.0.0
+lightglue==0.0
+pillow==12.1.1
+nvidia-nccl-cu13==2.28.9
+rosbags==0.10.10
+pyiqa==0.1.15.post15
+python-slugify==8.0.4
+iopath==0.1.10
+tokenizers==0.22.1
+torchvision==0.20.1+cu121
+certifi==2025.4.26
+Pygments==2.19.1
+ipywidgets==8.1.7
+nvidia-cuda-runtime-cu12==12.1.105
+scikit-learn==1.8.0
+kagglesdk==0.1.19
+setproctitle==1.3.6
+requests==2.32.3
+nvidia-cudnn-cu13==9.19.0.56
+gradio==5.47.2
+urllib3==1.26.20
+propcache==0.3.2
+tyro==0.9.21
+multiprocess==0.70.15
+text-unidecode==1.3
+orjson==3.11.3
+mpmath==1.3.0
+uvicorn==0.37.0
+lietorch==0.2
+moviepy==1.0.3
+networkx==3.6.1
+distlib==0.4.0
+torch_scatter==2.1.2
+yapf==0.43.0
+ipython==9.6.0
+spconv-cu120==2.3.6
+ninja==1.11.1.4
+fire==0.7.1
+torch==2.5.1+cu121
+nvidia-cuda-nvrtc-cu12==12.1.105
+pyarrow==22.0.0
+lark==1.3.0
+onnxruntime==1.22.0
+tabulate==0.9.0
+SharedArray==3.2.4
+pyDeprecate==0.3.2
+timm==1.0.21
+absl-py==2.4.0
+pytest==9.0.3
+markdown-it-py==4.0.0
+lmdb==2.2.0
+kiwisolver==1.5.0
+colorlog==6.9.0
+opencv-python-headless==4.13.0.92
+gdown==5.2.0
+roma==1.5.4
+matplotlib-inline==0.1.7
+cumm-cu120==0.4.11
+MarkupSafe==3.0.3
+pathspec==0.12.1
+open_clip_torch==3.2.0
+gsplat==1.5.3
+httpcore==0.17.3
+flatbuffers==25.2.10
+pydub==0.25.1
+opt-einsum-fx==0.1.4
+seaborn==0.13.2
+llvmlite==0.47.0
+smmap==5.0.2
+pycolmap==3.10.0
+nvidia-cusparselt-cu13==0.8.0
+attrs==25.3.0
+beartype==0.21.0
+pre_commit==4.5.1
+kornia==0.8.1
+cuda-toolkit==13.0.2
+jupyter_core==5.8.1
+typer==0.19.2
+datasets==2.16.1
+shtab==1.7.2
+pytz==2025.2
+nvidia-curand-cu12==10.3.2.106
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+shellingham==1.5.4
+python-dotenv==1.1.0
+Markdown==3.10.2
+nvidia-nccl-cu12==2.21.5
+wandb==0.19.11
+GitPython==3.1.44
+aiohttp==3.12.15
+ccimport==0.4.4
+hf_transfer==0.1.9
+jsonschema==4.23.0
+numba==0.65.0
+wheel==0.46.3
+xxhash==3.5.0
+accelerate==1.11.0
+PyYAML==6.0.2
+colorama==0.4.6
+numexpr==2.11.0
+kaggle==2.0.1
+rpds-py==0.25.1
+mapbox_earcut==1.0.3
+trimesh==4.8.3
+decord==0.6.0
+threadpoolctl==3.6.0
+sympy==1.13.1
+beautifulsoup4==4.13.5
+websockets==15.0.1
+hf-xet==1.1.5
+simple_knn==0.0.0
+jaraco.collections==5.1.0
+packaging==24.2
+packaging==26.0
+jaraco.functools==4.4.0
+jaraco.functools==4.0.1
+importlib_metadata==8.7.1
+importlib_metadata==8.0.0
+tomli==2.0.1
+tomli==2.4.0
+backports.tarfile==1.2.0
+zipp==3.23.0
+zipp==3.19.2
+typing_extensions==4.12.2
+jaraco.context==5.3.0
+jaraco.context==6.1.0
+typeguard==4.3.0
+autocommand==2.2.2
+platformdirs==4.4.0
+platformdirs==4.2.2
+jaraco.text==3.12.1
+jaraco.text==4.0.0
+more-itertools==10.3.0
+more-itertools==10.8.0
+wheel==0.45.1
+wheel==0.46.3
+inflect==7.3.1

exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/run-20260505_134952-91bfp1vi/files/wandb-metadata.json

@@ -0,0 +1,136 @@
+{
+  "os": "Linux-5.14.0-570.106.1.el9_6.x86_64-x86_64-with-glibc2.34",
+  "python": "CPython 3.11.15",
+  "startedAt": "2026-05-05T11:49:53.112298Z",
+  "args": [
+    "+training=finetune_dav3_gopro_defocus_l1",
+    "trainer.num_nodes=1",
+    "trainer.devices=4",
+    "checkpointing.load=/scratch-shared/qzhang1/C3G_outputs/exp_rsblur_v6_bordermask_from_18999/2026-05-05_06-22-33/checkpoints/best-psnr-val/psnr_19.072-info/global_step_2749.ckpt",
+    "checkpointing.load_weights_only=true",
+    "wandb.mode=online",
+    "wandb.name=gopro_defocus_from_rsblur_v6_19072"
+  ],
+  "program": "-m src.main",
+  "git": {
+    "remote": "https://github.com/cvlab-kaist/C3G.git",
+    "commit": "39242766ca5dffc45736113334d879a67d165228"
+  },
+  "email": "zswjsw@gmail.com",
+  "root": "/scratch-shared/qzhang1/C3G_outputs/exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53",
+  "host": "gcn142.local.snellius.surf.nl",
+  "executable": "/home/qzhang1/miniconda3/envs/c3g/bin/python3.11",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA H100",
+  "gpu_count": 4,
+  "disk": {
+    "/": {
+      "total": "405460803584",
+      "used": "4923211776"
+    }
+  },
+  "memory": {
+    "total": "811317751808"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H100",
+      "memoryTotal": "100485038080",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H100",
+      "memoryTotal": "100485038080",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H100",
+      "memoryTotal": "100485038080",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H100",
+      "memoryTotal": "100485038080",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    }
+  ],
+  "slurm": {
+    "cluster_name": "snellius",
+    "conf": "/var/spool/slurm/slurmd/conf-cache/slurm.conf",
+    "cpu_bind": "quiet,mask_cpu:0xFFFFFFFFFFFFFFFF",
+    "cpu_bind_list": "0xFFFFFFFFFFFFFFFF",
+    "cpu_bind_type": "mask_cpu:",
+    "cpu_bind_verbose": "quiet",
+    "cpus_on_node": "64",
+    "cpus_per_task": "64",
+    "distribution": "block",
+    "eclibr": "0",
+    "ecplug": "1",
+    "erlast": "srun",
+    "ersbac": "1",
+    "ersrun": "1",
+    "export_env": "ALL",
+    "get_user_env": "1",
+    "gpus_on_node": "4",
+    "gpus_per_node": "4",
+    "gtids": "0",
+    "job_account": "gusei15043",
+    "job_cpus_per_node": "64",
+    "job_end_time": "1778050293",
+    "job_gid": "74106",
+    "job_gpus": "0,1,2,3",
+    "job_group": "qzhang1",
+    "job_id": "22474941",
+    "job_name": "sys/dashboard/sys/code_server/form",
+    "job_nodelist": "gcn142",
+    "job_num_nodes": "1",
+    "job_partition": "gpu_h100",
+    "job_qos": "normal",
+    "job_start_time": "1777963892",
+    "job_uid": "74645",
+    "job_user": "qzhang1",
+    "jobid": "22474941",
+    "launch_node_ipaddr": "172.18.62.142",
+    "localid": "0",
+    "mpi_type": "pmi2",
+    "nnodes": "1",
+    "nodeid": "0",
+    "nodelist": "gcn142",
+    "nprocs": "1",
+    "ntasks": "1",
+    "oom_kill_step": "0",
+    "prio_process": "0",
+    "procid": "0",
+    "script_context": "prolog_task",
+    "srun_comm_host": "172.18.62.142",
+    "srun_comm_port": "46017",
+    "step_gpus": "0,1,2,3",
+    "step_gres": "gres/cpu:0",
+    "step_id": "1",
+    "step_launcher_port": "46017",
+    "step_nodelist": "gcn142",
+    "step_num_nodes": "1",
+    "step_num_tasks": "1",
+    "step_resv_ports": "17870-17871",
+    "step_tasks_per_node": "1",
+    "stepid": "1",
+    "submit_dir": "/var/www/ood/apps/sys/dashboard",
+    "submit_host": "sys11.local.snellius.surf.nl",
+    "task_pid": "2231309",
+    "tasks_per_node": "1",
+    "topology_addr": "root.ibsw42.gcn142",
+    "topology_addr_pattern": "switch.switch.node",
+    "tres_per_task": "cpu=64",
+    "umask": "0027"
+  },
+  "cudaVersion": "13.2"
+}

exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/run-20260505_134952-91bfp1vi/files/wandb-summary.json

@@ -0,0 +1 @@
+{"diag/log_aniso_mean":5.756680488586426,"projection":{"filenames":["media/images/projection_1_8a38d1c3c9c41d01f026.png"],"_type":"images/separated","width":800,"height":306,"format":"png","count":1},"diag/scale_min_mean":0.0008003668626770377,"_runtime":7.016545147,"trainer/global_step":0,"diag/aniso_mean":9.784027e+06,"diag/log_aniso_p99":7.483924388885498,"val/psnr":15.567266464233398,"info/n_gaussians_coarse_epoch":12800,"_wandb":{"runtime":13},"comparison":{"width":1384,"height":906,"format":"png","count":1,"filenames":["media/images/comparison_0_fde3f179d74ba9a3c0df.png"],"captions":["502"],"_type":"images/separated"},"cameras":{"filenames":["media/images/cameras_2_6c8af7739e560fc41875.png"],"_type":"images/separated","width":800,"height":306,"format":"png","count":1},"epoch":0,"_step":4,"diag/scale_max_mean":18.98053741455078,"val/ssim":0.33824753761291504,"_timestamp":1.7779818001286516e+09,"info/n_gaussians_coarse_step":12800,"diag/aniso_max":3.8765508e+07,"val/lpips":0.7240225076675415}

exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/run-20260505_134952-91bfp1vi/logs/debug-core.log

@@ -0,0 +1,13 @@
+{"time":"2026-05-05T13:49:52.091756199+02:00","level":"INFO","msg":"main: starting server","port-filename":"/scratch-shared/qzhang1/tmp/tmpm450e80m/port-2231583.txt","pid":2231583,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-05-05T13:49:52.58829374+02:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2231583}
+{"time":"2026-05-05T13:49:52.5883002+02:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":42331,"Zone":""}}
+{"time":"2026-05-05T13:49:52.632024274+02:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:51220"}
+{"time":"2026-05-05T13:49:53.114305083+02:00","level":"INFO","msg":"handleInformInit: received","streamId":"91bfp1vi","id":"127.0.0.1:51220"}
+{"time":"2026-05-05T13:49:53.475351889+02:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"91bfp1vi","id":"127.0.0.1:51220"}
+{"time":"2026-05-05T13:50:06.774923002+02:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:51220"}
+{"time":"2026-05-05T13:50:06.775077441+02:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-05-05T13:50:06.775067301+02:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:51220"}
+{"time":"2026-05-05T13:50:06.7752359+02:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:51220"}
+{"time":"2026-05-05T13:50:07.757267879+02:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:51220"}
+{"time":"2026-05-05T13:50:07.757291138+02:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:51220"}
+{"time":"2026-05-05T13:50:07.757303688+02:00","level":"INFO","msg":"server is closed"}

exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/run-20260505_134952-91bfp1vi/logs/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2026-05-05T13:49:53.115315174+02:00","level":"INFO","msg":"stream: starting","core version":"0.19.11","symlink path":"/scratch-shared/qzhang1/C3G_outputs/exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/run-20260505_134952-91bfp1vi/logs/debug-core.log"}
+{"time":"2026-05-05T13:49:53.473610235+02:00","level":"INFO","msg":"created new stream","id":"91bfp1vi"}
+{"time":"2026-05-05T13:49:53.475335599+02:00","level":"INFO","msg":"stream: started","id":"91bfp1vi"}
+{"time":"2026-05-05T13:49:53.475443838+02:00","level":"INFO","msg":"sender: started","stream_id":"91bfp1vi"}
+{"time":"2026-05-05T13:49:53.475504407+02:00","level":"INFO","msg":"handler: started","stream_id":"91bfp1vi"}
+{"time":"2026-05-05T13:49:53.475451708+02:00","level":"INFO","msg":"writer: Do: started","stream_id":"91bfp1vi"}
+{"time":"2026-05-05T13:49:54.336482342+02:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2026-05-05T13:50:06.775080191+02:00","level":"INFO","msg":"stream: closing","id":"91bfp1vi"}
+{"time":"2026-05-05T13:50:06.77516645+02:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2026-05-05T13:50:06.775294229+02:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2026-05-05T13:50:07.467926865+02:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-05-05T13:50:07.756723943+02:00","level":"INFO","msg":"handler: closed","stream_id":"91bfp1vi"}
+{"time":"2026-05-05T13:50:07.756787683+02:00","level":"INFO","msg":"writer: Close: closed","stream_id":"91bfp1vi"}
+{"time":"2026-05-05T13:50:07.756794253+02:00","level":"INFO","msg":"sender: closed","stream_id":"91bfp1vi"}
+{"time":"2026-05-05T13:50:07.75711223+02:00","level":"INFO","msg":"stream: closed","id":"91bfp1vi"}

exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/run-20260505_134952-91bfp1vi/logs/debug.log

@@ -0,0 +1,85 @@
+2026-05-05 13:49:52,978 INFO    MainThread:2231583 [wandb_setup.py:_flush():70] Current SDK version is 0.19.11
+2026-05-05 13:49:52,979 INFO    MainThread:2231583 [wandb_setup.py:_flush():70] Configure stats pid to 2231583
+2026-05-05 13:49:52,979 INFO    MainThread:2231583 [wandb_setup.py:_flush():70] Loading settings from /home/qzhang1/.config/wandb/settings
+2026-05-05 13:49:52,979 INFO    MainThread:2231583 [wandb_setup.py:_flush():70] Loading settings from /gpfs/home2/qzhang1/C3G/wandb/settings
+2026-05-05 13:49:52,979 INFO    MainThread:2231583 [wandb_setup.py:_flush():70] Loading settings from environment variables
+2026-05-05 13:49:52,979 INFO    MainThread:2231583 [wandb_init.py:setup_run_log_directory():724] Logging user logs to /scratch-shared/qzhang1/C3G_outputs/exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/run-20260505_134952-91bfp1vi/logs/debug.log
+2026-05-05 13:49:52,979 INFO    MainThread:2231583 [wandb_init.py:setup_run_log_directory():725] Logging internal logs to /scratch-shared/qzhang1/C3G_outputs/exp_gopro_defocus_from_rsblur_v6_19072/2026-05-05_13-47-53/wandb/run-20260505_134952-91bfp1vi/logs/debug-internal.log
+2026-05-05 13:49:52,979 INFO    MainThread:2231583 [wandb_init.py:init():852] calling init triggers
+2026-05-05 13:49:52,979 INFO    MainThread:2231583 [wandb_init.py:init():857] wandb.init called with sweep_config: {}
+config: {'model': {'encoder': {'backbone': {'name': 'croco', 'model': 'ViTLarge_BaseDecoder', 'patch_embed_cls': 'PatchEmbedDust3R', 'asymmetry_decoder': True, 'intrinsics_embed_loc': 'encoder', 'intrinsics_embed_degree': 4, 'intrinsics_embed_type': 'token'}, 'name': 'da3_resplat', 'opacity_mapping': {'initial': 0.0, 'final': 0.0, 'warm_up': 1}, 'num_monocular_samples': 32, 'num_surfaces': 1, 'predict_opacity': False, 'gaussians_per_pixel': 1, 'gaussian_adapter': {'gaussian_scale_min': 0.5, 'gaussian_scale_max': 15.0, 'sh_degree': 3}, 'd_feature': 128, 'visualizer': {'num_samples': 8, 'min_resolution': 256, 'export_ply': False}, 'apply_bounds_shim': True, 'gs_params_head_type': 'dpt_gs', 'pose_free': True, 'pretrained_weights': '', 'da3_checkpoint': '/gpfs/scratch1/shared/qzhang1/da3_pretrained/DA3-GIANT', 'resplat_checkpoint': '/gpfs/scratch1/shared/qzhang1/resplat_pretrained/resplat-base-dl3dv-256x448-view8-1934a04c.pth', 'freeze_da3': True, 'freeze_resplat': True, 'use_pred_pose': True}, 'decoder': {'name': 'splatting_cuda', 'background_color': [0.0, 0.0, 0.0], 'make_scale_invariant': True, 'low_pass_filter': 0.3, 'decrease_lpf_step': -1}, 'refiner': None, 'diffusion_head': {'enc_k': 256, 'enc_df': 128, 'enc_df_out': 128, 'enc_d_point': 64, 'enc_cross_depth': 4, 'enc_self_depth': 2, 'dec_dim_model': 64, 'dec_cross_depth': 3, 'dec_num_virtual': 256, 'train_num_steps': 3, 'infer_num_steps': 3, 'use_cam_cond': False, 'use_vggt_priors': True, 'use_cam_pred': False}}, 'loss': {'mse_l1': {'weight': 0.5, 'empty_render_thresh': 0.02}, 'lpips': {'weight': 0.15, 'apply_after_step': 0}, 'ssim': {'weight': 0.05}}, 'wandb': {'project': 'DAV3', 'entity': 'zswjsw-the-university-of-sydney', 'name': 'gopro_defocus_from_rsblur_v6_19072', 'mode': 'online', 'tags': ['gopro_defocus', 'diffusion_head', 'da3_resplat', 'finetune', 'pred_pose', 'single_view', 'curriculum']}, 'mode': 'train', 'data_loader': {'train': {'num_workers': 4, 'persistent_workers': True, 'batch_size': 1, 'seed': 1234}, 'test': {'num_workers': 2, 'persistent_workers': False, 'batch_size': 1, 'seed': 2345}, 'val': {'num_workers': 1, 'persistent_workers': False, 'batch_size': 1, 'seed': 3456}}, 'optimizer': {'lr': 2e-05, 'warm_up_steps': 200, 'backbone_lr_multiplier': 0.01}, 'checkpointing': {'load': '/scratch-shared/qzhang1/C3G_outputs/exp_rsblur_v6_bordermask_from_18999/2026-05-05_06-22-33/checkpoints/best-psnr-val/psnr_19.072-info/global_step_2749.ckpt', 'every_n_train_steps': 500, 'save_top_k': 1, 'save_weights_only': False, 'load_weights_only': True}, 'train': {'depth_mode': 'depth', 'extended_visualization': False, 'print_log_every_n_steps': 10, 'context_view_loss': False, 'random_select_context_view': False, 'freeze_encoder': True, 'share_weight': 0.0, 'l_flow_weight': 0.0, 'cam_loss_weight': 0.0}, 'test': {'output_path': 'outputs/test', 'align_pose': True, 'pose_align_steps': 100, 'rot_opt_lr': 0.005, 'trans_opt_lr': 0.005, 'compute_scores': True, 'save_image': True, 'save_video': False, 'save_compare': False, 'use_tto': False, 'tto_steps': 200, 'tto_lr': 0.001, 'tto_geo_lr_scale': 0.1}, 'seed': 111123, 'trainer': {'max_steps': 20000, 'val_check_interval': 500, 'gradient_clip_val': 1.0, 'num_nodes': 1, 'accumulate_grad_batches': 4, 'devices': 4, 'strategy': 'ddp', 'enable_progress_bar': True, 'num_sanity_val_steps': 0, 'precision': 'bf16-mixed'}, 'dataset': {'gopro_defocus': {'make_baseline_1': False, 'relative_pose': False, 'augment': False, 'background_color': [0.0, 0.0, 0.0], 'overfit_to_scene': None, 'skip_bad_shape': False, 'view_sampler': {'name': 'bounded', 'num_target_views': 1, 'num_context_views': 2, 'min_distance_between_context_views': 2, 'max_distance_between_context_views': 6, 'min_distance_to_context_views': 0, 'warm_up_steps': 0, 'initial_min_distance_between_context_views': 2, 'initial_max_distance_between_context_views': 6}, 'name': 'gopro_defocus', 'roots': ['/scratch-shared/qzhang1/datasets/gopro_defocus/Go_pro_defocus_train'], 'near': 0.1, 'far': 100.0, 'input_image_shape': [336, 336], 'original_image_shape': [336, 336], 'cameras_are_circular': False, 'baseline_min': 0.0, 'baseline_max': 10000000000.0, 'max_fov': 180.0, 'val_ids_start': 502, 'samples_per_scene': 1, 'val_samples_per_scene': 1}}, '_wandb': {}}
+2026-05-05 13:49:53,100 INFO    MainThread:2231583 [wandb_init.py:init():893] starting backend
+2026-05-05 13:49:53,100 INFO    MainThread:2231583 [wandb_init.py:init():897] sending inform_init request
+2026-05-05 13:49:53,111 INFO    MainThread:2231583 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-05-05 13:49:53,112 INFO    MainThread:2231583 [wandb_init.py:init():907] backend started and connected
+2026-05-05 13:49:53,115 INFO    MainThread:2231583 [wandb_init.py:init():1005] updated telemetry
+2026-05-05 13:49:53,134 INFO    MainThread:2231583 [wandb_init.py:init():1029] communicating run to backend with 90.0 second timeout
+2026-05-05 13:49:54,293 INFO    MainThread:2231583 [wandb_init.py:init():1104] starting run threads in backend
+2026-05-05 13:49:54,770 INFO    MainThread:2231583 [wandb_run.py:_console_start():2573] atexit reg
+2026-05-05 13:49:54,770 INFO    MainThread:2231583 [wandb_run.py:_redirect():2421] redirect: wrap_raw
+2026-05-05 13:49:54,771 INFO    MainThread:2231583 [wandb_run.py:_redirect():2490] Wrapping output streams.
+2026-05-05 13:49:54,771 INFO    MainThread:2231583 [wandb_run.py:_redirect():2513] Redirects installed.
+2026-05-05 13:49:54,776 INFO    MainThread:2231583 [wandb_init.py:init():1150] run started, returning control to user process
+2026-05-05 13:50:06,774 INFO    MsgRouterThr:2231583 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.
+2026-05-05 13:50:08,594 ERROR   MainThread:2231583 [redirect.py:_on_write():664] [no run ID] error in stdout callback
+Traceback (most recent call last):
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
+    cb(written_data)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2494, in <lambda>
+    lambda data: self._console_raw_callback("stdout", data),
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 406, in wrapper
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 464, in wrapper_fn
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1639, in _console_raw_callback
+    self._backend.interface.publish_output_raw(name, data)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 762, in publish_output_raw
+    self._publish_output_raw(o)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
+    self._publish(rec)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
+    self._sock_client.send_record_publish(record)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
+    self.send_server_request(server_req)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
+    self._send_message(msg)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
+    self._sendall_with_error_handle(header + data)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
+    sent = self._sock.send(data)
+           ^^^^^^^^^^^^^^^^^^^^^
+BrokenPipeError: [Errno 32] Broken pipe
+2026-05-05 13:50:08,600 ERROR   MainThread:2231583 [redirect.py:_on_write():664] [no run ID] error in stdout callback
+Traceback (most recent call last):
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/redirect.py", line 662, in _on_write
+    cb(written_data)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2494, in <lambda>
+    lambda data: self._console_raw_callback("stdout", data),
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 406, in wrapper
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 464, in wrapper_fn
+    return func(self, *args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 1639, in _console_raw_callback
+    self._backend.interface.publish_output_raw(name, data)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 762, in publish_output_raw
+    self._publish_output_raw(o)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/interface/interface_shared.py", line 38, in _publish_output_raw
+    self._publish(rec)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/interface/interface_sock.py", line 39, in _publish
+    self._sock_client.send_record_publish(record)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 174, in send_record_publish
+    self.send_server_request(server_req)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 154, in send_server_request
+    self._send_message(msg)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 151, in _send_message
+    self._sendall_with_error_handle(header + data)
+  File "/home/qzhang1/miniconda3/envs/c3g/lib/python3.11/site-packages/wandb/sdk/lib/sock_client.py", line 130, in _sendall_with_error_handle
+    sent = self._sock.send(data)
+           ^^^^^^^^^^^^^^^^^^^^^
+BrokenPipeError: [Errno 32] Broken pipe