add droplidar and dinov2 models

.gitignore

@@ -2,3 +2,5 @@
 **/my_experiment
 **/wandb
 **/visualizations
+**/epoch_*.json
+**/epoch_*.pth

pix2poly/224/building_fusion/.hydra/config.yaml

@@ -0,0 +1,122 @@
+host:
+  name: gin
+  data_root: /data/rsulzer/${..dataset.name}_dataset
+  model_root: /data/rsulzer/${..dataset.name}_output
+  multi_gpu: false
+  device: cuda
+  update_pbar_every: 1
+  ldof_exe: /user/rsulzer/home/cpp/line-DOF-metric/build/calculate_DoF
+  dino_v2_repo: /user/rsulzer/home/remote_python/dinov2
+run_type:
+  name: release
+  batch_size: 16
+  train_subset: null
+  val_subset: null
+  test_subset: null
+  logging: INFO
+  num_workers: 16
+  log_to_wandb: true
+dataset:
+  name: PixelsPointsPolygons
+  size: ${..experiment.encoder.in_size}
+  path: ${host.data_root}/data/${.size}
+  annotations:
+    train: ${..path}/annotations/${...experiment.annotation_type}/annotations_${...experiment.country}_train.json
+    val: ${..path}/annotations/${...experiment.annotation_type}/annotations_${...experiment.country}_val.json
+    test: ${..path}/annotations/${...experiment.annotation_type}/annotations_${...experiment.country}_test.json
+  ffl_stats:
+    train: ${..path}/ffl/train/stats-${...experiment.country}.pt
+    val: ${..path}/ffl/val/stats-${...experiment.country}.pt
+    test: ${..path}/ffl/test/stats-${...experiment.country}.pt
+  train_subset: ${..run_type.train_subset}
+  val_subset: ${..run_type.val_subset}
+  test_subset: ${..run_type.test_subset}
+experiment:
+  encoder:
+    name: early_fusion_vit
+    use_images: true
+    use_lidar: true
+    in_size: 224
+    in_height: ${.in_size}
+    in_width: ${.in_size}
+    in_voxel_size:
+      x: 8
+      'y': 8
+      z: 100
+    max_num_points_per_voxel: 64
+    max_num_voxels:
+      train: 784
+      test: 784
+    out_feature_width: 28
+    out_feature_height: 28
+    vit:
+      type: vit_small_patch${..patch_size}_${..in_size}.dino
+      checkpoint_file: ${....host.model_root}/backbones/dino_deitsmall8_pretrain.pth
+      pretrained: true
+    patch_size: 8
+    patch_feature_size: 28
+    patch_feature_height: ${.patch_feature_size}
+    patch_feature_width: ${.patch_feature_size}
+    patch_feature_dim: 384
+    num_patches: 784
+    out_feature_dim: ${..model.decoder.in_feature_dim}
+    image_mean:
+    - 0.0
+    - 0.0
+    - 0.0
+    image_std:
+    - 1.0
+    - 1.0
+    - 1.0
+    image_max_pixel_value: 255.0
+    augmentations:
+    - D4
+    - ColorJitter
+    - GaussNoise
+    - Normalize
+  model:
+    name: pix2poly
+    decoder:
+      in_feature_size: ${...encoder.patch_feature_size}
+      in_feature_width: ${.in_feature_size}
+      in_feature_height: ${.in_feature_size}
+      in_feature_dim: 256
+    tokenizer:
+      num_bins: ${...encoder.in_size}
+      shuffle_tokens: false
+      n_vertices: 192
+      max_len: null
+      pad_idx: null
+      generation_steps: null
+    fusion: patch_concat
+    sinkhorn_iterations: 100
+    label_smoothing: 0.0
+    vertex_loss_weight: 1.0
+    perm_loss_weight: 10.0
+    batch_size: ${...run_type.batch_size}
+    start_epoch: 0
+    num_epochs: 200
+    milestone: 0
+    learning_rate: 0.0003
+    weight_decay: 0.0001
+  name: building_fusion
+  group_name: v3_${.model.name}
+  lidar_dropout: null
+  country: CH
+  annotation_type: buildings
+training:
+  save_best: true
+  save_latest: true
+  save_every: 10
+  val_every: 1
+  best_val_loss: 10000000.0
+  best_val_iou: 0.0
+evaluation:
+  split: val
+  pred_file: ${..output_dir}/predictions_${..experiment.country}_${.split}/${..checkpoint}.json
+  modes:
+  - iou
+  eval_file: results/metrics
+output_dir: ${.host.model_root}/${.experiment.model.name}/${.experiment.encoder.in_size}/${.experiment.name}
+checkpoint: null
+num_workers: ${.run_type.num_workers}

pix2poly/224/building_fusion/.hydra/hydra.yaml

@@ -0,0 +1,165 @@
+hydra:
+  run:
+    dir: ${...output_dir}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - host=gin
+    - experiment=p2p_fusion_building_ann
+    - run_type=release
+  job:
+    name: train
+    chdir: null
+    override_dirname: experiment=p2p_fusion_building_ann,host=gin,run_type=release
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /run/netsop/u/home-sam/home/rsulzer/remote_python/pixelspointspolygons
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /run/netsop/u/home-sam/home/rsulzer/remote_python/pixelspointspolygons/config
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /data/rsulzer/PixelsPointsPolygons_output/pix2poly/224/building_fusion
+    choices:
+      evaluation: val
+      training: default
+      experiment: p2p_fusion_building_ann
+      model@experiment.model: pix2poly
+      encoder@experiment.encoder: early_fusion_vit
+      dataset: p3
+      run_type: release
+      host: gin
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

pix2poly/224/building_fusion/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- host=gin
+- experiment=p2p_fusion_building_ann
+- run_type=release

pix2poly/224/dinov2/predictions_CH_test/best_val_iou.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:885e18a6b32ab993013ed2aa5679b889ccc8aadbb1fa8c9c5835fec67b1c9d76
+size 8038122

pix2poly/224/droplidar50/checkpoints/best_val_iou.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c59131c648411f51d711a69d01be0575cba0fc4c4693f18c936cb396c07ba98c
+oid sha256:ab3906c56bc30f0494bfba541c5ac4dca4e6d1971b048740eaf1e0167f821d9e
 size 415470058

pix2poly/224/droplidar50/checkpoints/best_val_loss.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0849339aa774cb14b36297e485de8367139a34b22324874f21b203fa3be0d4ac
+oid sha256:1ff17ce33bf2d9f904ebc9619480b2a9eb68e60d10a348c6e911affa7465c162
 size 415471297

pix2poly/224/droplidar50/checkpoints/latest.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e97e714f98b1c64b82ed8338cdf76f3b80412539e68a642f994f447da7013284
+oid sha256:1d5d511bbab67dd0f62be1c0baeeda2e8c307c42ae843044ec172c37b97bf634
 size 415458080

pix2poly/224/droplidar50/predictions_CH_val/best_val_iou.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a9bb7924edd3d65f911ca3c5a123b37167f2033d79d8dd48e854bdb8912a9b57
-size 275136
+oid sha256:a19854cc394e253a6692ef25becbd0f265d4db0b9a78bbd7c1cccb8743ed03ec
+size 477324

pix2poly/224/early_fusion_bs2x16_mnv64/.hydra/config.yaml

@@ -1,20 +1,21 @@
 host:
   name: gin
-  data_root: /data/rsulzer/${..dataset.name}
+  data_root: /data/rsulzer/${..dataset.name}_dataset
   model_root: /data/rsulzer/${..dataset.name}_output
   multi_gpu: false
   device: cuda
   update_pbar_every: 1
   ldof_exe: /user/rsulzer/home/cpp/line-DOF-metric/build/calculate_DoF
+  dino_v2_repo: /user/rsulzer/home/remote_python/dinov2
 run_type:
-  name: debug
+  name: release
   batch_size: 16
-  train_subset: 256
-  val_subset: 32
-  test_subset: 32
-  logging: DEBUG
-  num_workers: 0
-  log_to_wandb: false
+  train_subset: null
+  val_subset: null
+  test_subset: null
+  logging: INFO
+  num_workers: 16
+  log_to_wandb: true
 dataset:
   name: PixelsPointsPolygons
   size: ${..experiment.encoder.in_size}
@@ -100,6 +101,7 @@ experiment:
     weight_decay: 0.0001
   name: early_fusion_bs2x16_mnv64
   group_name: v2_${.model.name}
+  lidar_dropout: null
   country: all
 training:
   save_best: true
@@ -114,7 +116,6 @@ evaluation:
   modes:
   - iou
   eval_file: results/metrics
-experiment.name: debug
 output_dir: ${.host.model_root}/${.experiment.model.name}/${.experiment.encoder.in_size}/${.experiment.name}
 checkpoint: best_val_iou
 num_workers: ${.run_type.num_workers}

pix2poly/224/early_fusion_bs2x16_mnv64/.hydra/hydra.yaml

@@ -112,16 +112,15 @@ hydra:
     hydra:
     - hydra.mode=RUN
     task:
-    - run_type=debug
-    - host=gin
     - checkpoint=best_val_iou
     - experiment=p2p_fusion
     - +image_file=demo_data/image0_CH_val.tif
     - +lidar_file=demo_data/lidar0_CH_val.copc.laz
+    - host=gin
   job:
     name: predict_demo
     chdir: null
-    override_dirname: +image_file=demo_data/image0_CH_val.tif,+lidar_file=demo_data/lidar0_CH_val.copc.laz,checkpoint=best_val_iou,experiment=p2p_fusion,host=gin,run_type=debug
+    override_dirname: +image_file=demo_data/image0_CH_val.tif,+lidar_file=demo_data/lidar0_CH_val.copc.laz,checkpoint=best_val_iou,experiment=p2p_fusion,host=gin
     id: ???
     num: ???
     config_name: config
@@ -154,7 +153,7 @@ hydra:
       model@experiment.model: pix2poly
       encoder@experiment.encoder: early_fusion_vit
       dataset: p3
-      run_type: debug
+      run_type: release
       host: gin
       hydra/env: default
       hydra/callbacks: null

pix2poly/224/early_fusion_bs2x16_mnv64/.hydra/overrides.yaml

@@ -1,6 +1,5 @@
-- run_type=debug
-- host=gin
 - checkpoint=best_val_iou
 - experiment=p2p_fusion
 - +image_file=demo_data/image0_CH_val.tif
 - +lidar_file=demo_data/lidar0_CH_val.copc.laz
+- host=gin

pix2poly/224/image_bs2x16/.hydra/config.yaml

@@ -0,0 +1,110 @@
+host:
+  name: g5k
+  data_root: /home/rsulzer/data_group/${..dataset.name}_dataset
+  model_root: /home/rsulzer/data_group/${..dataset.name}_output
+  multi_gpu: true
+  device: cuda
+  update_pbar_every: 60
+  ldof_exe: null
+  dino_v2_repo: /home/rsulzer/python/dinov2
+run_type:
+  name: release
+  batch_size: 16
+  train_subset: null
+  val_subset: null
+  test_subset: null
+  logging: INFO
+  num_workers: 16
+  log_to_wandb: true
+dataset:
+  name: PixelsPointsPolygons
+  size: ${..experiment.encoder.in_size}
+  path: ${host.data_root}/data/${.size}
+  annotations:
+    train: ${..path}/annotations/annotations_${...experiment.country}_train.json
+    val: ${..path}/annotations/annotations_${...experiment.country}_val.json
+    test: ${..path}/annotations/annotations_${...experiment.country}_test.json
+  ffl_stats:
+    train: ${..path}/ffl/train/stats-${...experiment.country}.pt
+    val: ${..path}/ffl/val/stats-${...experiment.country}.pt
+    test: ${..path}/ffl/test/stats-${...experiment.country}.pt
+  train_subset: ${..run_type.train_subset}
+  val_subset: ${..run_type.val_subset}
+  test_subset: ${..run_type.test_subset}
+experiment:
+  encoder:
+    name: vit
+    use_images: true
+    use_lidar: false
+    type: vit_small_patch${.patch_size}_${.in_size}.dino
+    checkpoint_file: ${...host.model_root}/backbones/dino_deitsmall8_pretrain.pth
+    pretrained: true
+    in_size: 224
+    in_height: ${.in_size}
+    in_width: ${.in_size}
+    patch_size: 8
+    patch_feature_size: 28
+    patch_feature_height: ${.patch_feature_size}
+    patch_feature_width: ${.patch_feature_size}
+    patch_feature_dim: 384
+    num_patches: 784
+    out_feature_dim: ${..model.decoder.in_feature_dim}
+    image_mean:
+    - 0.0
+    - 0.0
+    - 0.0
+    image_std:
+    - 1.0
+    - 1.0
+    - 1.0
+    image_max_pixel_value: 255.0
+    augmentations:
+    - D4
+    - ColorJitter
+    - GaussNoise
+    - Normalize
+  model:
+    name: pix2poly
+    decoder:
+      in_feature_size: ${...encoder.patch_feature_size}
+      in_feature_width: ${.in_feature_size}
+      in_feature_height: ${.in_feature_size}
+      in_feature_dim: 256
+    tokenizer:
+      num_bins: ${...encoder.in_size}
+      shuffle_tokens: false
+      n_vertices: 192
+      max_len: null
+      pad_idx: null
+      generation_steps: null
+    fusion: patch_concat
+    sinkhorn_iterations: 100
+    label_smoothing: 0.0
+    vertex_loss_weight: 1.0
+    perm_loss_weight: 10.0
+    batch_size: ${...run_type.batch_size}
+    start_epoch: 0
+    num_epochs: 200
+    milestone: 0
+    learning_rate: 0.0003
+    weight_decay: 0.0001
+  name: image_bs2x16
+  group_name: v3_pix2poly
+  lidar_dropout: null
+  country: CH
+training:
+  save_best: true
+  save_latest: true
+  save_every: 10
+  val_every: 1
+  best_val_loss: 10000000.0
+  best_val_iou: 0.0
+evaluation:
+  split: val
+  pred_file: ${..output_dir}/predictions_${..experiment.country}_${.split}/${..checkpoint}.json
+  modes:
+  - iou
+  eval_file: results/metrics
+output_dir: ${.host.model_root}/${.experiment.model.name}/${.experiment.encoder.in_size}/${.experiment.name}
+checkpoint: null
+num_workers: ${.run_type.num_workers}

pix2poly/224/image_bs2x16/.hydra/hydra.yaml

@@ -0,0 +1,167 @@
+hydra:
+  run:
+    dir: ${...output_dir}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - run_type=release
+    - host=g5k
+    - experiment=p2p_image
+    - experiment.name=image_bs2x16
+    - experiment.group_name=v3_pix2poly
+  job:
+    name: train
+    chdir: null
+    override_dirname: experiment.group_name=v3_pix2poly,experiment.name=image_bs2x16,experiment=p2p_image,host=g5k,run_type=release
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /home/rsulzer/python/PixelsPointsPolygons
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/rsulzer/python/PixelsPointsPolygons/config
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/rsulzer/data_group/PixelsPointsPolygons_output/pix2poly/224/image_bs2x16
+    choices:
+      evaluation: val
+      training: default
+      experiment: p2p_image
+      model@experiment.model: pix2poly
+      encoder@experiment.encoder: vit
+      dataset: p3
+      run_type: release
+      host: g5k
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

pix2poly/224/image_bs2x16/.hydra/overrides.yaml

@@ -0,0 +1,5 @@
+- run_type=release
+- host=g5k
+- experiment=p2p_image
+- experiment.name=image_bs2x16
+- experiment.group_name=v3_pix2poly

pix2poly/224/image_bs2x16/checkpoints/best_val_iou.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57ae8f4db9e0d31d17e951e297a14af11c3ace15a94fbf493960f54c32832422
+size 383278438

pix2poly/224/image_bs2x16/checkpoints/best_val_loss.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91340473a55e36998de5b0ef0592d42f6546ba8b9bc8cd62f29061f3db4f3070
+size 383279628

pix2poly/224/image_bs2x16/checkpoints/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4e8e8b6bd84ca7355ff6dd5a2a4c69af49ec33f038161a332deb580de3a6027
+size 383266562

pix2poly/224/image_bs2x16/predictions_CH_val/best_val_iou.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:417117afafadfe360062181df75c55caedac3d370159a5b17cdfec528687196e
+size 467261

pix2poly/224/p2p_fusion_droplidar50_bs2x16/.hydra/config.yaml

@@ -0,0 +1,121 @@
+host:
+  name: g5k
+  data_root: /home/rsulzer/data_group/${..dataset.name}_dataset
+  model_root: /home/rsulzer/data_group/${..dataset.name}_output
+  multi_gpu: true
+  device: cuda
+  update_pbar_every: 60
+  ldof_exe: null
+  dino_v2_repo: /home/rsulzer/python/dinov2
+run_type:
+  name: release
+  batch_size: 16
+  train_subset: null
+  val_subset: null
+  test_subset: null
+  logging: INFO
+  num_workers: 16
+  log_to_wandb: true
+dataset:
+  name: PixelsPointsPolygons
+  size: ${..experiment.encoder.in_size}
+  path: ${host.data_root}/data/${.size}
+  annotations:
+    train: ${..path}/annotations/annotations_${...experiment.country}_train.json
+    val: ${..path}/annotations/annotations_${...experiment.country}_val.json
+    test: ${..path}/annotations/annotations_${...experiment.country}_test.json
+  ffl_stats:
+    train: ${..path}/ffl/train/stats-${...experiment.country}.pt
+    val: ${..path}/ffl/val/stats-${...experiment.country}.pt
+    test: ${..path}/ffl/test/stats-${...experiment.country}.pt
+  train_subset: ${..run_type.train_subset}
+  val_subset: ${..run_type.val_subset}
+  test_subset: ${..run_type.test_subset}
+experiment:
+  encoder:
+    name: early_fusion_vit
+    use_images: true
+    use_lidar: true
+    in_size: 224
+    in_height: ${.in_size}
+    in_width: ${.in_size}
+    in_voxel_size:
+      x: 8
+      'y': 8
+      z: 100
+    max_num_points_per_voxel: 64
+    max_num_voxels:
+      train: 784
+      test: 784
+    out_feature_width: 28
+    out_feature_height: 28
+    vit:
+      type: vit_small_patch${..patch_size}_${..in_size}.dino
+      checkpoint_file: ${....host.model_root}/backbones/dino_deitsmall8_pretrain.pth
+      pretrained: true
+    patch_size: 8
+    patch_feature_size: 28
+    patch_feature_height: ${.patch_feature_size}
+    patch_feature_width: ${.patch_feature_size}
+    patch_feature_dim: 384
+    num_patches: 784
+    out_feature_dim: ${..model.decoder.in_feature_dim}
+    image_mean:
+    - 0.0
+    - 0.0
+    - 0.0
+    image_std:
+    - 1.0
+    - 1.0
+    - 1.0
+    image_max_pixel_value: 255.0
+    augmentations:
+    - D4
+    - ColorJitter
+    - GaussNoise
+    - Normalize
+  model:
+    name: pix2poly
+    decoder:
+      in_feature_size: ${...encoder.patch_feature_size}
+      in_feature_width: ${.in_feature_size}
+      in_feature_height: ${.in_feature_size}
+      in_feature_dim: 256
+    tokenizer:
+      num_bins: ${...encoder.in_size}
+      shuffle_tokens: false
+      n_vertices: 192
+      max_len: null
+      pad_idx: null
+      generation_steps: null
+    fusion: patch_concat
+    sinkhorn_iterations: 100
+    label_smoothing: 0.0
+    vertex_loss_weight: 1.0
+    perm_loss_weight: 10.0
+    batch_size: ${...run_type.batch_size}
+    start_epoch: 0
+    num_epochs: 200
+    milestone: 0
+    learning_rate: 0.0003
+    weight_decay: 0.0001
+  name: p2p_fusion_droplidar50_bs2x16
+  group_name: v3_${.model.name}
+  lidar_dropout: 0.5
+  country: CH
+training:
+  save_best: true
+  save_latest: true
+  save_every: 10
+  val_every: 1
+  best_val_loss: 10000000.0
+  best_val_iou: 0.0
+evaluation:
+  split: val
+  pred_file: ${..output_dir}/predictions_${..experiment.country}_${.split}/${..checkpoint}.json
+  modes:
+  - iou
+  eval_file: results/metrics
+output_dir: ${.host.model_root}/${.experiment.model.name}/${.experiment.encoder.in_size}/${.experiment.name}
+checkpoint: null
+num_workers: ${.run_type.num_workers}

pix2poly/224/p2p_fusion_droplidar50_bs2x16/.hydra/hydra.yaml

@@ -0,0 +1,166 @@
+hydra:
+  run:
+    dir: ${...output_dir}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - run_type=release
+    - host=g5k
+    - experiment=p2p_fusion_droplidar50
+    - experiment.name=p2p_fusion_droplidar50_bs2x16
+  job:
+    name: train
+    chdir: null
+    override_dirname: experiment.name=p2p_fusion_droplidar50_bs2x16,experiment=p2p_fusion_droplidar50,host=g5k,run_type=release
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /home/rsulzer/python/PixelsPointsPolygons
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/rsulzer/python/PixelsPointsPolygons/config
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/rsulzer/data_group/PixelsPointsPolygons_output/pix2poly/224/p2p_fusion_droplidar50_bs2x16
+    choices:
+      evaluation: val
+      training: default
+      experiment: p2p_fusion_droplidar50
+      model@experiment.model: pix2poly
+      encoder@experiment.encoder: early_fusion_vit
+      dataset: p3
+      run_type: release
+      host: g5k
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

pix2poly/224/p2p_fusion_droplidar50_bs2x16/.hydra/overrides.yaml

@@ -0,0 +1,4 @@
+- run_type=release
+- host=g5k
+- experiment=p2p_fusion_droplidar50
+- experiment.name=p2p_fusion_droplidar50_bs2x16

pix2poly/224/p2p_fusion_droplidar50_bs2x16/checkpoints/best_val_iou.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f37d0972ee805354d8d07132ba62bf9e9940a0c24bbd42d6c2464148bc7300d6
+size 415475370

pix2poly/224/p2p_fusion_droplidar50_bs2x16/checkpoints/best_val_loss.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e68dcb8a7cd84c837df11619cc74850c331456609d4cce9ad790a7325993eb3d
+size 415476609

pix2poly/224/p2p_fusion_droplidar50_bs2x16/checkpoints/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21c30f76381250bf79a282d93c4460de2bcdb591ed2bd666b6fb5ec96195f483
+size 415463392

pix2poly/224/p2p_fusion_droplidar50_bs2x16/predictions_CH_val/best_val_iou.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1e16399f927d12c76f3c337e6c3ecc1ba58882ea9cc29ecba62131a2a0b0477
+size 466962

pix2poly/224/p2p_fusion_droplidar50_bs4x16/.hydra/config.yaml

@@ -0,0 +1,120 @@
+host:
+  name: jeanzay
+  data_root: /linkhome/rech/genlgm01/uku93eu/data/${..dataset.name}_dataset
+  model_root: /linkhome/rech/genlgm01/uku93eu/data/${..dataset.name}_output
+  multi_gpu: true
+  device: cuda
+  update_pbar_every: 60
+  ldof_exe: null
+run_type:
+  name: release
+  batch_size: 16
+  train_subset: null
+  val_subset: null
+  test_subset: null
+  logging: INFO
+  num_workers: 16
+  log_to_wandb: true
+dataset:
+  name: PixelsPointsPolygons
+  size: ${..experiment.encoder.in_size}
+  path: ${host.data_root}/data/${.size}
+  annotations:
+    train: ${..path}/annotations/annotations_${...experiment.country}_train.json
+    val: ${..path}/annotations/annotations_${...experiment.country}_val.json
+    test: ${..path}/annotations/annotations_${...experiment.country}_test.json
+  ffl_stats:
+    train: ${..path}/ffl/train/stats-${...experiment.country}.pt
+    val: ${..path}/ffl/val/stats-${...experiment.country}.pt
+    test: ${..path}/ffl/test/stats-${...experiment.country}.pt
+  train_subset: ${..run_type.train_subset}
+  val_subset: ${..run_type.val_subset}
+  test_subset: ${..run_type.test_subset}
+experiment:
+  encoder:
+    name: early_fusion_vit
+    use_images: true
+    use_lidar: true
+    in_size: 224
+    in_height: ${.in_size}
+    in_width: ${.in_size}
+    in_voxel_size:
+      x: 8
+      'y': 8
+      z: 100
+    max_num_points_per_voxel: 64
+    max_num_voxels:
+      train: 784
+      test: 784
+    out_feature_width: 28
+    out_feature_height: 28
+    vit:
+      type: vit_small_patch${..patch_size}_${..in_size}.dino
+      checkpoint_file: ${....host.model_root}/backbones/dino_deitsmall8_pretrain.pth
+      pretrained: true
+    patch_size: 8
+    patch_feature_size: 28
+    patch_feature_height: ${.patch_feature_size}
+    patch_feature_width: ${.patch_feature_size}
+    patch_feature_dim: 384
+    num_patches: 784
+    out_feature_dim: ${..model.decoder.in_feature_dim}
+    image_mean:
+    - 0.0
+    - 0.0
+    - 0.0
+    image_std:
+    - 1.0
+    - 1.0
+    - 1.0
+    image_max_pixel_value: 255.0
+    augmentations:
+    - D4
+    - ColorJitter
+    - GaussNoise
+    - Normalize
+  model:
+    name: pix2poly
+    decoder:
+      in_feature_size: ${...encoder.patch_feature_size}
+      in_feature_width: ${.in_feature_size}
+      in_feature_height: ${.in_feature_size}
+      in_feature_dim: 256
+    tokenizer:
+      num_bins: ${...encoder.in_size}
+      shuffle_tokens: false
+      n_vertices: 192
+      max_len: null
+      pad_idx: null
+      generation_steps: null
+    fusion: patch_concat
+    sinkhorn_iterations: 100
+    label_smoothing: 0.0
+    vertex_loss_weight: 1.0
+    perm_loss_weight: 10.0
+    batch_size: ${...run_type.batch_size}
+    start_epoch: 0
+    num_epochs: 200
+    milestone: 0
+    learning_rate: 0.0003
+    weight_decay: 0.0001
+  name: p2p_fusion_droplidar50_bs4x16
+  group_name: v3_${.model.name}
+  lidar_dropout: 0.5
+  country: CH
+training:
+  save_best: true
+  save_latest: true
+  save_every: 10
+  val_every: 1
+  best_val_loss: 10000000.0
+  best_val_iou: 0.0
+evaluation:
+  split: val
+  pred_file: ${..output_dir}/predictions_${..experiment.country}_${.split}/${..checkpoint}.json
+  modes:
+  - iou
+  eval_file: results/metrics
+output_dir: ${.host.model_root}/${.experiment.model.name}/${.experiment.encoder.in_size}/${.experiment.name}
+checkpoint: null
+num_workers: ${.run_type.num_workers}

pix2poly/224/p2p_fusion_droplidar50_bs4x16/.hydra/hydra.yaml

@@ -0,0 +1,166 @@
+hydra:
+  run:
+    dir: ${...output_dir}
+  sweep:
+    dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - run_type=release
+    - host=jz
+    - experiment=p2p_fusion_droplidar50
+    - experiment.name=p2p_fusion_droplidar50_bs4x16
+  job:
+    name: train
+    chdir: null
+    override_dirname: experiment.name=p2p_fusion_droplidar50_bs4x16,experiment=p2p_fusion_droplidar50,host=jz,run_type=release
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /lustre/fswork/projects/rech/cso/uku93eu/python/PixelsPointsPolygons
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /lustre/fswork/projects/rech/cso/uku93eu/python/PixelsPointsPolygons/config
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /linkhome/rech/genlgm01/uku93eu/data/PixelsPointsPolygons_output/pix2poly/224/p2p_fusion_droplidar50_bs4x16
+    choices:
+      evaluation: val
+      training: default
+      experiment: p2p_fusion_droplidar50
+      model@experiment.model: pix2poly
+      encoder@experiment.encoder: early_fusion_vit
+      dataset: p3
+      run_type: release
+      host: jz
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

pix2poly/224/p2p_fusion_droplidar50_bs4x16/.hydra/overrides.yaml

@@ -0,0 +1,4 @@
+- run_type=release
+- host=jz
+- experiment=p2p_fusion_droplidar50
+- experiment.name=p2p_fusion_droplidar50_bs4x16

pix2poly/224/p2p_fusion_droplidar50_bs4x16/checkpoints/best_val_iou.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cea42f1ce35f4bcf51a97fc37942a09252ad853ea7290e675e8e44f03ac689fe
+size 415475370

pix2poly/224/p2p_fusion_droplidar50_bs4x16/checkpoints/best_val_loss.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a55d7007e64362aa20bc5a1ef3bc3899f100db0ae628fb31c7657e4639643d53
+size 415476609

pix2poly/224/p2p_fusion_droplidar50_bs4x16/checkpoints/latest.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36560a265ff89144795aac9bd54128625610753ae1fd2f67f563e579f09d3f0e
+size 415463392

pix2poly/224/p2p_fusion_droplidar50_bs4x16/predictions_CH_val/best_val_iou.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:965e363bf33ea4f1d7ec1e6030f4947685d51b6cbacc1576a2963b9b34e378ec
+size 448205

pix2poly/224/v4_image_vit_bs4x16/.hydra/config.yaml

@@ -6,23 +6,25 @@ host:
   device: cuda
   update_pbar_every: 1
   ldof_exe: /user/rsulzer/home/cpp/line-DOF-metric/build/calculate_DoF
+  dino_v2_repo: /user/rsulzer/home/remote_python/dinov2
 run_type:
-  name: debug
+  name: release
   batch_size: 16
-  train_subset: 256
-  val_subset: 32
-  test_subset: 32
-  logging: DEBUG
-  num_workers: 0
+  train_subset: null
+  val_subset: null
+  test_subset: null
+  logging: INFO
+  num_workers: 16
   log_to_wandb: false
 dataset:
   name: PixelsPointsPolygons
   size: ${..experiment.encoder.in_size}
   path: ${host.data_root}/data/${.size}
+  annotation_type: blocks
   annotations:
-    train: ${..path}/annotations/annotations_${...experiment.country}_train.json
-    val: ${..path}/annotations/annotations_${...experiment.country}_val.json
-    test: ${..path}/annotations/annotations_${...experiment.country}_test.json
+    train: ${..path}/annotations/${..annotation_type}/annotations_${...experiment.country}_train.json
+    val: ${..path}/annotations/${..annotation_type}/annotations_${...experiment.country}_val.json
+    test: ${..path}/annotations/${..annotation_type}/annotations_${...experiment.country}_test.json
   ffl_stats:
     train: ${..path}/ffl/train/stats-${...experiment.country}.pt
     val: ${..path}/ffl/val/stats-${...experiment.country}.pt
@@ -89,7 +91,8 @@ experiment:
     weight_decay: 0.0001
   name: v4_image_vit_bs4x16
   group_name: v2_${.model.name}
-  country: CH
+  lidar_dropout: null
+  country: NY
 training:
   save_best: true
   save_latest: true
@@ -106,4 +109,3 @@ evaluation:
 output_dir: ${.host.model_root}/${.experiment.model.name}/${.experiment.encoder.in_size}/${.experiment.name}
 checkpoint: best_val_iou
 num_workers: ${.run_type.num_workers}
-image_file: demo_data/image0_CH_val.tif

pix2poly/224/v4_image_vit_bs4x16/.hydra/hydra.yaml

@@ -112,15 +112,17 @@ hydra:
     hydra:
     - hydra.mode=RUN
     task:
-    - run_type=debug
+    - run_type=release
     - host=gin
+    - run_type.log_to_wandb=false
     - checkpoint=best_val_iou
     - experiment=p2p_image
-    - +image_file=demo_data/image0_CH_val.tif
+    - experiment.country=NY
+    - evaluation=val
   job:
-    name: predict_demo
+    name: predict
     chdir: null
-    override_dirname: +image_file=demo_data/image0_CH_val.tif,checkpoint=best_val_iou,experiment=p2p_image,host=gin,run_type=debug
+    override_dirname: checkpoint=best_val_iou,evaluation=val,experiment.country=NY,experiment=p2p_image,host=gin,run_type.log_to_wandb=false,run_type=release
     id: ???
     num: ???
     config_name: config
@@ -153,7 +155,7 @@ hydra:
       model@experiment.model: pix2poly
       encoder@experiment.encoder: vit
       dataset: p3
-      run_type: debug
+      run_type: release
       host: gin
       hydra/env: default
       hydra/callbacks: null

pix2poly/224/v4_image_vit_bs4x16/.hydra/overrides.yaml

@@ -1,5 +1,7 @@
-- run_type=debug
+- run_type=release
 - host=gin
+- run_type.log_to_wandb=false
 - checkpoint=best_val_iou
 - experiment=p2p_image
-- +image_file=demo_data/image0_CH_val.tif
+- experiment.country=NY
+- evaluation=val

pix2poly/224/v4_image_vit_bs4x16/predictions_CH_test/best_val_iou.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e0683082c44df1131aa70b071c6d0d368c4f9fd077e9a487707881515fd85fa
+size 7780182

pix2poly/224/v4_image_vit_bs4x16/predictions_CH_val/best_val_iou.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1126f59e7ed29ec45143e31ec7de667abe64c89df29376d1a8323ffe0b5845a3
+size 458422