diff --git a/biotine_unpaired/dynamic/scheduler_config.json b/biotine_unpaired/dynamic/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..13ba576441b70aaf967ceb76f1383c9f4ec1c7da --- /dev/null +++ b/biotine_unpaired/dynamic/scheduler_config.json @@ -0,0 +1,19 @@ +{ + "_class_name": "DDIMScheduler", + "_diffusers_version": "0.35.2", + "beta_end": 0.02, + "beta_schedule": "linear", + "beta_start": 0.0001, + "clip_sample": false, + "clip_sample_range": 1.0, + "dynamic_thresholding_ratio": 0.995, + "num_train_timesteps": 3000, + "prediction_type": "v_prediction", + "rescale_betas_zero_snr": false, + "sample_max_value": 1.0, + "set_alpha_to_one": true, + "steps_offset": 0, + "thresholding": true, + "timestep_spacing": "leading", + "trained_betas": null +} diff --git a/chromalive/dynamic/scheduler_config.json b/chromalive/dynamic/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..13ba576441b70aaf967ceb76f1383c9f4ec1c7da --- /dev/null +++ b/chromalive/dynamic/scheduler_config.json @@ -0,0 +1,19 @@ +{ + "_class_name": "DDIMScheduler", + "_diffusers_version": "0.35.2", + "beta_end": 0.02, + "beta_schedule": "linear", + "beta_start": 0.0001, + "clip_sample": false, + "clip_sample_range": 1.0, + "dynamic_thresholding_ratio": 0.995, + "num_train_timesteps": 3000, + "prediction_type": "v_prediction", + "rescale_betas_zero_snr": false, + "sample_max_value": 1.0, + "set_alpha_to_one": true, + "steps_offset": 0, + "thresholding": true, + "timestep_spacing": "leading", + "trained_betas": null +} diff --git a/chromalive/my_conf/dataset/BBBC021/BBBC021_196_docetaxel.yaml b/chromalive/my_conf/dataset/BBBC021/BBBC021_196_docetaxel.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b76f4af62e3afa15e8e78937eb473d9cbac562a --- /dev/null +++ b/chromalive/my_conf/dataset/BBBC021/BBBC021_196_docetaxel.yaml @@ -0,0 +1,16 @@ +name: BBBC021_196_docetaxel +path: /projects/static2dynamic/datasets/BBBC021/196x196/docetaxel +data_shape: [3, 196, 196] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.ConvertImageDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} # passed dtype must be accessible as a "torch" attribute + - _target_: torchvision.transforms.Normalize + mean: [0.5, 0.5, 0.5] + std: [0.5, 0.5, 0.5] + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry +expected_initial_data_range: [0, 255] +expected_dtype: torch.uint8 diff --git a/chromalive/my_conf/dataset/BBBC021/BBBC021_196_docetaxel_fully_ordered_inference.py b/chromalive/my_conf/dataset/BBBC021/BBBC021_196_docetaxel_fully_ordered_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..29419d6196c43f0aa4700ca97214606d539dc509 --- /dev/null +++ b/chromalive/my_conf/dataset/BBBC021/BBBC021_196_docetaxel_fully_ordered_inference.py @@ -0,0 +1,10 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.BBBC021.BBBC021_196_fully_ordered_inference import dataset + +dataset = replace( + dataset, + name=dataset.name + "_docetaxel", + path="/projects/static2dynamic/datasets/BBBC021/196x196/docetaxel", + path_to_single_parquet="/projects/static2dynamic/datasets/BBBC021/196x196/BBBC021_196_docetaxel__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet", +) diff --git a/chromalive/my_conf/dataset/BBBC021/BBBC021_196_docetaxel_hard_aug_inference.py b/chromalive/my_conf/dataset/BBBC021/BBBC021_196_docetaxel_hard_aug_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..bd135b459e112c4860f979c12d5bf7064e5032ba --- /dev/null +++ b/chromalive/my_conf/dataset/BBBC021/BBBC021_196_docetaxel_hard_aug_inference.py @@ -0,0 +1,9 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.BBBC021.BBBC021_196_hard_aug_inference import dataset + +dataset = replace( + dataset, + name=dataset.name + "_docetaxel", + path="/projects/static2dynamic/datasets/BBBC021/196x196/docetaxel_hard_augmented", +) diff --git a/chromalive/my_conf/dataset/BBBC021/BBBC021_196_docetaxel_inference.py b/chromalive/my_conf/dataset/BBBC021/BBBC021_196_docetaxel_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..9be4073e5a6906688538d23ece1bb9a85ebe3baf --- /dev/null +++ b/chromalive/my_conf/dataset/BBBC021/BBBC021_196_docetaxel_inference.py @@ -0,0 +1,9 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.BBBC021.BBBC021_196_inference import dataset + +dataset = replace( + dataset, + name=dataset.name + "_docetaxel", + path="/projects/static2dynamic/datasets/BBBC021/196x196/docetaxel", +) diff --git a/chromalive/my_conf/dataset/BBBC021/BBBC021_196_nocodazole_fully_ordered_inference.py b/chromalive/my_conf/dataset/BBBC021/BBBC021_196_nocodazole_fully_ordered_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..1fa5b8cacad96fee9f711905a1cf0d206ccbfa41 --- /dev/null +++ b/chromalive/my_conf/dataset/BBBC021/BBBC021_196_nocodazole_fully_ordered_inference.py @@ -0,0 +1,27 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.BBBC021.BBBC021_196_fully_ordered_inference import dataset + +# Nocodazole classes + DMSO +CLASSES_IN_ORDER = ( + "DMSO", + "nocodazole_0.001", + "nocodazole_0.003", + "nocodazole_0.01", + "nocodazole_0.03", + "nocodazole_0.1", + "nocodazole_0.3", + "nocodazole_1.0", + "nocodazole_3.0", +) +assert dataset.dataset_params is not None +ds_params = replace(dataset.dataset_params, sorting_func=lambda subdir: CLASSES_IN_ORDER.index(subdir.name)) + +# Path and name +dataset = replace( + dataset, + name=dataset.name + "_nocodazole", + dataset_params=ds_params, + path="/projects/static2dynamic/datasets/BBBC021/196x196/nocodazole", + path_to_single_parquet="/projects/static2dynamic/datasets/BBBC021/196x196/BBBC021_196_nocodazole__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet", +) diff --git a/chromalive/my_conf/dataset/BBBC048/bbbc048.yaml b/chromalive/my_conf/dataset/BBBC048/bbbc048.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10d8775e4a9a54bd5167de52272b15c4e3fc0d46 --- /dev/null +++ b/chromalive/my_conf/dataset/BBBC048/bbbc048.yaml @@ -0,0 +1,21 @@ +name: BBBC048_fully_ordered + +path: /projects/static2dynamic/datasets/Jurkat/brightfield_reprocessed_full_circle_augmented +path_to_single_parquet: /projects/static2dynamic/Thomas/ordering_datasets/facebook_dinov2-with-registers-giant_dataset_preproc/BBBC048/BBBC048__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet + +data_shape: [ 1, 48, 48 ] + +transforms: + _target_: torchvision.transforms.v2.Compose + transforms: + - _target_: torchvision.transforms.v2.ToDtype + dtype: ${torch_dtype:float32} # passed dtype must be accessible as a "torch" attribute + scale: true + - _target_: torchvision.transforms.v2.Normalize + mean: [ 0.5 ] + std: [ 0.5 ] + +expected_initial_data_range: [ 0, 255 ] +expected_dtype: torch.uint8 + +fully_ordered: true diff --git a/chromalive/my_conf/dataset/BBBC048/bbbc048_inference.py b/chromalive/my_conf/dataset/BBBC048/bbbc048_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..502b4e3398627114cff7a979e77aad44259a0779 --- /dev/null +++ b/chromalive/my_conf/dataset/BBBC048/bbbc048_inference.py @@ -0,0 +1,7 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.BBBC048.bbbc048_inference import dataset + +dataset = replace( + dataset, path="/projects/static2dynamic/datasets/Jurkat/brightfield_reprocessed_full_circle_augmented" +) diff --git a/chromalive/my_conf/dataset/BBBC048/bbbc048_inference_fully_ordered.py b/chromalive/my_conf/dataset/BBBC048/bbbc048_inference_fully_ordered.py new file mode 100644 index 0000000000000000000000000000000000000000..ce50175d0bae05703b4e782f0b79fa33c9a33749 --- /dev/null +++ b/chromalive/my_conf/dataset/BBBC048/bbbc048_inference_fully_ordered.py @@ -0,0 +1,9 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.BBBC048.bbbc048_inference_fully_ordered import dataset + +dataset = replace( + dataset, + path="/projects/static2dynamic/datasets/Jurkat/brightfield_reprocessed_full_circle_augmented", + path_to_single_parquet="/projects/static2dynamic/Thomas/ordering_datasets/facebook_dinov2-with-registers-giant_dataset_preproc/BBBC048/BBBC048__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet", +) diff --git a/chromalive/my_conf/dataset/ChromaLive6h/ChromaLive6h_3ch_png.yaml b/chromalive/my_conf/dataset/ChromaLive6h/ChromaLive6h_3ch_png.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa3f1f1a9e0047d8ae8803fe387ea029199b08a0 --- /dev/null +++ b/chromalive/my_conf/dataset/ChromaLive6h/ChromaLive6h_3ch_png.yaml @@ -0,0 +1,19 @@ +name: chromaLive6h_3ch_png_patches_380px +path: /projects/static2dynamic/datasets/20231017ChromaLive_6hr_4ch/MIP_normalized/paired_dataset/patches +data_shape: [ 3, 128, 128 ] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.transforms.Resize + size: 128 + - _target_: torchvision.transforms.ConvertImageDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} # passed dtype must be accessible as a "torch" attribute + - _target_: torchvision.transforms.Normalize + mean: [ 0.5, 0.5, 0.5 ] # move to [-1:1] + std: [ 0.5, 0.5, 0.5 ] + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry +expected_initial_data_range: [ 0, 255 ] +expected_dtype: torch.uint8 +selected_dists: diff --git a/chromalive/my_conf/dataset/ChromaLive6h/ChromaLive6h_3ch_png_high_doses_fully_ordered.yaml b/chromalive/my_conf/dataset/ChromaLive6h/ChromaLive6h_3ch_png_high_doses_fully_ordered.yaml new file mode 100644 index 0000000000000000000000000000000000000000..095d088c10dbf722352ca184b9e348249897635d --- /dev/null +++ b/chromalive/my_conf/dataset/ChromaLive6h/ChromaLive6h_3ch_png_high_doses_fully_ordered.yaml @@ -0,0 +1,27 @@ +name: chromaLive6h_3ch_png_patches_380px_doses_10_11_combined_fully_ordered + +path: /projects/static2dynamic/datasets/20231017ChromaLive_6hr_4ch/MIP_normalized/patches_fc_many_augs +path_to_single_parquet: /projects/static2dynamic/Thomas/ordering_datasets/facebook_dinov2-with-registers-giant_dataset_preproc/heldout_test_trajs_2_doses_10_11_combined__chromaLive6h_3ch_png_patches_380px_fc_many_augs/heldout_test_trajs_2_doses_10_11_combined__chromaLive6h_3ch_png_patches_380px_fc_many_augs__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet + +data_shape: [ 3, 128, 128 ] + +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.transforms.Resize + size: 128 + - _target_: torchvision.transforms.ConvertImageDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} # passed dtype must be accessible as a "torch" attribute + - _target_: torchvision.transforms.Normalize + mean: [ 0.5, 0.5, 0.5 ] # move to [-1:1] + std: [ 0.5, 0.5, 0.5 ] + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry + +expected_initial_data_range: [ 0, 255 ] +expected_dtype: torch.uint8 + +selected_dists: + +fully_ordered: true diff --git a/chromalive/my_conf/dataset/ChromaLive6h/ChromaLive6h_4ch_tif.yaml b/chromalive/my_conf/dataset/ChromaLive6h/ChromaLive6h_4ch_tif.yaml new file mode 100644 index 0000000000000000000000000000000000000000..694a2ba580d61c98606c8a5e60213ede4d0b5b5a --- /dev/null +++ b/chromalive/my_conf/dataset/ChromaLive6h/ChromaLive6h_4ch_tif.yaml @@ -0,0 +1,19 @@ +name: chromaLive6h_4ch_tif_patches_380px +path: /projects/static2dynamic/datasets/20231017ChromaLive_6hr_4ch/MIP_normalized/paired_dataset/patches +data_shape: [ 4, 128, 128 ] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.transforms.Resize + size: 128 + - _target_: torchvision.transforms.ConvertImageDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} # passed dtype must be accessible as a "torch" attribute + - _target_: torchvision.transforms.Normalize + mean: [ 0.5, 0.5, 0.5, 0.5 ] # move to [-1:1] + std: [ 0.5, 0.5, 0.5, 0.5 ] + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry +expected_initial_data_range: [ 0, 65536 ] +expected_dtype: torch.uint16 +selected_dists: [ 'time_1', 'time_3', 'time_5', 'time_7', 'time_9', 'time_11', 'time_13' ] diff --git a/chromalive/my_conf/dataset/ChromaLive6h/chromalive6h_3ch_png_fc_augs_inference.py b/chromalive/my_conf/dataset/ChromaLive6h/chromalive6h_3ch_png_fc_augs_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..7385ed328f08df9537b56822e47acdfe5b59c652 --- /dev/null +++ b/chromalive/my_conf/dataset/ChromaLive6h/chromalive6h_3ch_png_fc_augs_inference.py @@ -0,0 +1,9 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.ChromaLive6h.chromalive6h_3ch_png_inference import dataset + +dataset = replace( + dataset, + name="chromaLive6h_3ch_png_patches_380px_fc_augs", + path="/projects/static2dynamic/datasets/20231017ChromaLive_6hr_4ch/MIP_normalized/patches_fc_augs", +) diff --git a/chromalive/my_conf/dataset/ChromaLive6h/chromalive6h_3ch_png_fc_many_augs_inference.py b/chromalive/my_conf/dataset/ChromaLive6h/chromalive6h_3ch_png_fc_many_augs_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..9df2b90a3d0d14c629f6cf74f3fc7897d7bad769 --- /dev/null +++ b/chromalive/my_conf/dataset/ChromaLive6h/chromalive6h_3ch_png_fc_many_augs_inference.py @@ -0,0 +1,9 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.ChromaLive6h.chromalive6h_3ch_png_inference import dataset + +dataset = replace( + dataset, + name="chromaLive6h_3ch_png_patches_380px_fc_many_augs", + path="/projects/static2dynamic/datasets/20231017ChromaLive_6hr_4ch/MIP_normalized/patches_fc_many_augs", +) diff --git a/chromalive/my_conf/dataset/ChromaLive6h/chromalive6h_3ch_png_fully_ordered_inference.py b/chromalive/my_conf/dataset/ChromaLive6h/chromalive6h_3ch_png_fully_ordered_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..f8e0a27a92df21579a44138a19ee24915d466d69 --- /dev/null +++ b/chromalive/my_conf/dataset/ChromaLive6h/chromalive6h_3ch_png_fully_ordered_inference.py @@ -0,0 +1,9 @@ +from dataclass import replace + +from GaussianProxy.conf.dataset.ChromaLive6h.chromalive6h_3ch_png_fully_ordered_inference import dataset + +dataset = replace( + dataset, + path="/projects/static2dynamic/datasets/20231017ChromaLive_6hr_4ch/MIP_normalized/paired_dataset/patches", + path_to_single_parquet="/projects/static2dynamic/datasets/20231017ChromaLive_6hr_4ch/MIP_normalized/paired_dataset/chromaLive6h_3ch_png_patches_380px__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet", +) diff --git a/chromalive/my_conf/dataset/ChromaLive6h/chromalive6h_3ch_png_hard_aug_inference.py b/chromalive/my_conf/dataset/ChromaLive6h/chromalive6h_3ch_png_hard_aug_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..c3c2703586c9673d9b7aa219bd55d103612cd61d --- /dev/null +++ b/chromalive/my_conf/dataset/ChromaLive6h/chromalive6h_3ch_png_hard_aug_inference.py @@ -0,0 +1,9 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.ChromaLive6h.chromalive6h_3ch_png_inference import dataset + +dataset = replace( + dataset, + name="chromaLive6h_3ch_png_patches_380px_hard_aug", + path="/projects/static2dynamic/datasets/20231017ChromaLive_6hr_4ch/MIP_normalized/paired_dataset/patches_hard_augmented", +) diff --git a/chromalive/my_conf/dataset/ChromaLive6h/chromalive6h_3ch_png_inference.py b/chromalive/my_conf/dataset/ChromaLive6h/chromalive6h_3ch_png_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..a6365c4c355acf8d3920e660bc327b87dd553b27 --- /dev/null +++ b/chromalive/my_conf/dataset/ChromaLive6h/chromalive6h_3ch_png_inference.py @@ -0,0 +1,8 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.ChromaLive6h.chromalive6h_3ch_png_inference import dataset + +dataset = replace( + dataset, + path="/projects/static2dynamic/datasets/20231017ChromaLive_6hr_4ch/MIP_normalized/paired_dataset/patches", +) diff --git a/chromalive/my_conf/dataset/DeepCycle/deepcycle_brightfield_to_3D_inference.py b/chromalive/my_conf/dataset/DeepCycle/deepcycle_brightfield_to_3D_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..5d5c7af253559ce6fbec7561ff59e4a9f9c7e4e6 --- /dev/null +++ b/chromalive/my_conf/dataset/DeepCycle/deepcycle_brightfield_to_3D_inference.py @@ -0,0 +1,3 @@ +from GaussianProxy.conf.dataset.DeepCycle.deepcycle_brightfield_to_3D_inference import dataset + +dataset.path = "/projects/imagesets2/DeepCycle/cells/128x128" diff --git a/chromalive/my_conf/dataset/DeepCycle/deepcycle_hoechst_brightfield_to_3D_inference.py b/chromalive/my_conf/dataset/DeepCycle/deepcycle_hoechst_brightfield_to_3D_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..4ffcf9696a44cd5652089f937d88e2b630002f32 --- /dev/null +++ b/chromalive/my_conf/dataset/DeepCycle/deepcycle_hoechst_brightfield_to_3D_inference.py @@ -0,0 +1,3 @@ +from GaussianProxy.conf.dataset.DeepCycle.deepcycle_hoechst_brightfield_to_3D_inference import dataset + +dataset.path = "/projects/imagesets2/DeepCycle/cells/128x128" diff --git a/chromalive/my_conf/dataset/DeepCycle/deepcycle_inference.py b/chromalive/my_conf/dataset/DeepCycle/deepcycle_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..f4f7cd69d76b72676b9513a5e44dc91ccf130028 --- /dev/null +++ b/chromalive/my_conf/dataset/DeepCycle/deepcycle_inference.py @@ -0,0 +1,3 @@ +from GaussianProxy.conf.dataset.DeepCycle.deepcycle_inference import dataset + +dataset.path = "/projects/imagesets2/DeepCycle/cells/128x128" diff --git a/chromalive/my_conf/dataset/DeepCycle/deepcycle_markers_inference.py b/chromalive/my_conf/dataset/DeepCycle/deepcycle_markers_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..129730fd4014e545f15100c50b4fd60481782e17 --- /dev/null +++ b/chromalive/my_conf/dataset/DeepCycle/deepcycle_markers_inference.py @@ -0,0 +1,3 @@ +from GaussianProxy.conf.dataset.DeepCycle.deepcycle_markers_inference import dataset + +dataset.path = "/projects/imagesets2/DeepCycle/cells/128x128" diff --git a/chromalive/my_conf/dataset/Jurkat/Jurkat.yaml b/chromalive/my_conf/dataset/Jurkat/Jurkat.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6dae2eb460d55c87abf178bad5c7a2f4d43587cb --- /dev/null +++ b/chromalive/my_conf/dataset/Jurkat/Jurkat.yaml @@ -0,0 +1,16 @@ +name: Jurkat +path: /projects/static2dynamic/datasets/Jurkat/rgb_images_all_cell_cycles +data_shape: [3, 66, 66] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.ConvertImageDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} # passed dtype must be accessible as a "torch" attribute + - _target_: torchvision.transforms.Normalize + mean: [0.5, 0.5, 0.5] + std: [0.5, 0.5, 0.5] + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry +expected_initial_data_range: [0, 255] +expected_dtype: torch.uint8 diff --git a/chromalive/my_conf/dataset/Jurkat/Jurkat_brightfield_inference.py b/chromalive/my_conf/dataset/Jurkat/Jurkat_brightfield_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..9d9cf49c06507bbebc2f57319f952ce9d5a3073b --- /dev/null +++ b/chromalive/my_conf/dataset/Jurkat/Jurkat_brightfield_inference.py @@ -0,0 +1,3 @@ +from GaussianProxy.conf.dataset.Jurkat.Jurkat_brightfield_inference import dataset + +dataset.path = "/projects/static2dynamic/datasets/Jurkat/brightfield_reprocessed" diff --git a/chromalive/my_conf/dataset/Jurkat/Jurkat_fully_ordered.yaml b/chromalive/my_conf/dataset/Jurkat/Jurkat_fully_ordered.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24e6e21de17a84ad77fba0ddd54fa0f770ea3cdf --- /dev/null +++ b/chromalive/my_conf/dataset/Jurkat/Jurkat_fully_ordered.yaml @@ -0,0 +1,18 @@ +name: Jurkat_fully_ordered_dinov2_regs_giant_ds_preproc +path: /projects/static2dynamic/datasets/Jurkat/rgb_images_all_cell_cycles +path_to_single_parquet: /projects/static2dynamic/datasets/Jurkat/Jurkat__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet +data_shape: [ 3, 66, 66 ] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.ConvertImageDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} # passed dtype must be accessible as a "torch" attribute + - _target_: torchvision.transforms.Normalize + mean: [ 0.5, 0.5, 0.5 ] + std: [ 0.5, 0.5, 0.5 ] + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry +expected_initial_data_range: [ 0, 255 ] +expected_dtype: torch.uint8 +fully_ordered: true diff --git a/chromalive/my_conf/dataset/Jurkat/Jurkat_inference.py b/chromalive/my_conf/dataset/Jurkat/Jurkat_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..8f9c110d8ff54daa5c3671481b9580b407d01724 --- /dev/null +++ b/chromalive/my_conf/dataset/Jurkat/Jurkat_inference.py @@ -0,0 +1,8 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.Jurkat.Jurkat_inference import dataset + +dataset = replace( + dataset, + path="/projects/static2dynamic/datasets/Jurkat/rgb_images_all_cell_cycles", +) diff --git a/chromalive/my_conf/dataset/NASH_fibrosis/NASH_fibrosis.yaml b/chromalive/my_conf/dataset/NASH_fibrosis/NASH_fibrosis.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a53dc5ec231d267cb6fcdc50c7166e5a792c0d32 --- /dev/null +++ b/chromalive/my_conf/dataset/NASH_fibrosis/NASH_fibrosis.yaml @@ -0,0 +1,24 @@ +name: NASH_fibrosis +path: /projects/static2dynamic/datasets/NASH/prepared_data/fibrosis +data_shape: [ 3, 128, 128 ] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + # Convert to float32 (and normalize to [0, 1]) before resizing + - _target_: torchvision.transforms.ConvertImageDtype + dtype: ${torch_dtype:float32} + # Random crop from 299x299 to 192x192, then resize to 128x128 + - _target_: torchvision.transforms.RandomCrop + size: 192 + - _target_: torchvision.transforms.Resize + size: 128 + # Normalize to [-1, 1] + - _target_: torchvision.transforms.Normalize + mean: [ 0.5, 0.5, 0.5 ] + std: [ 0.5, 0.5, 0.5 ] + # Random 8x square augmentations + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry +selected_dists: +expected_initial_data_range: [ 0, 255 ] diff --git a/chromalive/my_conf/dataset/NASH_fibrosis/NASH_fibrosis_fully_ordered_inference.py b/chromalive/my_conf/dataset/NASH_fibrosis/NASH_fibrosis_fully_ordered_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..58a1b0331176b90a44a2209db56dba19005ef998 --- /dev/null +++ b/chromalive/my_conf/dataset/NASH_fibrosis/NASH_fibrosis_fully_ordered_inference.py @@ -0,0 +1,4 @@ +from GaussianProxy.conf.dataset.NASH_fibrosis.NASH_fibrosis_fully_ordered_inference import dataset + +dataset.path = "/projects/static2dynamic/datasets/NASH/prepared_data/fibrosis" +dataset.path_to_single_parquet = "/projects/static2dynamic/datasets/NASH/prepared_data/NASH_fibrosis__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet" diff --git a/chromalive/my_conf/dataset/NASH_steatosis/NASH_steatosis_fully_ordered.yaml b/chromalive/my_conf/dataset/NASH_steatosis/NASH_steatosis_fully_ordered.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ee2b473282b4c7882ddced978e0b4165481f237 --- /dev/null +++ b/chromalive/my_conf/dataset/NASH_steatosis/NASH_steatosis_fully_ordered.yaml @@ -0,0 +1,26 @@ +name: NASH_steatosis_fully_ordered_dinov2_regs_giant_ds_preproc +path: /projects/static2dynamic/datasets/NASH/prepared_data/steatosis +path_to_single_parquet: /projects/static2dynamic/datasets/NASH/prepared_data/NASH_steatosis__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet +data_shape: [ 3, 128, 128 ] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + # Convert to float32 (and normalize to [0, 1]) before resizing + - _target_: torchvision.transforms.ConvertImageDtype + dtype: ${torch_dtype:float32} + # Random crop from 299x299 to 192x192, then resize to 128x128 + - _target_: torchvision.transforms.RandomCrop + size: 192 + - _target_: torchvision.transforms.Resize + size: 128 + # Normalize to [-1, 1] + - _target_: torchvision.transforms.Normalize + mean: [ 0.5, 0.5, 0.5 ] + std: [ 0.5, 0.5, 0.5 ] + # Random 8x square augmentations + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry +selected_dists: +expected_initial_data_range: [ 0, 255 ] +fully_ordered: true diff --git a/chromalive/my_conf/dataset/biotine/biotine_png_128.yaml b/chromalive/my_conf/dataset/biotine/biotine_png_128.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49ffb167b04d435655c47b359aa8df480319f587 --- /dev/null +++ b/chromalive/my_conf/dataset/biotine/biotine_png_128.yaml @@ -0,0 +1,19 @@ +name: biotine_png +path: /projects/static2dynamic/datasets/biotine/3_channels_min_99_perc_normalized_rgb_stacks_png/patches_255 +data_shape: [ 3, 128, 128 ] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.transforms.Resize + size: 128 + # ConvertImageDtype also scales to [0; 1] (from the *implicit* expected range that depends on the incoming dtype...) + - _target_: torchvision.transforms.ConvertImageDtype + dtype: ${torch_dtype:float32} + - _target_: torchvision.transforms.Normalize + mean: [ 0.5, 0.5, 0.5 ] + std: [ 0.5, 0.5, 0.5 ] + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry +selected_dists: +expected_initial_data_range: [ 0, 255 ] diff --git a/chromalive/my_conf/dataset/biotine/biotine_png_128_fully_ordered.yaml b/chromalive/my_conf/dataset/biotine/biotine_png_128_fully_ordered.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3cceb83582d9dff9cd838ae1c5f4ac34f99b2af2 --- /dev/null +++ b/chromalive/my_conf/dataset/biotine/biotine_png_128_fully_ordered.yaml @@ -0,0 +1,21 @@ +name: biotine_png_fully_ordered +path: /projects/static2dynamic/datasets/biotine/3_channels_min_99_perc_normalized_rgb_stacks_png/patches_255 +path_to_single_parquet: /projects/static2dynamic/datasets/biotine/3_channels_min_99_perc_normalized_rgb_stacks_png/biotine_png__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet +data_shape: [ 3, 128, 128 ] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.transforms.Resize + size: 128 + # ConvertImageDtype also scales to [0; 1] (from the *implicit* expected range that depends on the incoming dtype...) + - _target_: torchvision.transforms.ConvertImageDtype + dtype: ${torch_dtype:float32} + - _target_: torchvision.transforms.Normalize + mean: [ 0.5, 0.5, 0.5 ] + std: [ 0.5, 0.5, 0.5 ] + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry +selected_dists: +expected_initial_data_range: [ 0, 255 ] +fully_ordered: true diff --git a/chromalive/my_conf/dataset/biotine/biotine_png_128_fully_ordered_inference.py b/chromalive/my_conf/dataset/biotine/biotine_png_128_fully_ordered_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..c0ba77af280d7f14f8cf140a19b146664524c38b --- /dev/null +++ b/chromalive/my_conf/dataset/biotine/biotine_png_128_fully_ordered_inference.py @@ -0,0 +1,18 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.biotine.biotine_png_128_inference import dataset +from GaussianProxy.utils.data import ContinuousTimeImageDataset + +assert dataset.dataset_params is not None +updated_ds_params = replace( + dataset.dataset_params, + dataset_class=ContinuousTimeImageDataset, +) + +dataset = replace( + dataset, + fully_ordered=True, + path="/projects/static2dynamic/datasets/biotine/3_channels_min_99_perc_normalized_rgb_stacks_png/patches_255", + path_to_single_parquet="/projects/static2dynamic/datasets/biotine/3_channels_min_99_perc_normalized_rgb_stacks_png/biotine_png__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet", + dataset_params=updated_ds_params, +) diff --git a/chromalive/my_conf/dataset/biotine/biotine_png_128_hard_aug.yaml b/chromalive/my_conf/dataset/biotine/biotine_png_128_hard_aug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07044d90813fe3acadb1a0d996adc1277f60ccbe --- /dev/null +++ b/chromalive/my_conf/dataset/biotine/biotine_png_128_hard_aug.yaml @@ -0,0 +1,16 @@ +name: biotine_png_hard_aug +path: /projects/static2dynamic/datasets/biotine/3_channels_min_99_perc_normalized_rgb_stacks_png/patches_255_hard_augmented +data_shape: [ 3, 128, 128 ] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.transforms.Resize + size: 128 + # ConvertImageDtype also scales to [0; 1] (from the *implicit* expected range that depends on the incoming dtype...) + - _target_: torchvision.transforms.ConvertImageDtype + dtype: ${torch_dtype:float32} + - _target_: torchvision.transforms.Normalize + mean: [ 0.5, 0.5, 0.5 ] + std: [ 0.5, 0.5, 0.5 ] +selected_dists: +expected_initial_data_range: [ 0, 255 ] diff --git a/chromalive/my_conf/dataset/biotine/biotine_png_128_hard_aug_inference.py b/chromalive/my_conf/dataset/biotine/biotine_png_128_hard_aug_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..5fda38ec48345e97464fead2ea49563c1c484cc9 --- /dev/null +++ b/chromalive/my_conf/dataset/biotine/biotine_png_128_hard_aug_inference.py @@ -0,0 +1,8 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.biotine.biotine_png_128_hard_aug_inference import dataset + +dataset = replace( + dataset, + path="/projects/static2dynamic/datasets/biotine/3_channels_min_99_perc_normalized_rgb_stacks_png/patches_255_hard_augmented", +) diff --git a/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy.yaml b/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b458e5fc57edb75c902a3ddb5519f9064950f555 --- /dev/null +++ b/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy.yaml @@ -0,0 +1,18 @@ +name: diabetic_retinopathy +path: /projects/static2dynamic/datasets/DiabeticRetinopathy/prepared_dataset/train +data_shape: [3, 256, 256] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.transforms.Resize + size: 256 # single int => image resized to (size * aspect_ratio, size) or (size, size * aspect_ratio) with aspect_ratio >= 1 preserved + - _target_: torchvision.transforms.v2.CenterCrop + size: 256 # square centered crop + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.ConvertImageDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} # passed dtype must be accessible as a "torch" attribute + - _target_: torchvision.transforms.Normalize + mean: [0.5, 0.5, 0.5] + std: [0.5, 0.5, 0.5] +expected_initial_data_range: [0, 255] +expected_dtype: torch.uint8 diff --git a/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_crop_fully_ordered.yaml b/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_crop_fully_ordered.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a485ac70368ed67577f3e43766139422a2315b58 --- /dev/null +++ b/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_crop_fully_ordered.yaml @@ -0,0 +1,25 @@ +name: diabetic_retinopathy_2048_crop_fully_ordered + +path: /projects/static2dynamic/datasets/DiabeticRetinopathy/prepared_dataset_full_circle_augmented +path_to_single_parquet: /projects/static2dynamic/Thomas/ordering_datasets/facebook_dinov2-with-registers-giant_dataset_preproc/balanced_classes__diabetic_retinopathy_2048_crop/balanced_classes__diabetic_retinopathy_2048_crop__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet + +data_shape: [3, 256, 256] + +transforms: + _target_: torchvision.transforms.v2.Compose + transforms: + - _target_: torchvision.transforms.v2.CenterCrop + size: 2048 + - _target_: torchvision.transforms.v2.Resize + size: 256 + - _target_: torchvision.transforms.v2.ToDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} + scale: true + - _target_: torchvision.transforms.v2.Normalize + mean: [0.5, 0.5, 0.5] + std: [0.5, 0.5, 0.5] + +expected_initial_data_range: [0, 255] +expected_dtype: torch.uint8 + +fully_ordered: true diff --git a/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_precrop_fully_ordered.yaml b/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_precrop_fully_ordered.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4ecf72521e33c733056c5d0927ed58711dad04b --- /dev/null +++ b/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_precrop_fully_ordered.yaml @@ -0,0 +1,21 @@ +name: diabetic_retinopathy_2048_precrop_fully_ordered + +path: /projects/static2dynamic/datasets/DiabeticRetinopathy/prepared_dataset_full_circle_augmented_2048_crop +path_to_single_parquet: /projects/static2dynamic/Thomas/ordering_datasets/facebook_dinov2-with-registers-giant_dataset_preproc/diabetic_retinopathy_full_circle_augs_2048_precrop/diabetic_retinopathy_full_circle_augs_2048_precrop__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet + +data_shape: [ 3, 256, 256 ] + +transforms: + _target_: torchvision.transforms.v2.Compose + transforms: + - _target_: torchvision.transforms.v2.ToDtype + dtype: ${torch_dtype:float32} + scale: true + - _target_: torchvision.transforms.v2.Normalize + mean: [ 0.5, 0.5, 0.5 ] + std: [ 0.5, 0.5, 0.5 ] + +expected_initial_data_range: [ 0, 255 ] +expected_dtype: torch.uint8 + +fully_ordered: true diff --git a/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_precrop_fully_ordered_inference.py b/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_precrop_fully_ordered_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..4e3f3c6ec6fc7e00fdcb66dc52987ec5eb87b459 --- /dev/null +++ b/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_precrop_fully_ordered_inference.py @@ -0,0 +1,18 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.diabetic_retinopathy.diabetic_retinopathy_full_circle_augs_2048_precrop_inference import ( + dataset, +) +from GaussianProxy.utils.data import ContinuousTimeImageDataset + +assert dataset.dataset_params is not None +ds_params = replace(dataset.dataset_params, dataset_class=ContinuousTimeImageDataset) + + +dataset = replace( + dataset, + path="/projects/static2dynamic/datasets/DiabeticRetinopathy/prepared_dataset_full_circle_augmented_2048_crop", + path_to_single_parquet="/projects/static2dynamic/Thomas/ordering_datasets/facebook_dinov2-with-registers-giant_dataset_preproc/diabetic_retinopathy_full_circle_augs_2048_precrop/diabetic_retinopathy_full_circle_augs_2048_precrop__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet", + fully_ordered=True, + dataset_params=ds_params, +) diff --git a/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_fully_ordered_inference.py b/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_fully_ordered_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..c03ee29545d35168424e0d9541358fd77f66791f --- /dev/null +++ b/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_fully_ordered_inference.py @@ -0,0 +1,15 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.diabetic_retinopathy.diabetic_retinopathy_inference import dataset +from GaussianProxy.utils.data import ContinuousTimeImageDataset + +assert dataset.dataset_params is not None +updated_ds_params = replace(dataset.dataset_params, dataset_class=ContinuousTimeImageDataset) + +dataset = replace( + dataset, + fully_ordered=True, + path="/projects/static2dynamic/datasets/DiabeticRetinopathy/prepared_dataset/train", + path_to_single_parquet="/projects/static2dynamic/datasets/DiabeticRetinopathy/prepared_dataset/diabetic_retinopathy__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet", + dataset_params=updated_ds_params, +) diff --git a/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_inference.py b/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..be31075247d1bb02e35ebcd777f233e0ecba9a55 --- /dev/null +++ b/chromalive/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_inference.py @@ -0,0 +1,5 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.diabetic_retinopathy.diabetic_retinopathy_inference import dataset + +dataset = replace(dataset, path="/projects/static2dynamic/datasets/DiabeticRetinopathy/prepared_dataset/train") diff --git a/chromalive/my_conf/dataset/ependymal_context/ependymal_context_inference.py b/chromalive/my_conf/dataset/ependymal_context/ependymal_context_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..f9514fb17a5910cb32bfd7692534a4fefe2ee78d --- /dev/null +++ b/chromalive/my_conf/dataset/ependymal_context/ependymal_context_inference.py @@ -0,0 +1,32 @@ +from torch import float32 +from torchvision.transforms import Compose, ConvertImageDtype, Normalize + +from GaussianProxy.conf.training_conf import DataSet, DatasetParams +from GaussianProxy.utils.data import ImageDataset + +DEFINITION = 256 +NUMBER_OF_CHANNELS = 3 + +transforms = Compose( + transforms=[ + ConvertImageDtype(float32), + Normalize(mean=[0.5] * NUMBER_OF_CHANNELS, std=[0.5] * NUMBER_OF_CHANNELS), + ] +) + +ds_params = DatasetParams( + file_extension="png", + key_transform=str, + sorting_func=lambda subdir: int(subdir.name), + dataset_class=ImageDataset, +) + +dataset = DataSet( + name="ependymal_context", + data_shape=(NUMBER_OF_CHANNELS, DEFINITION, DEFINITION), + transforms=transforms, + expected_initial_data_range=(0, 255), + dataset_params=ds_params, + path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_context", + selected_dists=["0", "3", "5", "7", "9", "16", "30"], # ignore .REMOVED_IMAGES/16 (imaging artifacts) +) diff --git a/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_fc_augs_sep_gt_inference.py b/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_fc_augs_sep_gt_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..b79360ec8e27af2d354347b1a11416d801dba83e --- /dev/null +++ b/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_fc_augs_sep_gt_inference.py @@ -0,0 +1,19 @@ +from dataclasses import replace +from pathlib import Path + +from GaussianProxy.conf.dataset.ependymal_cutout.ependymal_cutout_inference import dataset + +# note the data organization is special with separate ground truth folder +assert dataset.dataset_params is not None +dataset_params = replace( + dataset.dataset_params, + sorting_func=lambda subdir: str(subdir.name) if isinstance(subdir, Path) else str(subdir), +) # subdirs are "ground_truth" and "all_imgs" now... + +dataset = replace( + dataset, + name="ependymal_cutout_01_noised_full_circle_augs", + path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_noised_0.1_crop_full_circle_augmented", + separate_gt_starting_class_path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_noised_0.1_crop_full_circle_augmented/ground_truths/1", + dataset_params=dataset_params, +) diff --git a/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_separate_gt_fully_ordered_inference.py b/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_separate_gt_fully_ordered_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..a3c385439ee36d2153e9909e3e3131c23a82ef03 --- /dev/null +++ b/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_separate_gt_fully_ordered_inference.py @@ -0,0 +1,18 @@ +from dataclasses import replace + +from GaussianProxy.utils.data import ContinuousTimeImageDataset +from my_conf.dataset.ependymal_cutout.ependymal_cutout_01_noised_separate_gt_inference import dataset + +assert dataset.dataset_params is not None +dataset_params = replace( + dataset.dataset_params, + dataset_class=ContinuousTimeImageDataset, +) + +dataset = replace( + dataset, + fully_ordered=True, + path_to_single_parquet="/projects/static2dynamic/datasets/ependymal/ependymal_cutout_01_noised__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet", + dataset_params=dataset_params, + selected_dists=["all_imgs"], +) diff --git a/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_03_noised_separate_gt_inference.py b/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_03_noised_separate_gt_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..4a1c4aedf4feae0aef44851b048c35192dbc051f --- /dev/null +++ b/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_03_noised_separate_gt_inference.py @@ -0,0 +1,17 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.ependymal_cutout.ependymal_cutout_inference import dataset + +# note the data organization is special with separate ground truth folder +assert dataset.dataset_params is not None +dataset_params = replace( + dataset.dataset_params, sorting_func=lambda subdir: str(subdir.name) +) # subdirs are "ground_truth" and "all_imgs" now... + +dataset = replace( + dataset, + name="ependymal_cutout_03_noised", + path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_noised_0.3_crop/", + dataset_params=dataset_params, + separate_gt_starting_class_path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_noised_0.3_crop/ground_truths/1", +) diff --git a/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_fully_ordered_inference.py b/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_fully_ordered_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..8aec4fa3fd790e59f62f2c67728c9eca0bfcfff6 --- /dev/null +++ b/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_fully_ordered_inference.py @@ -0,0 +1,9 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.ependymal_cutout.ependymal_cutout_fully_ordered_inference import dataset + +dataset = replace( + dataset, + path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_crop", + path_to_single_parquet="/projects/static2dynamic/datasets/ependymal/ependymal_cutout__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet", +) diff --git a/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_inference.py b/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..3458d8b0f72c8fc1042d2fcc4feb2b64bb96ece2 --- /dev/null +++ b/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_inference.py @@ -0,0 +1,8 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.ependymal_cutout.ependymal_cutout_inference import dataset + +dataset = replace( + dataset, + path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_crop", +) diff --git a/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_white_bg_separate_gt_fully_ordered_inference.py b/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_white_bg_separate_gt_fully_ordered_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..1fd379f5de2accb4ff47bc112191f2e7232ea35d --- /dev/null +++ b/chromalive/my_conf/dataset/ependymal_cutout/ependymal_cutout_white_bg_separate_gt_fully_ordered_inference.py @@ -0,0 +1,19 @@ +from dataclasses import replace + +from GaussianProxy.utils.data import ContinuousTimeImageDataset +from my_conf.dataset.ependymal_cutout.ependymal_cutout_white_bg_separate_gt_inference import dataset + +assert dataset.dataset_params is not None +dataset_params = replace( + dataset.dataset_params, + dataset_class=ContinuousTimeImageDataset, +) + +dataset = replace( + dataset, + fully_ordered=True, + path_to_single_parquet="/projects/static2dynamic/datasets/ependymal/ependymal_cutout_white_bg__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet", + dataset_params=dataset_params, + selected_dists=["all_imgs"], + separate_gt_starting_class_path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_white_bg_crop/ground_truths/1", +) diff --git a/chromalive/my_conf/dataset/human_embryo/human_embryo_inference.py b/chromalive/my_conf/dataset/human_embryo/human_embryo_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..8358a86aab489a3ff63dabce6f0095df11dcac98 --- /dev/null +++ b/chromalive/my_conf/dataset/human_embryo/human_embryo_inference.py @@ -0,0 +1,5 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.human_embryo.human_embryo_inference import dataset + +dataset = replace(dataset, path="/projects/imagesets3/2022_Gomez/reformated_phases/phases") diff --git a/chromalive/my_conf/dataset/imagenet_n01917289_hard_aug_inference.py b/chromalive/my_conf/dataset/imagenet_n01917289_hard_aug_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..5092fe7e19d910949c4816bd99d0a8b5a9531d81 --- /dev/null +++ b/chromalive/my_conf/dataset/imagenet_n01917289_hard_aug_inference.py @@ -0,0 +1,36 @@ +from torch import float32 +from torchvision.transforms import Compose, ConvertImageDtype, Normalize, Resize +from torchvision.transforms.v2 import CenterCrop + +from GaussianProxy.conf.training_conf import DataSet, DatasetParams +from GaussianProxy.utils.data import ImageDataset + +DEFINITION = 128 +NUMBER_OF_CHANNELS = 3 + +transforms = Compose( + transforms=[ + # single int => image resized to (size * aspect_ratio, size) or (size, size * aspect_ratio) with aspect_ratio >= 1 preserved + Resize(DEFINITION), + CenterCrop(DEFINITION), # pyright: ignore[reportAttributeAccessIssue] + ConvertImageDtype(float32), + Normalize(mean=[0.5] * NUMBER_OF_CHANNELS, std=[0.5] * NUMBER_OF_CHANNELS), + ] +) + +ds_params = DatasetParams( + file_extension="JPEG", + key_transform=int, + sorting_func=lambda subdir: int(subdir.name), + dataset_class=ImageDataset, +) + +dataset = DataSet( + path="/localtmp/tboyer/augmented_imagenet_n01917289", + name="imagenet_n01917289_hard_aug_inference", + data_shape=(NUMBER_OF_CHANNELS, DEFINITION, DEFINITION), + transforms=transforms, + selected_dists=None, # not used + expected_initial_data_range=(0, 255), + dataset_params=ds_params, +) diff --git a/chromalive/my_conf/hydra/job_logging/custom.yaml b/chromalive/my_conf/hydra/job_logging/custom.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0351b01adac5db3c464d4990751266af706f6c6 --- /dev/null +++ b/chromalive/my_conf/hydra/job_logging/custom.yaml @@ -0,0 +1,35 @@ +version: 1 +formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + colorlog: + (): colorlog.ColoredFormatter + format: '[%(cyan)s%(asctime)s%(reset)s][%(blue)s%(name)s%(reset)s][%(log_color)s%(levelname)s%(reset)s] - %(message)s' + log_colors: + DEBUG: purple + INFO: green + WARNING: yellow + ERROR: red + CRITICAL: red +handlers: + console: + class: logging.StreamHandler + formatter: colorlog + stream: ext://sys.stdout + level: INFO + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra:run.dir}/logs.log # unify logs from launcher and script + level: DEBUG +root: + level: DEBUG + handlers: + - console + - file +logger: + matplotlib: + level: INFO + PIL: + level: INFO +disable_existing_loggers: true diff --git a/chromalive/my_conf/my_dataset_ordering_conf.py b/chromalive/my_conf/my_dataset_ordering_conf.py new file mode 100644 index 0000000000000000000000000000000000000000..ae7f219872e5bbe5e190f007090267c070413297 --- /dev/null +++ b/chromalive/my_conf/my_dataset_ordering_conf.py @@ -0,0 +1,65 @@ +import random +from pathlib import Path + +from scripts.dataset_ordering import Params + +################################################################################################################### +#################################################### Datasets ##################################################### +################################################################################################################### +# isort: off +from my_conf.dataset.ChromaLive6h.chromalive6h_3ch_png_fc_augs_inference import dataset + + +DOSES = ("10", "11") # we will combine doses 10 and 11 +all_trajectories_per_dose = { + dose: [f"{well}-{dose}_F{field}" for well in ("C", "D", "E") for field in ("0001", "0002", "0003", "0004")] + for dose in DOSES +} + +# fmt: off +times_to_remove = ( + (), # oo1 (no time removed) + (2, 4, 6, 8, 10, 12), # oo2 + (2, 3, 5, 6, 8, 9, 11, 12), # oo3 + (2, 3, 4, 6, 7, 8, 10, 11, 12), # oo4 + (2, 3, 4, 5, 6, 8, 9, 10, 11, 12), # oo5 +) +# fmt: on +# times removal scheme: +# 01 02 03 04 05 06 07 08 09 10 11 12 13 +# x x x x x x # oo2 +# x x x x x x x x # oo3 +# x x x x x x x x x # oo4 +# x x x x x x x x x x # oo5 + +held_out_trajs = r"|".join([r"|".join(random.sample(all_trajectories_per_dose[dose], 2)) for dose in DOSES]) +test_regex_list = [ + held_out_trajs + "".join(f"|time_{t:02d}" for t in times_to_remove[i]) for i in range(len(times_to_remove)) +] +filter_ds_regex_list = ["-10_F|-11_F" for _ in range(len(times_to_remove))] # doses 10 and 11 combined + +base_save_dir = Path("/projects/static2dynamic/Thomas/ordering_datasets") +# fmt: off +params = Params( + base_save_dir = base_save_dir, + experiment_names = [f"heldout_test_trajs_2_doses_10_11_combined_times_ooo{i+1}" for i in range(len(times_to_remove))], + datasets = [dataset for _ in range(len(times_to_remove))], + device = "cuda:0", + model_name = "facebook/dinov2-with-registers-giant", + batch_size = 2048 + 256, + use_model_preprocessor = False, + recompute_encodings = "no", + save_policy = "ask-before-overwrite", + seed = random.randint(0, 2**32 - 1), + spline_continuation_range = (0.1, 0.2), + nb_times_spline_eval = 50_000, + test_split_frac = None, + test_regexes = test_regex_list, + filter_ds_regex = filter_ds_regex_list, + spline_bc_type = "natural", + concatenate_train_test = True, + times_spacing_method = "evenly_spaced", + refit_models = True, + precomputed_encodings_path = base_save_dir / "facebook_dinov2-with-registers-giant_dataset_preproc" / "heldout_test_trajs_2_dose_10__chromaLive6h_3ch_png_patches_380px_fc_augs" / "facebook_dinov2-with-registers-giant_dataset_preproc_encodings.parquet", +) # ↑ was computed before and there are actually all files of the dataset in the encodings.parquet file (filtering happens after computing ...) +# fmt: on diff --git a/chromalive/my_conf/my_inference_conf.py b/chromalive/my_conf/my_inference_conf.py new file mode 100644 index 0000000000000000000000000000000000000000..05a555c90dbcd33be0f224c8ec297002b4929a83 --- /dev/null +++ b/chromalive/my_conf/my_inference_conf.py @@ -0,0 +1,84 @@ +# ruff: noqa: F401 + +from pathlib import Path + +from diffusers.schedulers.scheduling_dpmsolver_sde import DPMSolverSDEScheduler +from torch import bfloat16, float16, float32 + +from GaussianProxy.conf.inference_conf import InferenceConfig, ProfileConfig +from GaussianProxy.conf.training_conf import ( + EvaluationStrategy, + ForwardNoising, + ForwardNoisingLinearScaling, + InversionRegenerationOnly, + InvertedRegeneration, + IterativeInvertedRegeneration, + MetricsComputation, + SimilarityWithTrainData, + SimpleGeneration, + VideoGenerationFromNoise, +) + +# -------------------------------------------- Dataset -------------------------------------------- +from my_conf.dataset.ependymal_cutout.ependymal_cutout_01_noised_fc_augs_sep_gt_fully_ordered_inference import dataset + +# --------------------------------------------- Model --------------------------------------------- +root_experiments_path = Path("/", "projects", "static2dynamic", "Thomas", "experiments") +project_name = "GaussianProxy_v3" +# folder_name = dataset.name.removesuffix("_fully_ordered").removesuffix("_separate_gt") + "_JZ_download" +folder_name = "ependymal_cutout_noised0.1_fc_augs" +run_path = root_experiments_path / project_name / folder_name +assert run_path.exists(), f"'{run_path}' does not exist" + +# ------------------------------------------- Scheduler ------------------------------------------- +scheduler_type = None # defaults to DDIM + +# ------------------------------------------ Evaluations ------------------------------------------ +# fmt: off +eval_strats = [ + InvertedRegeneration( + nb_inversion_diffusion_timesteps = 200, + nb_diffusion_timesteps = 100, + nb_video_timesteps = 100, + nb_video_times_in_parallel = 10, + nb_generated_samples = 16, + n_rows_displayed = 4, + selected_times = None, + plate_name_to_simulate = None, + gen_times_type = "evenly_spaced_from_inversion", + split = "test", + ) +] +# fmt: on + +# ------------------------------------------ Final Config ----------------------------------------- +# fmt: off +inference_conf = InferenceConfig( + # Choose the experiment (== trained model weights) + root_experiments_path = root_experiments_path, + project_name = project_name, + run_name = folder_name, + saved_model_foldername = "last_model", + # Choose a custom scheduler + scheduler_type = scheduler_type, + scheduler_config_path = None, + import_orig_config = True, + # Output directory (where to put the generated images / tensors) + output_dir = run_path / "inferences", + # Device + device = "cuda:2", + # Optimizations + compile = True, + dtype = float32, + # Data + dataset = dataset, + true_label_time_0 = None, + # Evaluations + evaluation_strategies = eval_strats, # type: ignore[reportArgumentType] + # Profiling + profiling = ProfileConfig(), # off by default + # Debug + debug = False, + # Temp Dir + tmpdir_location = "/localtmp/tboyer/.tmpdir", +) diff --git a/chromalive/my_conf/my_training_conf.py b/chromalive/my_conf/my_training_conf.py new file mode 100644 index 0000000000000000000000000000000000000000..c757c21f6fc78a50a6f104bb20d7dd60ee621d36 --- /dev/null +++ b/chromalive/my_conf/my_training_conf.py @@ -0,0 +1,199 @@ +from pathlib import Path + +from omegaconf import MISSING + +################################################################################################### +############################################ Base conf ############################################ +################################################################################################### +# These are generic classes that need full instantiation +from GaussianProxy.conf.training_conf import ( + Accelerate, + AccelerateLaunchArgs, + Checkpointing, + Config, + DataLoader, + DDIMSchedulerConfig, + Evaluation, + ForwardNoising, # noqa: F401 + InvertedRegeneration, # noqa: F401 + IterativeInvertedRegeneration, # noqa: F401 + LinearLRConfig, # noqa: F401 + MetricsComputation, # noqa: F401 + OneCycleLRConfig, # noqa: F401 + SimilarityWithTrainData, # noqa: F401 + SimpleGeneration, # noqa: F401 + Slurm, + Training, +) + +################################################################################################### +########################################## Defaults conf ########################################## +################################################################################################### +defaults = [ + {"dataset": "ChromaLive6h/ChromaLive6h_3ch_png_high_doses_fully_ordered"}, + "hydra/job_logging/custom", + "_self_", +] +# fmt: off + +# ------------------------------------------- Job launch ------------------------------------------ +slurm = Slurm( + enabled = False, + monitor = False, + account = "icr@a100", + partition = "a100", + constraint = "a100", + qos = "dev", + nodes = 1, + num_gpus = 8, + max_num_requeue = 3, + output_folder = "${hydra:run.dir}", + email = "tboyer@bio.ens.psl.eu", +) + +accelerate_launch_args = AccelerateLaunchArgs( + machine_rank = 0, + num_machines = 1, + gpu_ids = "0,1,3,", + rdzv_backend = "static", + same_network = "true", + mixed_precision = "bf16", + num_processes = 3, + main_process_port = 29502, + dynamo_backend = "inductor", +) + +accelerate = Accelerate( + launch_args = accelerate_launch_args, + offline = False, # TODO: move this arg that does not belong here (make it general like debug) +) + +# ---------------------------------------------- Data --------------------------------------------- +data_loader = DataLoader( + num_workers = 4, + train_prefetch_factor = 4, + pin_memory = True, + persistent_workers = True, +) + +# -------------------------------------------- Training ------------------------------------------- +training = Training( + gradient_accumulation_steps = 1, + train_batch_size = 48 + 8, + max_grad_norm = 1, + nb_time_samplings = 500_000, + unpaired_data = False, + reweight_sampling = False, +) + +checkpointing = Checkpointing( + checkpoints_total_limit = 3, + resume_from_checkpoint = True, + checkpoint_every_n_steps = 5000, + chckpt_base_path = Path("/localtmp/tboyer/static2dynamic"), +) + +lr_scheduler = OneCycleLRConfig( + max_lr = 2e-4, +) + +# ------------------------------------------- Evaluation ------------------------------------------ +# naming convention is lowercase + underscore; has to be respected for debug args modification +metrics_compute = MetricsComputation( + nb_samples_to_gen_per_time = 10_000, + batch_size = 256, + nb_diffusion_timesteps = 50, + selected_times = ["time_01", "time_04", "time_07", "time_10", "time_13"], + augmentations_for_metrics_comp = [], + also_compute_metrics_on_all_times = False, + dtype = "bfloat16", +) + +simple_generation = SimpleGeneration( + nb_diffusion_timesteps = 50, + n_rows_displayed = 4, # TODO: merge training & evaluation configs + nb_generated_samples = 16, # TODO: merge training & evaluation configs +) + +inverted_regeneration = InvertedRegeneration( + nb_diffusion_timesteps = 50, + nb_inversion_diffusion_timesteps = 100, + n_rows_displayed = 8, # TODO: not used in training! + nb_generated_samples = 16, + nb_video_times_in_parallel = 4, # TODO: not used in training! + nb_video_timesteps = 50, + gen_times_type = "evenly_spaced_from_inversion", +) + +sim_with_train = SimilarityWithTrainData( # must be put after metrics_compute! + nb_generated_samples = -1, # TODO: not used + batch_size = 4096, + nb_batches_shown = -1, # TODO: not used + n_rows_displayed = -1, # TODO: not used + nb_diffusion_timesteps = -1, # TODO: not used +) + +evaluation = Evaluation( + every_n_opt_steps = 20_000, + batch_size = 16, # TODO: remove this and use config from above + nb_video_timesteps = 50, # TODO: remove this and use config from above + strategies = [simple_generation, inverted_regeneration, metrics_compute, sim_with_train], +) + +# ------------------------------------------- Diffusion ------------------------------------------- +dynamic = DDIMSchedulerConfig( + num_train_timesteps = 3000, + clip_sample = False, + clip_sample_range = 1, + thresholding = True, + sample_max_value = 1, + prediction_type = "v_prediction", + rescale_betas_zero_snr = False, + timestep_spacing = "leading", +) + +# ---------------------------------------------- Model -------------------------------------------- +from my_conf.net.net_128_3 import net, time_encoder # noqa: E402 + +# ------------------------------------------ Final Config ----------------------------------------- +config = Config( + # defaults + defaults = defaults, + # model + dynamic = dynamic, + net = net, + time_encoder = time_encoder, + # script + launcher_script_parent_folder = "/workspaces/biocomp/tboyer/sources/GaussianProxy", + script = "train", + # experiment variables + exp_parent_folder = "/projects/static2dynamic/Thomas/experiments", + project = MISSING, + run_name = MISSING, + # hydra + hydra = {"run": {"dir": "${exp_parent_folder}/${project}/${run_name}"}}, + # slurm + slurm = slurm, + # accelerate + accelerate = accelerate, + # misc. + debug = False, + profile = False, + tmpdir_location = "/localtmp/tboyer/.tmpdir", + # experiment tracker + logger = "wandb", + entity = "thomasboyer", + resume_method = "new_run", + # checkpointing + checkpointing = checkpointing, + # dataset + dataset = MISSING, + # dataloaders + dataloaders = data_loader, + # training + training = training, + # evaluation + evaluation = evaluation, + # optimizer + lr_scheduler = lr_scheduler, +) diff --git a/chromalive/my_conf/net/net_128_3.py b/chromalive/my_conf/net/net_128_3.py new file mode 100644 index 0000000000000000000000000000000000000000..994280ed4d0a7e6e1410e51f2b058157fb17c962 --- /dev/null +++ b/chromalive/my_conf/net/net_128_3.py @@ -0,0 +1,22 @@ +from GaussianProxy.conf.training_conf import UNet2DConditionModelConfig, TimeEncoderConfig + +cross_attn_dim = 64 + +net = UNet2DConditionModelConfig( + sample_size=128, + in_channels=3, + out_channels=3, + down_block_types=("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"), + block_out_channels=(64, 128, 256), + layers_per_block=2, + act_fn="silu", + cross_attention_dim=cross_attn_dim, +) + +time_encoder = TimeEncoderConfig( + encoding_dim=128, + time_embed_dim=cross_attn_dim, + flip_sin_to_cos=True, + downscale_freq_shift=1, +) diff --git a/chromalive/my_conf/net/net_128_3_big.py b/chromalive/my_conf/net/net_128_3_big.py new file mode 100644 index 0000000000000000000000000000000000000000..9a12e57ccfc47d7fd0a9ac9cf3647764361adac7 --- /dev/null +++ b/chromalive/my_conf/net/net_128_3_big.py @@ -0,0 +1,22 @@ +from GaussianProxy.conf.training_conf import UNet2DConditionModelConfig, TimeEncoderConfig + +cross_attn_dim = 64 + +net = UNet2DConditionModelConfig( + sample_size=128, + in_channels=3, + out_channels=3, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D"), + block_out_channels=(128, 256, 512), + layers_per_block=2, + act_fn="silu", + cross_attention_dim=cross_attn_dim, +) + +time_encoder = TimeEncoderConfig( + encoding_dim=128, + time_embed_dim=cross_attn_dim, + flip_sin_to_cos=True, + downscale_freq_shift=1, +) diff --git a/chromalive/my_conf/net/net_128_4.py b/chromalive/my_conf/net/net_128_4.py new file mode 100644 index 0000000000000000000000000000000000000000..0dd136ffaff1c94858ad50bb075d780b7a49bc86 --- /dev/null +++ b/chromalive/my_conf/net/net_128_4.py @@ -0,0 +1,22 @@ +from GaussianProxy.conf.training_conf import UNet2DConditionModelConfig, TimeEncoderConfig + +cross_attn_dim = 64 + +net = UNet2DConditionModelConfig( + sample_size=128, + in_channels=4, + out_channels=4, + down_block_types=("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"), + block_out_channels=(64, 128, 256), + layers_per_block=2, + act_fn="silu", + cross_attention_dim=cross_attn_dim, +) + +time_encoder = TimeEncoderConfig( + encoding_dim=128, + time_embed_dim=cross_attn_dim, + flip_sin_to_cos=True, + downscale_freq_shift=1, +) diff --git a/chromalive/my_conf/net/net_196_3_11M.py b/chromalive/my_conf/net/net_196_3_11M.py new file mode 100644 index 0000000000000000000000000000000000000000..e13a50b93f46f33a9a15b3d39525646609e41cb2 --- /dev/null +++ b/chromalive/my_conf/net/net_196_3_11M.py @@ -0,0 +1,22 @@ +from GaussianProxy.conf.training_conf import UNet2DConditionModelConfig, TimeEncoderConfig + +cross_attn_dim = 64 + +net = UNet2DConditionModelConfig( + sample_size=196, + in_channels=3, + out_channels=3, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D"), + block_out_channels=(96, 128, 128), + layers_per_block=2, + act_fn="silu", + cross_attention_dim=cross_attn_dim, +) + +time_encoder = TimeEncoderConfig( + encoding_dim=128, + time_embed_dim=cross_attn_dim, + flip_sin_to_cos=True, + downscale_freq_shift=1, +) diff --git a/chromalive/my_conf/net/net_196_3_12M.py b/chromalive/my_conf/net/net_196_3_12M.py new file mode 100644 index 0000000000000000000000000000000000000000..83fa0274707bdde54fd0e1141deec96f779b5393 --- /dev/null +++ b/chromalive/my_conf/net/net_196_3_12M.py @@ -0,0 +1,22 @@ +from GaussianProxy.conf.training_conf import UNet2DConditionModelConfig, TimeEncoderConfig + +cross_attn_dim = 64 + +net = UNet2DConditionModelConfig( + sample_size=196, + in_channels=3, + out_channels=3, + down_block_types=("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"), + block_out_channels=(96, 128, 128), + layers_per_block=2, + act_fn="silu", + cross_attention_dim=cross_attn_dim, +) + +time_encoder = TimeEncoderConfig( + encoding_dim=128, + time_embed_dim=cross_attn_dim, + flip_sin_to_cos=True, + downscale_freq_shift=1, +) diff --git a/chromalive/my_conf/net/net_196_3_4M.py b/chromalive/my_conf/net/net_196_3_4M.py new file mode 100644 index 0000000000000000000000000000000000000000..1f099e27892046be5f1559b9e05032a8469d815d --- /dev/null +++ b/chromalive/my_conf/net/net_196_3_4M.py @@ -0,0 +1,22 @@ +from GaussianProxy.conf.training_conf import UNet2DConditionModelConfig, TimeEncoderConfig + +cross_attn_dim = 64 + +net = UNet2DConditionModelConfig( + sample_size=196, + in_channels=3, + out_channels=3, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D"), + block_out_channels=(32, 64, 96), + layers_per_block=2, + act_fn="silu", + cross_attention_dim=cross_attn_dim, +) + +time_encoder = TimeEncoderConfig( + encoding_dim=128, + time_embed_dim=cross_attn_dim, + flip_sin_to_cos=True, + downscale_freq_shift=1, +) diff --git a/chromalive/my_conf/net/net_256_3_20M.py b/chromalive/my_conf/net/net_256_3_20M.py new file mode 100644 index 0000000000000000000000000000000000000000..af270af831f48c45847a38cb3714d32033855957 --- /dev/null +++ b/chromalive/my_conf/net/net_256_3_20M.py @@ -0,0 +1,22 @@ +from GaussianProxy.conf.training_conf import UNet2DConditionModelConfig, TimeEncoderConfig + +cross_attn_dim = 64 + +net = UNet2DConditionModelConfig( + sample_size=256, + in_channels=3, + out_channels=3, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D"), + block_out_channels=(64, 128, 224), + layers_per_block=2, + act_fn="silu", + cross_attention_dim=cross_attn_dim, +) + +time_encoder = TimeEncoderConfig( + encoding_dim=128, + time_embed_dim=cross_attn_dim, + flip_sin_to_cos=True, + downscale_freq_shift=1, +) diff --git a/chromalive/my_conf/net/net_256_3_6M.py b/chromalive/my_conf/net/net_256_3_6M.py new file mode 100644 index 0000000000000000000000000000000000000000..96a31d7fa08df8eb436f02c98b53cb154ac10585 --- /dev/null +++ b/chromalive/my_conf/net/net_256_3_6M.py @@ -0,0 +1,22 @@ +from GaussianProxy.conf.training_conf import TimeEncoderConfig, UNet2DConditionModelConfig + +cross_attn_dim = 64 + +net = UNet2DConditionModelConfig( + sample_size=256, + in_channels=3, + out_channels=3, + down_block_types=("DownBlock2D", "DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D", "UpBlock2D"), + block_out_channels=(32, 64, 128), + layers_per_block=2, + act_fn="silu", + cross_attention_dim=cross_attn_dim, +) + +time_encoder = TimeEncoderConfig( + encoding_dim=128, + time_embed_dim=cross_attn_dim, + flip_sin_to_cos=True, + downscale_freq_shift=1, +) diff --git a/chromalive/my_conf/net/net_256_3_7M.py b/chromalive/my_conf/net/net_256_3_7M.py new file mode 100644 index 0000000000000000000000000000000000000000..b47baf8af721f2e80ad3650c59cb7c6150420880 --- /dev/null +++ b/chromalive/my_conf/net/net_256_3_7M.py @@ -0,0 +1,22 @@ +from GaussianProxy.conf.training_conf import TimeEncoderConfig, UNet2DConditionModelConfig + +cross_attn_dim = 64 + +net = UNet2DConditionModelConfig( + sample_size=256, + in_channels=3, + out_channels=3, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D"), + block_out_channels=(32, 96, 128), + layers_per_block=2, + act_fn="silu", + cross_attention_dim=cross_attn_dim, +) + +time_encoder = TimeEncoderConfig( + encoding_dim=128, + time_embed_dim=cross_attn_dim, + flip_sin_to_cos=True, + downscale_freq_shift=1, +) diff --git a/chromalive/my_conf/net/net_256_3_8M.py b/chromalive/my_conf/net/net_256_3_8M.py new file mode 100644 index 0000000000000000000000000000000000000000..aaf714c9366c83c288206d012341f1f68c49a302 --- /dev/null +++ b/chromalive/my_conf/net/net_256_3_8M.py @@ -0,0 +1,22 @@ +from GaussianProxy.conf.training_conf import TimeEncoderConfig, UNet2DConditionModelConfig + +cross_attn_dim = 64 + +net = UNet2DConditionModelConfig( + sample_size=256, + in_channels=3, + out_channels=3, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D"), + block_out_channels=(64, 96, 128), + layers_per_block=2, + act_fn="silu", + cross_attention_dim=cross_attn_dim, +) + +time_encoder = TimeEncoderConfig( + encoding_dim=128, + time_embed_dim=cross_attn_dim, + flip_sin_to_cos=True, + downscale_freq_shift=1, +) diff --git a/chromalive/my_conf/net/net_48_1.py b/chromalive/my_conf/net/net_48_1.py new file mode 100644 index 0000000000000000000000000000000000000000..df3328413ffc47ff49eb7860ee153939a7fccc22 --- /dev/null +++ b/chromalive/my_conf/net/net_48_1.py @@ -0,0 +1,23 @@ +from GaussianProxy.conf.training_conf import TimeEncoderConfig, UNet2DConditionModelConfig + +cross_attn_dim = 32 + +net = UNet2DConditionModelConfig( + sample_size=48, + in_channels=1, + out_channels=1, + down_block_types=("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"), + block_out_channels=(16, 32, 64), + norm_num_groups=8, + layers_per_block=2, + act_fn="silu", + cross_attention_dim=cross_attn_dim, +) + +time_encoder = TimeEncoderConfig( + encoding_dim=64, + time_embed_dim=cross_attn_dim, + flip_sin_to_cos=True, + downscale_freq_shift=1, +) diff --git a/chromalive/my_conf/net/net_66_1.py b/chromalive/my_conf/net/net_66_1.py new file mode 100644 index 0000000000000000000000000000000000000000..0e2344c593b081433a24dadf810f9451c59ead8e --- /dev/null +++ b/chromalive/my_conf/net/net_66_1.py @@ -0,0 +1,23 @@ +from GaussianProxy.conf.training_conf import TimeEncoderConfig, UNet2DConditionModelConfig + +cross_attn_dim = 32 + +net = UNet2DConditionModelConfig( + sample_size=66, + in_channels=1, + out_channels=1, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D"), + block_out_channels=(16, 32, 64), + norm_num_groups=8, + layers_per_block=2, + act_fn="silu", + cross_attention_dim=cross_attn_dim, +) + +time_encoder = TimeEncoderConfig( + encoding_dim=64, + time_embed_dim=cross_attn_dim, + flip_sin_to_cos=True, + downscale_freq_shift=1, +) diff --git a/chromalive/my_conf/net/net_66_3_2M.py b/chromalive/my_conf/net/net_66_3_2M.py new file mode 100644 index 0000000000000000000000000000000000000000..43f1f32ac81b719f3be41e19ea2d0e1eaeed5477 --- /dev/null +++ b/chromalive/my_conf/net/net_66_3_2M.py @@ -0,0 +1,23 @@ +from GaussianProxy.conf.training_conf import UNet2DConditionModelConfig, TimeEncoderConfig + +cross_attn_dim = 64 + +net = UNet2DConditionModelConfig( + sample_size=66, + in_channels=3, + out_channels=3, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D"), + block_out_channels=(24, 40, 72), + norm_num_groups=8, + layers_per_block=2, + act_fn="silu", + cross_attention_dim=cross_attn_dim, +) + +time_encoder = TimeEncoderConfig( + encoding_dim=128, + time_embed_dim=cross_attn_dim, + flip_sin_to_cos=True, + downscale_freq_shift=1, +) diff --git a/chromalive/my_conf/net/net_Jurkat.py b/chromalive/my_conf/net/net_Jurkat.py new file mode 100644 index 0000000000000000000000000000000000000000..d2caa03e0745f70db79d5cf5b15f5033b43c2300 --- /dev/null +++ b/chromalive/my_conf/net/net_Jurkat.py @@ -0,0 +1,22 @@ +from GaussianProxy.conf.training_conf import UNet2DConditionModelConfig, TimeEncoderConfig + +cross_attn_dim = 64 + +net = UNet2DConditionModelConfig( + sample_size=66, # torch.compile seems to *silently* crash on this sample size (gives NaN gradients)! + in_channels=3, + out_channels=3, + down_block_types=("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"), + block_out_channels=(64, 128, 256), + layers_per_block=2, + act_fn="silu", + cross_attention_dim=cross_attn_dim, +) + +time_encoder = TimeEncoderConfig( + encoding_dim=128, + time_embed_dim=cross_attn_dim, + flip_sin_to_cos=True, + downscale_freq_shift=1, +) diff --git a/chromalive/my_conf/net/net_diabetic_retinopathy.py b/chromalive/my_conf/net/net_diabetic_retinopathy.py new file mode 100644 index 0000000000000000000000000000000000000000..deff587c95295fd429062a9ffd3e280a8530e92d --- /dev/null +++ b/chromalive/my_conf/net/net_diabetic_retinopathy.py @@ -0,0 +1,22 @@ +from GaussianProxy.conf.training_conf import UNet2DConditionModelConfig, TimeEncoderConfig + +cross_attn_dim = 64 + +net = UNet2DConditionModelConfig( + sample_size=256, + in_channels=3, + out_channels=3, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D"), + block_out_channels=(64, 128, 256), + layers_per_block=2, + act_fn="silu", + cross_attention_dim=cross_attn_dim, +) + +time_encoder = TimeEncoderConfig( + encoding_dim=128, + time_embed_dim=cross_attn_dim, + flip_sin_to_cos=True, + downscale_freq_shift=1, +) diff --git a/chromalive/my_conf/scheduler/DDIM_3k_vpred_tresh_SNR_trailing.json b/chromalive/my_conf/scheduler/DDIM_3k_vpred_tresh_SNR_trailing.json new file mode 100644 index 0000000000000000000000000000000000000000..f7994596907c2a827c8d9941c86adb19ed8e43f2 --- /dev/null +++ b/chromalive/my_conf/scheduler/DDIM_3k_vpred_tresh_SNR_trailing.json @@ -0,0 +1,19 @@ +{ + "_class_name": "DDIMScheduler", + "_diffusers_version": "0.32.2", + "beta_end": 0.02, + "beta_schedule": "linear", + "beta_start": 0.0001, + "clip_sample": false, + "clip_sample_range": 1.0, + "dynamic_thresholding_ratio": 0.995, + "num_train_timesteps": 3000, + "prediction_type": "v_prediction", + "rescale_betas_zero_snr": true, + "sample_max_value": 1.0, + "set_alpha_to_one": true, + "steps_offset": 0, + "thresholding": true, + "timestep_spacing": "trailing", + "trained_betas": null +} \ No newline at end of file diff --git a/chromalive/my_conf/scheduler/DDIM_3k_vpred_tresh_leading.json b/chromalive/my_conf/scheduler/DDIM_3k_vpred_tresh_leading.json new file mode 100644 index 0000000000000000000000000000000000000000..5ad012575a06c66b4dfe590012c318cf4712f09f --- /dev/null +++ b/chromalive/my_conf/scheduler/DDIM_3k_vpred_tresh_leading.json @@ -0,0 +1,19 @@ +{ + "_class_name": "DDIMScheduler", + "_diffusers_version": "0.32.2", + "beta_end": 0.02, + "beta_schedule": "linear", + "beta_start": 0.0001, + "clip_sample": false, + "clip_sample_range": 1.0, + "dynamic_thresholding_ratio": 0.995, + "num_train_timesteps": 3000, + "prediction_type": "v_prediction", + "rescale_betas_zero_snr": false, + "sample_max_value": 1.0, + "set_alpha_to_one": true, + "steps_offset": 0, + "thresholding": true, + "timestep_spacing": "leading", + "trained_betas": null +} \ No newline at end of file diff --git a/chromalive/net/config.json b/chromalive/net/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e90d472fe3b97457bf5d1211578056012f13bdae --- /dev/null +++ b/chromalive/net/config.json @@ -0,0 +1,64 @@ +{ + "_class_name": "UNet2DConditionModel", + "_diffusers_version": "0.35.2", + "act_fn": "silu", + "addition_embed_type": null, + "addition_embed_type_num_heads": 64, + "addition_time_embed_dim": null, + "attention_head_dim": 8, + "attention_type": "default", + "block_out_channels": [ + 64, + 128, + 256 + ], + "center_input_sample": false, + "class_embed_type": null, + "class_embeddings_concat": false, + "conv_in_kernel": 3, + "conv_out_kernel": 3, + "cross_attention_dim": 64, + "cross_attention_norm": null, + "down_block_types": [ + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D" + ], + "downsample_padding": 1, + "dropout": 0.0, + "dual_cross_attention": false, + "encoder_hid_dim": null, + "encoder_hid_dim_type": null, + "flip_sin_to_cos": true, + "freq_shift": 0, + "in_channels": 3, + "layers_per_block": 2, + "mid_block_only_cross_attention": null, + "mid_block_scale_factor": 1, + "mid_block_type": "UNetMidBlock2DCrossAttn", + "norm_eps": 1e-05, + "norm_num_groups": 32, + "num_attention_heads": null, + "num_class_embeds": null, + "only_cross_attention": false, + "out_channels": 3, + "projection_class_embeddings_input_dim": null, + "resnet_out_scale_factor": 1.0, + "resnet_skip_time_act": false, + "resnet_time_scale_shift": "default", + "reverse_transformer_layers_per_block": null, + "sample_size": 128, + "time_cond_proj_dim": null, + "time_embedding_act_fn": null, + "time_embedding_dim": null, + "time_embedding_type": "positional", + "timestep_post_act": null, + "transformer_layers_per_block": 1, + "up_block_types": [ + "CrossAttnUpBlock2D", + "CrossAttnUpBlock2D", + "CrossAttnUpBlock2D" + ], + "upcast_attention": false, + "use_linear_projection": false +} diff --git a/chromalive/video_time_encoder/config.json b/chromalive/video_time_encoder/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b910a40e475aaa21c0c9fb367ce4dcfab1ba4941 --- /dev/null +++ b/chromalive/video_time_encoder/config.json @@ -0,0 +1,8 @@ +{ + "_class_name": "VideoTimeEncoding", + "_diffusers_version": "0.35.2", + "downscale_freq_shift": 1.0, + "encoding_dim": 128, + "flip_sin_to_cos": true, + "time_embed_dim": 64 +} diff --git a/docetaxel_skip_half_doses/my_conf/dataset/NASH_steatosis/NASH_steatosis_fully_ordered.yaml b/docetaxel_skip_half_doses/my_conf/dataset/NASH_steatosis/NASH_steatosis_fully_ordered.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae97d5fc908ae875341f041200f043b1d9d7bbe9 --- /dev/null +++ b/docetaxel_skip_half_doses/my_conf/dataset/NASH_steatosis/NASH_steatosis_fully_ordered.yaml @@ -0,0 +1,26 @@ +name: NASH_steatosis_fully_ordered_dinov2_regs_giant_ds_preproc +path: /lustre/fsn1/projects/rech/icr/ufc43hj/datasets/NASH/steatosis +path_to_single_parquet: /lustre/fsn1/projects/rech/icr/ufc43hj/datasets/NASH/NASH_steatosis__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet +data_shape: [ 3, 128, 128 ] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + # Convert to float32 (and normalize to [0, 1]) before resizing + - _target_: torchvision.transforms.ConvertImageDtype + dtype: ${torch_dtype:float32} + # Random crop from 299x299 to 192x192, then resize to 128x128 + - _target_: torchvision.transforms.RandomCrop + size: 192 + - _target_: torchvision.transforms.Resize + size: 128 + # Normalize to [-1, 1] + - _target_: torchvision.transforms.Normalize + mean: [ 0.5, 0.5, 0.5 ] + std: [ 0.5, 0.5, 0.5 ] + # Random 8x square augmentations + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry +selected_dists: +expected_initial_data_range: [ 0, 255 ] +fully_ordered: true diff --git a/docetaxel_skip_half_doses/my_conf/dataset/ependymal_context/ependymal_context_avg_t_fully_ordered.yaml b/docetaxel_skip_half_doses/my_conf/dataset/ependymal_context/ependymal_context_avg_t_fully_ordered.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57782d87c232d1b8d3017631f705179b0e6b0a4e --- /dev/null +++ b/docetaxel_skip_half_doses/my_conf/dataset/ependymal_context/ependymal_context_avg_t_fully_ordered.yaml @@ -0,0 +1,24 @@ +name: ependymal_context_avg_t_separate_gt_fully_ordered + +path: /lustre/fsn1/projects/rech/icr/ufc43hj/datasets/ependymal/prepared_dataset_context_avg_pt_from_01_noised_facebook_dinov2-with-registers-giant/all_imgs +path_to_single_parquet: /lustre/fsn1/projects/rech/icr/ufc43hj/datasets/ependymal/ependymal_context__continuous_time_predictions__avg_t_preds.parquet + +data_shape: [3, 256, 256] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.ConvertImageDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} # passed dtype must be accessible as a "torch" attribute + - _target_: torchvision.transforms.Normalize + mean: [0.5, 0.5, 0.5] + std: [0.5, 0.5, 0.5] + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry + +expected_initial_data_range: [0, 255] +expected_dtype: torch.uint8 + +selected_dists: None + +fully_ordered: true diff --git a/docetaxel_skip_half_doses/my_conf/dataset/ependymal_context/ependymal_context_fully_ordered.yaml b/docetaxel_skip_half_doses/my_conf/dataset/ependymal_context/ependymal_context_fully_ordered.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a7de34c5269bcc38520e4237d2a179529a776f35 --- /dev/null +++ b/docetaxel_skip_half_doses/my_conf/dataset/ependymal_context/ependymal_context_fully_ordered.yaml @@ -0,0 +1,23 @@ +name: ependymal_context_fully_ordered + +path: /lustre/fsn1/projects/rech/icr/ufc43hj/datasets/ependymal/prepared_dataset_context +path_to_single_parquet: /lustre/fsn1/projects/rech/icr/ufc43hj/datasets/ependymal/ependymal_context__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet + +data_shape: [3, 256, 256] + +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.ConvertImageDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} # passed dtype must be accessible as a "torch" attribute + - _target_: torchvision.transforms.Normalize + mean: [0.5, 0.5, 0.5] + std: [0.5, 0.5, 0.5] + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry + +expected_initial_data_range: [0, 255] +expected_dtype: torch.uint8 + +fully_ordered: true diff --git a/docetaxel_skip_half_doses/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_separate_gt_fully_ordered.yaml b/docetaxel_skip_half_doses/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_separate_gt_fully_ordered.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c12f125a27d852d80c9aca3fbfffc0285aa4ccd5 --- /dev/null +++ b/docetaxel_skip_half_doses/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_separate_gt_fully_ordered.yaml @@ -0,0 +1,26 @@ +name: ependymal_cutout_01_noised_separate_gt_fully_ordered + +path: /lustre/fsn1/projects/rech/icr/ufc43hj/datasets/ependymal/prepared_dataset_noised_0.1_crop/all_imgs +hard_aug_path: /lustre/fsn1/projects/rech/icr/ufc43hj/datasets/ependymal/prepared_dataset_noised_0.1_crop_hard_augmented/all_imgs +path_to_single_parquet: /lustre/fsn1/projects/rech/icr/ufc43hj/datasets/ependymal/ependymal_cutout_01_noised__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet + +data_shape: [3, 256, 256] + +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.ConvertImageDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} # passed dtype must be accessible as a "torch" attribute + - _target_: torchvision.transforms.Normalize + mean: [0.5, 0.5, 0.5] + std: [0.5, 0.5, 0.5] + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry + +expected_initial_data_range: [0, 255] +expected_dtype: torch.uint8 + +selected_dists: None + +fully_ordered: true diff --git a/docetaxel_skip_half_doses/my_conf/dataset/ependymal_cutout/ependymal_cutout_03_noised_separate_gt_fully_ordered.yaml b/docetaxel_skip_half_doses/my_conf/dataset/ependymal_cutout/ependymal_cutout_03_noised_separate_gt_fully_ordered.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e58db20d77174474867a4c3d7666ea0b50db51ae --- /dev/null +++ b/docetaxel_skip_half_doses/my_conf/dataset/ependymal_cutout/ependymal_cutout_03_noised_separate_gt_fully_ordered.yaml @@ -0,0 +1,25 @@ +name: ependymal_cutout_03_noised_separate_gt_fully_ordered + +path: /lustre/fsn1/projects/rech/icr/ufc43hj/datasets/ependymal/prepared_dataset_noised_0.3_crop/all_imgs +hard_aug_path: /lustre/fsn1/projects/rech/icr/ufc43hj/datasets/ependymal/prepared_dataset_noised_0.3_crop_hard_augmented/all_imgs +path_to_single_parquet: /lustre/fsn1/projects/rech/icr/ufc43hj/datasets/ependymal/ependymal_cutout_03_noised__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet + +data_shape: [3, 256, 256] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.ConvertImageDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} # passed dtype must be accessible as a "torch" attribute + - _target_: torchvision.transforms.Normalize + mean: [0.5, 0.5, 0.5] + std: [0.5, 0.5, 0.5] + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry + +expected_initial_data_range: [0, 255] +expected_dtype: torch.uint8 + +selected_dists: None + +fully_ordered: true diff --git a/docetaxel_skip_half_doses/my_conf/dataset/ependymal_cutout/ependymal_cutout_inference.py b/docetaxel_skip_half_doses/my_conf/dataset/ependymal_cutout/ependymal_cutout_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..79ed41ec4ed87d4fb3e8cb8b828f3c780e29e56a --- /dev/null +++ b/docetaxel_skip_half_doses/my_conf/dataset/ependymal_cutout/ependymal_cutout_inference.py @@ -0,0 +1,32 @@ +from torch import float32 +from torchvision.transforms import Compose, ConvertImageDtype, Normalize + +from GaussianProxy.conf.training_conf import DataSet, DatasetParams +from GaussianProxy.utils.data import ImageDataset + +DEFINITION = 256 +NUMBER_OF_CHANNELS = 3 + +transforms = Compose( + transforms=[ + ConvertImageDtype(float32), + Normalize(mean=[0.5] * NUMBER_OF_CHANNELS, std=[0.5] * NUMBER_OF_CHANNELS), + ] +) + +ds_params = DatasetParams( + file_extension="png", + key_transform=str, + sorting_func=lambda subdir: int(subdir.name), + dataset_class=ImageDataset, +) + +dataset = DataSet( + name="ependymal_cutout", + data_shape=(NUMBER_OF_CHANNELS, DEFINITION, DEFINITION), + transforms=transforms, + selected_dists=[1, 2, 3, 4, 5, 6], # 0 is the trash class! + expected_initial_data_range=(0, 255), + dataset_params=ds_params, + path="/lustre/fsn1/projects/rech/icr/ufc43hj/datasets/ependymal/prepared_dataset_crop", +) diff --git a/docetaxel_skip_half_doses/my_conf/dataset/ependymal_cutout/ependymal_cutout_white_bg_separate_gt_fully_ordered.yaml b/docetaxel_skip_half_doses/my_conf/dataset/ependymal_cutout/ependymal_cutout_white_bg_separate_gt_fully_ordered.yaml new file mode 100644 index 0000000000000000000000000000000000000000..224bf0a50a5c94c1267a98c01e6298927e30156c --- /dev/null +++ b/docetaxel_skip_half_doses/my_conf/dataset/ependymal_cutout/ependymal_cutout_white_bg_separate_gt_fully_ordered.yaml @@ -0,0 +1,25 @@ +name: ependymal_cutout_white_bg_separate_gt_fully_ordered + +path: /lustre/fsn1/projects/rech/icr/ufc43hj/datasets/ependymal/prepared_dataset_white_bg_crop/all_imgs +hard_aug_path: /lustre/fsn1/projects/rech/icr/ufc43hj/datasets/ependymal/prepared_dataset_white_bg_crop_hard_augmented/all_imgs +path_to_single_parquet: /lustre/fsn1/projects/rech/icr/ufc43hj/datasets/ependymal/ependymal_cutout_white_bg__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet + +data_shape: [3, 256, 256] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.ConvertImageDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} # passed dtype must be accessible as a "torch" attribute + - _target_: torchvision.transforms.Normalize + mean: [0.5, 0.5, 0.5] + std: [0.5, 0.5, 0.5] + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry + +expected_initial_data_range: [0, 255] +expected_dtype: torch.uint8 + +selected_dists: None + +fully_ordered: true diff --git a/docetaxel_skip_half_doses/my_conf/dataset/human_embryo/human_embryo_fully_ordered.yaml b/docetaxel_skip_half_doses/my_conf/dataset/human_embryo/human_embryo_fully_ordered.yaml new file mode 100644 index 0000000000000000000000000000000000000000..412151680442ae7f72ce4f15aca37850cae1db26 --- /dev/null +++ b/docetaxel_skip_half_doses/my_conf/dataset/human_embryo/human_embryo_fully_ordered.yaml @@ -0,0 +1,24 @@ +name: human_embryo + +path: /lustre/fsn1/projects/rech/icr/ufc43hj/datasets/human_embryo/phases +path_to_single_parquet: /lustre/fsn1/projects/rech/icr/ufc43hj/datasets/human_embryo/human_embryo__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet + +data_shape: [1, 256, 256] + +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.Resize + size: 256 + - _target_: torchvision.transforms.ConvertImageDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} # passed dtype must be accessible as a "torch" attribute + - _target_: torchvision.transforms.Normalize + mean: [0.5] + std: [0.5] + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry + +expected_initial_data_range: [0, 255] +expected_dtype: torch.uint8 +fully_ordered: true diff --git a/ependymal/my_conf/dataset/NASH_steatosis/NASH_steatosis_fully_ordered_inference.py b/ependymal/my_conf/dataset/NASH_steatosis/NASH_steatosis_fully_ordered_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..d9918cc6e00541668f1d52847f662fad05fcf329 --- /dev/null +++ b/ependymal/my_conf/dataset/NASH_steatosis/NASH_steatosis_fully_ordered_inference.py @@ -0,0 +1,4 @@ +from GaussianProxy.conf.dataset.NASH_steatosis.NASH_steatosis_fully_ordered_inference import dataset + +dataset.path = "/projects/static2dynamic/datasets/NASH/prepared_data/steatosis" +dataset.path_to_single_parquet = "/projects/static2dynamic/datasets/NASH/prepared_data/NASH_steatosis__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet" diff --git a/ependymal/my_conf/dataset/NASH_steatosis/NASH_steatosis_inference.py b/ependymal/my_conf/dataset/NASH_steatosis/NASH_steatosis_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..6e1d91853a23054a41429f141a048594f0446074 --- /dev/null +++ b/ependymal/my_conf/dataset/NASH_steatosis/NASH_steatosis_inference.py @@ -0,0 +1,3 @@ +from GaussianProxy.conf.dataset.NASH_steatosis.NASH_steatosis_inference import dataset + +dataset.path = "/projects/static2dynamic/datasets/NASH/prepared_data/steatosis" diff --git a/ependymal/my_conf/dataset/biotine/biotine_png_128_inference.py b/ependymal/my_conf/dataset/biotine/biotine_png_128_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..2f885668d8f708cce05382366a32dc71a3da7976 --- /dev/null +++ b/ependymal/my_conf/dataset/biotine/biotine_png_128_inference.py @@ -0,0 +1,8 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.biotine.biotine_png_128_inference import dataset + +dataset = replace( + dataset, + path="/projects/static2dynamic/datasets/biotine/3_channels_min_99_perc_normalized_rgb_stacks_png/patches_255", +) diff --git a/ependymal/my_conf/dataset/biotine/biotine_png_256.yaml b/ependymal/my_conf/dataset/biotine/biotine_png_256.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2097e5ff9e66cec4a39e2fccea9f31f8751cbe4 --- /dev/null +++ b/ependymal/my_conf/dataset/biotine/biotine_png_256.yaml @@ -0,0 +1,18 @@ +name: biotine_png +path: /projects/static2dynamic/datasets/biotine/3_channels_min_99_perc_normalized_rgb_stacks_png/patches_255 +data_shape: [ 3, 256, 256 ] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.ConvertImageDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} + - _target_: torchvision.transforms.Normalize + mean: [ 0.5, 0.5, 0.5 ] + std: [ 0.5, 0.5, 0.5 ] + - _target_: torchvision.transforms.transforms.Resize + size: 256 + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry +selected_dists: [ 1, 5, 10, 15, 19 ] +expected_initial_data_range: [ 0, 255 ] diff --git a/ependymal/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy.yaml b/ependymal/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b458e5fc57edb75c902a3ddb5519f9064950f555 --- /dev/null +++ b/ependymal/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy.yaml @@ -0,0 +1,18 @@ +name: diabetic_retinopathy +path: /projects/static2dynamic/datasets/DiabeticRetinopathy/prepared_dataset/train +data_shape: [3, 256, 256] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.transforms.Resize + size: 256 # single int => image resized to (size * aspect_ratio, size) or (size, size * aspect_ratio) with aspect_ratio >= 1 preserved + - _target_: torchvision.transforms.v2.CenterCrop + size: 256 # square centered crop + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.ConvertImageDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} # passed dtype must be accessible as a "torch" attribute + - _target_: torchvision.transforms.Normalize + mean: [0.5, 0.5, 0.5] + std: [0.5, 0.5, 0.5] +expected_initial_data_range: [0, 255] +expected_dtype: torch.uint8 diff --git a/ependymal/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_crop_inference.py b/ependymal/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_crop_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..84c134f0c7aa14e0f25ae95c629e26d719e7bd22 --- /dev/null +++ b/ependymal/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_crop_inference.py @@ -0,0 +1,9 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.diabetic_retinopathy.diabetic_retinopathy_full_circle_augs_2048_crop_inference import ( + dataset, +) + +dataset = replace( + dataset, path="/projects/static2dynamic/datasets/DiabeticRetinopathy/prepared_dataset_full_circle_augmented" +) diff --git a/ependymal/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_precrop_fully_ordered.yaml b/ependymal/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_precrop_fully_ordered.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4ecf72521e33c733056c5d0927ed58711dad04b --- /dev/null +++ b/ependymal/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_precrop_fully_ordered.yaml @@ -0,0 +1,21 @@ +name: diabetic_retinopathy_2048_precrop_fully_ordered + +path: /projects/static2dynamic/datasets/DiabeticRetinopathy/prepared_dataset_full_circle_augmented_2048_crop +path_to_single_parquet: /projects/static2dynamic/Thomas/ordering_datasets/facebook_dinov2-with-registers-giant_dataset_preproc/diabetic_retinopathy_full_circle_augs_2048_precrop/diabetic_retinopathy_full_circle_augs_2048_precrop__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet + +data_shape: [ 3, 256, 256 ] + +transforms: + _target_: torchvision.transforms.v2.Compose + transforms: + - _target_: torchvision.transforms.v2.ToDtype + dtype: ${torch_dtype:float32} + scale: true + - _target_: torchvision.transforms.v2.Normalize + mean: [ 0.5, 0.5, 0.5 ] + std: [ 0.5, 0.5, 0.5 ] + +expected_initial_data_range: [ 0, 255 ] +expected_dtype: torch.uint8 + +fully_ordered: true diff --git a/ependymal/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_precrop_inference.py b/ependymal/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_precrop_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..56a796fecda6f42247e83ee7b505cf738c625423 --- /dev/null +++ b/ependymal/my_conf/dataset/diabetic_retinopathy/diabetic_retinopathy_full_circle_augs_2048_precrop_inference.py @@ -0,0 +1,10 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.diabetic_retinopathy.diabetic_retinopathy_full_circle_augs_2048_precrop_inference import ( + dataset, +) + +dataset = replace( + dataset, + path="/projects/static2dynamic/datasets/DiabeticRetinopathy/prepared_dataset_full_circle_augmented_2048_crop", +) diff --git a/ependymal/my_conf/dataset/ependymal_context/ependymal_context.yaml b/ependymal/my_conf/dataset/ependymal_context/ependymal_context.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4301cfd2c648085445c33cb570d1a0941930efa0 --- /dev/null +++ b/ependymal/my_conf/dataset/ependymal_context/ependymal_context.yaml @@ -0,0 +1,16 @@ +name: ependymal_context +path: /projects/static2dynamic/datasets/ependymal/prepared_dataset_context +data_shape: [3, 256, 256] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.ConvertImageDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} # passed dtype must be accessible as a "torch" attribute + - _target_: torchvision.transforms.Normalize + mean: [0.5, 0.5, 0.5] + std: [0.5, 0.5, 0.5] + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry +expected_initial_data_range: [0, 255] +expected_dtype: torch.uint8 diff --git a/ependymal/my_conf/dataset/ependymal_context/ependymal_context_avg_t_fully_ordered_inference.py b/ependymal/my_conf/dataset/ependymal_context/ependymal_context_avg_t_fully_ordered_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..83ce32546f7aa4e097ca079b7ae975bbaec9c701 --- /dev/null +++ b/ependymal/my_conf/dataset/ependymal_context/ependymal_context_avg_t_fully_ordered_inference.py @@ -0,0 +1,31 @@ +from dataclasses import replace +from pathlib import Path + +from GaussianProxy.conf.dataset.ependymal_context.ependymal_context_fully_ordered_inference import dataset +from GaussianProxy.conf.training_conf import DatasetParams +from GaussianProxy.utils.data import ContinuousTimeImageDataset + + +# separate gt needs a workaround for sorting_func +def sorting_func(subdir: Path): + if subdir.name == "all_imgs": + return 0 + else: + raise ValueError(f"unexpected subdir: {subdir}") + + +ds_params = DatasetParams( + file_extension="png", + key_transform=str, + sorting_func=sorting_func, + dataset_class=ContinuousTimeImageDataset, +) + +dataset = replace( + dataset, + name="ependymal_context_avg_t_separate_gt_fully_ordered", + path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_context_avg_pt_from_01_noised_facebook_dinov2-with-registers-giant/all_imgs", + path_to_single_parquet="/projects/static2dynamic/datasets/ependymal/prepared_dataset_context_avg_pt_from_01_noised_facebook_dinov2-with-registers-giant/ependymal_context__continuous_time_predictions__avg_t_preds.parquet", + dataset_params=ds_params, + separate_gt_starting_class_path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_context/0" +) diff --git a/ependymal/my_conf/dataset/ependymal_context/ependymal_context_fully_ordered_inference.py b/ependymal/my_conf/dataset/ependymal_context/ependymal_context_fully_ordered_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..b62649854264f1653da9f1d727421730615f77b1 --- /dev/null +++ b/ependymal/my_conf/dataset/ependymal_context/ependymal_context_fully_ordered_inference.py @@ -0,0 +1,9 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.ependymal_context.ependymal_context_fully_ordered_inference import dataset + +dataset = replace( + dataset, + path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_context", + path_to_single_parquet="/projects/static2dynamic/datasets/ependymal/ependymal_context__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet", +) diff --git a/ependymal/my_conf/dataset/ependymal_context/ependymal_context_inference.py b/ependymal/my_conf/dataset/ependymal_context/ependymal_context_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..f9514fb17a5910cb32bfd7692534a4fefe2ee78d --- /dev/null +++ b/ependymal/my_conf/dataset/ependymal_context/ependymal_context_inference.py @@ -0,0 +1,32 @@ +from torch import float32 +from torchvision.transforms import Compose, ConvertImageDtype, Normalize + +from GaussianProxy.conf.training_conf import DataSet, DatasetParams +from GaussianProxy.utils.data import ImageDataset + +DEFINITION = 256 +NUMBER_OF_CHANNELS = 3 + +transforms = Compose( + transforms=[ + ConvertImageDtype(float32), + Normalize(mean=[0.5] * NUMBER_OF_CHANNELS, std=[0.5] * NUMBER_OF_CHANNELS), + ] +) + +ds_params = DatasetParams( + file_extension="png", + key_transform=str, + sorting_func=lambda subdir: int(subdir.name), + dataset_class=ImageDataset, +) + +dataset = DataSet( + name="ependymal_context", + data_shape=(NUMBER_OF_CHANNELS, DEFINITION, DEFINITION), + transforms=transforms, + expected_initial_data_range=(0, 255), + dataset_params=ds_params, + path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_context", + selected_dists=["0", "3", "5", "7", "9", "16", "30"], # ignore .REMOVED_IMAGES/16 (imaging artifacts) +) diff --git a/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout.yaml b/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout.yaml new file mode 100644 index 0000000000000000000000000000000000000000..67fc08a415914501a2d04527a3555ddc498f32f4 --- /dev/null +++ b/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout.yaml @@ -0,0 +1,17 @@ +name: ependymal_cutout +path: /projects/static2dynamic/datasets/ependymal/prepared_dataset_crop +data_shape: [ 3, 256, 256 ] +transforms: + _target_: torchvision.transforms.transforms.Compose + transforms: + - _target_: torchvision.transforms.ConvertImageDtype # this also scales to [0; 1] + dtype: ${torch_dtype:float32} # passed dtype must be accessible as a "torch" attribute + - _target_: torchvision.transforms.Normalize + mean: [ 0.5, 0.5, 0.5 ] + std: [ 0.5, 0.5, 0.5 ] + - _target_: torchvision.transforms.RandomHorizontalFlip + - _target_: torchvision.transforms.RandomVerticalFlip + - _target_: GaussianProxy.utils.data.RandomRotationSquareSymmetry +expected_initial_data_range: [ 0, 255 ] +expected_dtype: torch.uint8 +selected_dists: [ 1, 2, 3, 4, 5, 6 ] # 0 is the trash class! diff --git a/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_fc_augs_sep_gt_inference.py b/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_fc_augs_sep_gt_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..0297de6cf8a83d1822132568b745db3f4d587e35 --- /dev/null +++ b/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_fc_augs_sep_gt_inference.py @@ -0,0 +1,20 @@ +from dataclasses import replace +from pathlib import Path + +from GaussianProxy.conf.dataset.ependymal_cutout.ependymal_cutout_inference import dataset + +# note the data organization is special with separate ground truth folder +assert dataset.dataset_params is not None +dataset_params = replace( + dataset.dataset_params, + sorting_func=lambda subdir: str(subdir.name) if isinstance(subdir, Path) else str(subdir), +) # subdirs are "ground_truth" and "all_imgs" now... + +dataset = replace( + dataset, + name="ependymal_cutout_01_noised_full_circle_augs", + path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_noised_0.1_crop_full_circle_augmented", + separate_gt_starting_class_path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_noised_0.1_crop_full_circle_augmented/ground_truths/1", + dataset_params=dataset_params, + selected_dists=None, # no discrete folders anymore! +) diff --git a/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_separate_gt_fully_ordered_inference.py b/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_separate_gt_fully_ordered_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..a3c385439ee36d2153e9909e3e3131c23a82ef03 --- /dev/null +++ b/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_separate_gt_fully_ordered_inference.py @@ -0,0 +1,18 @@ +from dataclasses import replace + +from GaussianProxy.utils.data import ContinuousTimeImageDataset +from my_conf.dataset.ependymal_cutout.ependymal_cutout_01_noised_separate_gt_inference import dataset + +assert dataset.dataset_params is not None +dataset_params = replace( + dataset.dataset_params, + dataset_class=ContinuousTimeImageDataset, +) + +dataset = replace( + dataset, + fully_ordered=True, + path_to_single_parquet="/projects/static2dynamic/datasets/ependymal/ependymal_cutout_01_noised__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet", + dataset_params=dataset_params, + selected_dists=["all_imgs"], +) diff --git a/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_separate_gt_inference.py b/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_separate_gt_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..3cb66f6d9fa66d668624fb5839d8871e9da68344 --- /dev/null +++ b/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_01_noised_separate_gt_inference.py @@ -0,0 +1,19 @@ +from dataclasses import replace +from pathlib import Path + +from GaussianProxy.conf.dataset.ependymal_cutout.ependymal_cutout_inference import dataset + +# note the data organization is special with separate ground truth folder +assert dataset.dataset_params is not None +dataset_params = replace( + dataset.dataset_params, + sorting_func=lambda subdir: str(subdir.name) if isinstance(subdir, Path) else str(subdir), +) # subdirs are "ground_truth" and "all_imgs" now... + +dataset = replace( + dataset, + name="ependymal_cutout_01_noised", + path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_noised_0.1_crop/", + dataset_params=dataset_params, + separate_gt_starting_class_path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_noised_0.1_crop/ground_truths/1", +) diff --git a/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_03_noised_separate_gt_inference.py b/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_03_noised_separate_gt_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..4a1c4aedf4feae0aef44851b048c35192dbc051f --- /dev/null +++ b/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_03_noised_separate_gt_inference.py @@ -0,0 +1,17 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.ependymal_cutout.ependymal_cutout_inference import dataset + +# note the data organization is special with separate ground truth folder +assert dataset.dataset_params is not None +dataset_params = replace( + dataset.dataset_params, sorting_func=lambda subdir: str(subdir.name) +) # subdirs are "ground_truth" and "all_imgs" now... + +dataset = replace( + dataset, + name="ependymal_cutout_03_noised", + path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_noised_0.3_crop/", + dataset_params=dataset_params, + separate_gt_starting_class_path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_noised_0.3_crop/ground_truths/1", +) diff --git a/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_inference.py b/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..3458d8b0f72c8fc1042d2fcc4feb2b64bb96ece2 --- /dev/null +++ b/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_inference.py @@ -0,0 +1,8 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.ependymal_cutout.ependymal_cutout_inference import dataset + +dataset = replace( + dataset, + path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_crop", +) diff --git a/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_white_bg_separate_gt_fully_ordered_inference.py b/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_white_bg_separate_gt_fully_ordered_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..1fd379f5de2accb4ff47bc112191f2e7232ea35d --- /dev/null +++ b/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_white_bg_separate_gt_fully_ordered_inference.py @@ -0,0 +1,19 @@ +from dataclasses import replace + +from GaussianProxy.utils.data import ContinuousTimeImageDataset +from my_conf.dataset.ependymal_cutout.ependymal_cutout_white_bg_separate_gt_inference import dataset + +assert dataset.dataset_params is not None +dataset_params = replace( + dataset.dataset_params, + dataset_class=ContinuousTimeImageDataset, +) + +dataset = replace( + dataset, + fully_ordered=True, + path_to_single_parquet="/projects/static2dynamic/datasets/ependymal/ependymal_cutout_white_bg__continuous_time_predictions__facebook_dinov2-with-registers-giant_dataset_preproc.parquet", + dataset_params=dataset_params, + selected_dists=["all_imgs"], + separate_gt_starting_class_path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_white_bg_crop/ground_truths/1", +) diff --git a/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_white_bg_separate_gt_inference.py b/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_white_bg_separate_gt_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..807c4c6bcee124ac2b3c3ac0cb8a7251e60d71e6 --- /dev/null +++ b/ependymal/my_conf/dataset/ependymal_cutout/ependymal_cutout_white_bg_separate_gt_inference.py @@ -0,0 +1,16 @@ +from dataclasses import replace + +from GaussianProxy.conf.dataset.ependymal_cutout.ependymal_cutout_inference import dataset + +# note the data organization is special with separate ground truth folder +assert dataset.dataset_params is not None +dataset_params = replace( + dataset.dataset_params, sorting_func=lambda subdir: str(subdir.name) +) # subdirs are "ground_truth" and "all_imgs" now... + +dataset = replace( + dataset, + name="ependymal_cutout_white_bg", + path="/projects/static2dynamic/datasets/ependymal/prepared_dataset_white_bg_crop/", + dataset_params=dataset_params, +)