Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,60 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+CellPilot/images/model.png filter=lfs diff=lfs merge=lfs -text
+CellPilot/resources/CellViT/docs/figures/example.gif filter=lfs diff=lfs merge=lfs -text
+CellPilot/resources/CellViT/docs/figures/network_large.png filter=lfs diff=lfs merge=lfs -text
+CellPilot/resources/CellViT/docs/figures/qupath.gif filter=lfs diff=lfs merge=lfs -text
+CellPilot/resources/SimpleClick/assets/test_imgs/ssTEM_00.tiff filter=lfs diff=lfs merge=lfs -text
+MTdata/data_analysis/.ipynb_checkpoints/camelyon17-checkpoint.ipynb filter=lfs diff=lfs merge=lfs -text
+MTdata/data_analysis/CATCH.ipynb filter=lfs diff=lfs merge=lfs -text
+MTdata/data_analysis/Janow.ipynb filter=lfs diff=lfs merge=lfs -text
+MTdata/data_analysis/PAIP.ipynb filter=lfs diff=lfs merge=lfs -text
+MTdata/data_analysis/camelyon17.ipynb filter=lfs diff=lfs merge=lfs -text
+SAMHI/images/model.png filter=lfs diff=lfs merge=lfs -text
+SAMHI/resources/CellViT/docs/figures/example.gif filter=lfs diff=lfs merge=lfs -text
+SAMHI/resources/CellViT/docs/figures/network_large.png filter=lfs diff=lfs merge=lfs -text
+SAMHI/resources/CellViT/docs/figures/qupath.gif filter=lfs diff=lfs merge=lfs -text
+SAMHI/resources/SimpleClick/assets/test_imgs/ssTEM_00.tiff filter=lfs diff=lfs merge=lfs -text
+aws/dist/_awscrt.abi3.so filter=lfs diff=lfs merge=lfs -text
+aws/dist/_ruamel_yaml.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+aws/dist/aws filter=lfs diff=lfs merge=lfs -text
+aws/dist/aws_completer filter=lfs diff=lfs merge=lfs -text
+aws/dist/awscli/data/ac.index filter=lfs diff=lfs merge=lfs -text
+aws/dist/cryptography/hazmat/bindings/_openssl.abi3.so filter=lfs diff=lfs merge=lfs -text
+aws/dist/lib-dynload/_decimal.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+aws/dist/lib-dynload/_hashlib.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+aws/dist/lib-dynload/_ssl.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+aws/dist/lib-dynload/pyexpat.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+aws/dist/lib-dynload/unicodedata.cpython-311-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+aws/dist/libpython3.11.so.1.0 filter=lfs diff=lfs merge=lfs -text
+aws/dist/libsqlite3.so.0 filter=lfs diff=lfs merge=lfs -text
+repositories/CellViT/docs/figures/example.gif filter=lfs diff=lfs merge=lfs -text
+repositories/CellViT/docs/figures/network_large.png filter=lfs diff=lfs merge=lfs -text
+repositories/CellViT/docs/figures/qupath.gif filter=lfs diff=lfs merge=lfs -text
+repositories/SimpleClick/assets/test_imgs/ssTEM_00.tiff filter=lfs diff=lfs merge=lfs -text
+samples/CRAG_image.png filter=lfs diff=lfs merge=lfs -text
+samples/CoCaHis_image.png filter=lfs diff=lfs merge=lfs -text
+samples/Janowczyk_image.png filter=lfs diff=lfs merge=lfs -text
+samples/Kumar_image.png filter=lfs diff=lfs merge=lfs -text
+samples/MoNuSAC_image.png filter=lfs diff=lfs merge=lfs -text
+samples/TIGER_bulk_image.png filter=lfs diff=lfs merge=lfs -text
+samples/cellseg.png filter=lfs diff=lfs merge=lfs -text
+samples/cellvit_output.png filter=lfs diff=lfs merge=lfs -text
+samples/final/cellseg_0.png filter=lfs diff=lfs merge=lfs -text
+samples/final/cellseg_2.png filter=lfs diff=lfs merge=lfs -text
+samples/final/cellseg_6.png filter=lfs diff=lfs merge=lfs -text
+samples/results/cellseg_box_medsam.png filter=lfs diff=lfs merge=lfs -text
+samples/results/cellseg_box_ours.png filter=lfs diff=lfs merge=lfs -text
+samples/results/crag_box_gt.png filter=lfs diff=lfs merge=lfs -text
+samples/results/crag_box_medsam.png filter=lfs diff=lfs merge=lfs -text
+samples/results/crag_box_ours.png filter=lfs diff=lfs merge=lfs -text
+samples/results/crag_box_sam.png filter=lfs diff=lfs merge=lfs -text
+samples/results/monusac_box_gt.png filter=lfs diff=lfs merge=lfs -text
+samples/results/monusac_boxes_gt.png filter=lfs diff=lfs merge=lfs -text
+samples/results/monusac_boxes_medsam.png filter=lfs diff=lfs merge=lfs -text
+samples/results/monusac_boxes_ours.png filter=lfs diff=lfs merge=lfs -text
+samples/results/monusac_boxes_sam.png filter=lfs diff=lfs merge=lfs -text
+segment-anything/assets/masks1.png filter=lfs diff=lfs merge=lfs -text
+segment-anything/assets/minidemo.gif filter=lfs diff=lfs merge=lfs -text
+segment-anything/assets/notebook2.png filter=lfs diff=lfs merge=lfs -text

CellPilot/.gitmodules

@@ -0,0 +1,6 @@
+[submodule "resources/SimpleClick"]
+	path = resources/SimpleClick
+	url = https://github.com/philippendres/SimpleClick.git
+[submodule "resources/CellViT"]
+	path = resources/CellViT
+	url = https://github.com/philippendres/CellViT.git

CellPilot/README.md

@@ -0,0 +1,72 @@
+# CellPilot
+This repository contains the code of CellPilot, a deep learning-based method for the automatic and interactive segmentation of cells and glands in histological images. CellPilot uses [SAM](https://arxiv.org/abs/2304.02643) and [CellViT](https://arxiv.org/abs/2306.15350) and is fine-tuned on large-scale segmentation datasets of histological images.
+![Model](./images/model.png)
+
+## Key Features
+- **Automatic Segmentation**: CellPilot allows users to automatically segment cells in histological images.
+- **Interactive Segmentation**: CellPilot allows users to interactively segment cells and glands in histological images.
+
+## Setup
+1. Clone the repository with: `git clone https://github.com/philippendres/CellPilot.git`
+2. Create a new conda environment with the provided environment.yml file:
+    ```
+    conda env create -f environment.yml
+    conda activate histo3.10
+    ```
+3. Install the resources:
+    ```
+    cd resources
+    cd CellViT
+    git submodule init
+    git submodule update
+    pip install -e .
+    cd ..
+    cd SimpleClick
+    git submodule init
+    git submodule update
+    pip install -e .
+    cd ..
+    cd ..
+    ```
+4. Install our package: 
+    ```
+    pip install -e .
+    ```
+5.  - For inference: Download the weights of the CellPilot model and the CellViT model: [CellPilot](https://1drv.ms/u/c/696e40c0eaa91ac3/EfwN6MP4IqpHityPWHnHkkgBh5MyZEdYGVf9soXDs0gKOg?e=M8DVNQ), [CellViT](https://drive.google.com/uc?export=download&id=1tVYAapUo1Xt8QgCN22Ne1urbbCZkah8q)
+    - For training: Download the weights of the SAM model: [SAM](https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth)
+    - For model comparisons: Download the weights of the SimpleClick and the MedSAM model: [SimpleClick](https://drive.google.com/file/d/1dLAEFXhnk_Nebq3Net11sf6MjRCBEe0O/view?usp=drive_link), [MedSAM](https://drive.google.com/file/d/1UAmWL88roYR7wKlnApw5Bcuzf2iQgk6_/view?usp=drive_link)
+
+## Usage
+### App
+Run the gradio webapplication with the following command:
+```
+python app.py --model_dir <model_dir> --model_name <model_name> --cellvit_model <cellvit_model>
+```
+The app has the following arguments:
+- **model_dir**: The directory where the CellPilot and the CellViT model are stored.
+- **model_name**: The name of the CellPilot model.
+- **cellvit_model**: The name of the CellViT model.
+
+The command above will generate a link to a webapplication where you can upload your own images and segment them with SAMHI.
+The app will look like this:
+![App](./images/app.png)
+The app has the following features:
+- **Upload Image**: Upload your own image to segment in the upper left corner.
+- **Auto Segment**: Automatically segment the uploaded image with CellPilot.
+- **Add Mask**: Interactively add a mask with CellPilot by drawing points and bounding boxes on the image.
+- **Refine Mask**: Refine an existing mask by drawing points and bounding boxes on the image.
+- **Remove Mask**: Remove an existing mask by clicking on it.
+- **Move the Image**: Move the image with the arrow symbols.
+- **Zoom the Image**: Zoom the image with the zoom bar.
+
+<!-- ### Training
+Check that the data is stored in the structure given in [data_processing.md](./samhi/data_processing/data_processing.md)
+Run the training script with the following command:
+```
+python train.py
+```
+The evaluation script has the following arguments:
+- **cluster**: The cluster to run the training on.
+- **model_dir**: The directory where the SAMHI and the CellViT model are stored. -->
+
+

CellPilot/cellpilot/data_processing/data_fetching.py

@@ -0,0 +1,473 @@
+import os
+import numpy as np
+
+class DataFetcher:
+    def __init__(self, data_directory, cluster):
+        self.data_directory = data_directory
+        self.cluster = cluster
+
+    def len_bcss(self):
+        return len(os.listdir(os.path.join(self.data_directory, 'BCSS/0_Public-data-Amgad2019_0.25MPP', 'masks')))
+
+    def len_camelyon(self):
+        self.cam_len = [len(os.listdir(os.path.join(self.data_directory, 'CAMELYON/CAMELYON16/images/'))), 
+                            len(os.listdir(os.path.join(self.data_directory, 'CAMELYON/CAMELYON17/images/')))]
+        return  self.cam_len[0] + self.cam_len[1]
+
+    def len_cellseg(self):
+        self.cellseg_len = [
+            len(os.listdir(os.path.join(self.data_directory, 'CellSeg/NeurIPS22-CellSeg', 'Testing/Hidden/images/'))),
+            len(os.listdir(os.path.join(self.data_directory, 'CellSeg/NeurIPS22-CellSeg', 'Testing/Public/images/'))),
+            len(os.listdir(os.path.join(self.data_directory, 'CellSeg/NeurIPS22-CellSeg', 'Training/images/'))),
+            len(os.listdir(os.path.join(self.data_directory, 'CellSeg/NeurIPS22-CellSeg', 'Tuning/images/')))
+        ]
+        return sum(self.cellseg_len)
+
+    def len_cocahis(self):
+        return 82
+
+    def len_conic(self):
+        return len(os.listdir(os.path.join(self.data_directory, 'CoNIC/labels_png/')))
+
+    def len_cpm(self):
+        self.cpm_len = [len(os.listdir(os.path.join(self.data_directory, 'CPM_15_and_17/cpm15/Labels_png/'))), 
+                        len(os.listdir(os.path.join(self.data_directory, 'CPM_15_and_17/cpm17/test/Labels_png/'))), 
+                        len(os.listdir(os.path.join(self.data_directory, 'CPM_15_and_17/cpm17/train/Labels_png/')))]
+        return self.cpm_len[0] + self.cpm_len[1] + self.cpm_len[2]  
+    
+    def len_crag(self):
+        self.crag_len = [len(os.listdir(os.path.join(self.data_directory, 'CRAG/cell_CRAG/train2017/labels/'))),
+                        len(os.listdir(os.path.join(self.data_directory, 'CRAG/cell_CRAG/val2017/labels/')))]
+        return self.crag_len[0] + self.crag_len[1]
+    
+    def len_cryonuseg(self):
+        return 30
+
+    def len_glas(self):
+        return 60 + 20 + 85
+
+    def len_icia2018(self):
+        return 10
+
+    def len_janowczyk(self):
+        return 141
+    
+    def len_kpi(self):
+        if self.cluster != "denbi":
+            data_directories = [os.path.join(self.data_directory, 'KPI/', 'Task1_patch_level/data'), os.path.join(self.data_directory, 'KPI/', 'val/Task1_patch_level/data')]
+        else:
+            data_directories = [os.path.join(self.data_directory, 'KPI/', 'KPIs24 Training Data/Task1_patch_level/data'), os.path.join(self.data_directory, 'KPI/', 'KPIs24 Validation Data/Task1_patch_level/data')]
+        arr_length = 2
+        for (i0, data_directory) in enumerate(data_directories):
+            dir_list = sorted(os.listdir(data_directory))
+            arr_length += len(dir_list)
+            for (i1, d1) in enumerate(dir_list):
+                subdir_list = sorted(os.listdir(os.path.join(data_directory, d1)))
+                arr_length += len(subdir_list)
+        pointer = np.zeros(arr_length, dtype=int)
+        length = np.zeros(arr_length, dtype=int)
+        pointer[0] = len(data_directories)
+        len_last_dir_list = 0
+        len_last_subdir_list = 0
+        for (i0, data_directory) in enumerate(data_directories):
+            dir_list = sorted(os.listdir(data_directory))
+            len_last_dir_list = len(dir_list)
+            pointer[pointer[i0]] = pointer[i0] + len_last_dir_list
+            for (i1, d1) in enumerate(dir_list):
+                subdir_list = sorted(os.listdir(os.path.join(data_directory, d1)))
+                len_last_subdir_list = len(subdir_list)
+                for (i2, d2) in enumerate(subdir_list):
+                    file_list = os.listdir(os.path.join(data_directory, d1, d2,'img'))
+                    length[pointer[pointer[i0] + i1] + i2] = len(file_list)
+                    length[pointer[i0] + i1] += len(file_list)
+                    length[i0] += len(file_list)
+                if i1 < len(dir_list) - 1:
+                    pointer[pointer[i0] + i1 + 1] = pointer[pointer[i0] + i1] + len(subdir_list)
+                
+            if i0 < len(data_directories) - 1:
+                pointer[i0 + 1] = pointer[pointer[i0] + len_last_dir_list - 1] + len_last_subdir_list
+        self.pointer = pointer
+        self.length = length
+        # pointer = array([ 2, 36,  6, 11, 16, 31,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0, 0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0, 0,  0, 40, 42, 44, 46,  0,  0,  0,  0,  0,  0,  0,  0])
+        # length = array([5214, 1643,  558,  607, 2263, 1786,   92,   96,  141,   86,  143, 135,  134,   71,  129,  138,  143,  187,  166,  146,  133,  111, 156,  177,  152,  147,  150,   93,  156,  218,  128,  358,  390, 313,  370,  355,  274,  299,  209,  861,  144,  130,  182,  117, 106,  103,  415,  446])
+        return self.length[0] + self.length[1]
+
+    def len_kumar(self):
+        self.kumar_len = [len(os.listdir(os.path.join(self.data_directory, 'Kumar', 'train/Labels_png'))),
+                            len(os.listdir(os.path.join(self.data_directory, 'Kumar', 'test_same/Labels_png'))),
+                            len(os.listdir(os.path.join(self.data_directory, 'Kumar', 'test_diff/Labels_png')))]
+        return self.kumar_len[0] + self.kumar_len[1] + self.kumar_len[2]
+    
+    def len_monusac(self):
+        return len(os.listdir(os.path.join(self.data_directory, 'MoNuSAC/masks/')))
+
+    def len_monuseg(self):
+        return 37
+
+    def len_nuclick(self):
+        return 1213 + 250 + 462 + 150
+
+    def len_paip2023(self):
+        return 50 + 50 + 3 + 3
+
+    def len_pannuke(self):
+        return 2656 #7466 
+    
+    def len_segpath(self):
+        return 10647 + 26509 + 24805 + 12273 + 14135 + 13231 + 25909 + 31178
+    
+    def len_segpc(self):
+        return 298 + 199
+
+    def len_tiger(self):
+        return 1879
+
+    def len_tnbc(self):
+        return 7 + 3 + 5 + 8 + 4 + 3 + 3 + 4 + 6 + 4 + 3
+
+    def len_wsss4luad(self):
+        return 40
+        
+    def get_cpm(self, idx):
+        data_directory = os.path.join(self.data_directory, 'CPM_15_and_17/')
+        if idx < self.cpm_len[0]:
+            mask_names = os.listdir(data_directory + 'cpm15/Labels_png/')
+            image_name = data_directory + 'cpm15/Images/' + mask_names[idx][:8] + ".png"
+            mask_name = data_directory + 'cpm15/Labels_png/' + mask_names[idx]
+        elif idx < self.cpm_len[0] + self.cpm_len[1]:
+            mask_names = os.listdir(data_directory + 'cpm17/test/Labels_png/')
+            image_name = data_directory + 'cpm17/test/Images/' + mask_names[idx-self.cpm_len[0]][:8] + ".png"
+            mask_name = data_directory + 'cpm17/test/Labels_png/' + mask_names[idx-self.cpm_len[0]]
+        else:
+            mask_names = os.listdir(data_directory + 'cpm17/train/Labels_png/')
+            image_name = data_directory + 'cpm17/train/Images/' + mask_names[idx-self.cpm_len[0]-self.cpm_len[1]][:8] + ".png"
+            mask_name = data_directory + 'cpm17/train/Labels_png/' + mask_names[idx-self.cpm_len[0]-self.cpm_len[1]]
+        return image_name, mask_name  
+    
+    def get_bcss(self, idx):
+        data_directory = os.path.join(self.data_directory, 'BCSS/0_Public-data-Amgad2019_0.25MPP/')
+        if self.cluster != "denbi":
+            mask_names = os.listdir(data_directory + 'masks/')
+            image_name = data_directory + 'rgbs_colorNormalized/' + mask_names[idx]
+            mask_name = data_directory + 'masks/' + mask_names[idx]
+        else:
+            image_names = os.listdir(data_directory + 'images/')
+            image_name = data_directory + 'images/' + image_names[idx]
+            mask_name = data_directory + 'masks/' + image_names[idx]
+        return image_name, mask_name
+        
+    
+    def get_camelyon(self, idx):
+        if idx < self.cam_len[0]:
+            data_directory = os.path.join(self.data_directory, 'CAMELYON/CAMELYON16/')
+        else:
+            data_directory = os.path.join(self.data_directory, 'CAMELYON/CAMELYON17/')
+            idx -= self.cam_len[0]
+        image_names = os.listdir(data_directory + 'images/')
+        image_name = data_directory + 'images/' + image_names[idx]
+        mask_name = data_directory + 'masks/' + image_names[idx][:-4] + '_mask.tif'
+        return image_name, mask_name
+
+    def get_cellseg(self, idx):
+        data_directory = os.path.join(self.data_directory, 'CellSeg/NeurIPS22-CellSeg/')
+        if idx < self.cellseg_len[0]:
+            image_names = os.listdir(data_directory + 'Testing/Hidden/images/')
+            image_name = data_directory + 'Testing/Hidden/images/' + image_names[idx]
+            mask_name = data_directory + 'Testing/Hidden/osilab_seg/' + image_names[idx][:14] + '_label.tiff'
+        elif idx < self.cellseg_len[0] + self.cellseg_len[1]:
+            image_names = os.listdir(data_directory + 'Testing/Public/images/')
+            image_name = data_directory + 'Testing/Public/images/' + image_names[idx-self.cellseg_len[0]]
+            mask_name = data_directory + 'Testing/Public/labels/' + image_names[idx-self.cellseg_len[0]][:12] + '_label.tiff'
+        elif idx < self.cellseg_len[0] + self.cellseg_len[1] + self.cellseg_len[2]:
+            image_names = os.listdir(data_directory + 'Training/images/')
+            image_name = data_directory + 'Training/images/' + image_names[idx-self.cellseg_len[0]-self.cellseg_len[1]]
+            mask_name = data_directory + 'Training/labels/' + image_names[idx-self.cellseg_len[0]-self.cellseg_len[1]][:10] + '_label.tiff'
+        else:
+            image_names = os.listdir(data_directory + 'Tuning/images/')
+            image_name = data_directory + 'Tuning/images/' + image_names[idx-self.cellseg_len[0]-self.cellseg_len[1]-self.cellseg_len[2]]
+            mask_name = data_directory + 'Tuning/labels/' + image_names[idx-self.cellseg_len[0]-self.cellseg_len[1]-self.cellseg_len[2]][:10] + '_label.tiff'
+        return image_name, mask_name
+
+    def get_cocahis(self, idx):
+        data_directory = os.path.join(self.data_directory, 'CoCaHis/')
+        #image_names = os.listdir(data_directory + 'images/')
+        image_name = data_directory + 'images/' + "HE_raw_" + str(idx) + ".png"
+        if self.cluster != "denbi":
+            mask_name = data_directory + 'GT/' + "GT_GT_majority_vote_" + str(idx) + ".png"
+        else:
+            mask_name = data_directory + 'labels/' + 'GT_GT_majority_vote_' + str(idx) + ".png"
+        return image_name, mask_name
+
+    def get_conic(self, idx):
+        data_directory = os.path.join(self.data_directory, 'CoNIC/')
+        mask_names = os.listdir(data_directory + 'labels_png/')
+        image_name = data_directory + 'images_png/' + mask_names[idx][:4] + ".png"
+        mask_name = data_directory + 'labels_png/' + mask_names[idx]
+        return image_name, mask_name
+
+    def get_crag(self, idx):
+        data_directory = os.path.join(self.data_directory, "CRAG/cell_CRAG/")
+        if idx < self.crag_len[0]:
+            mask_names = os.listdir(data_directory + 'train2017/labels/')
+            image_name = data_directory + 'train2017/' + mask_names[idx]
+            mask_name = data_directory + 'train2017/labels/' + mask_names[idx]
+        else:
+            mask_names = os.listdir(data_directory + 'val2017/labels/')
+            image_name = data_directory + 'val2017/' + mask_names[idx-self.crag_len[0]]
+            mask_name = data_directory + 'val2017/labels/' + mask_names[idx-self.crag_len[0]]
+        return image_name, mask_name
+    
+    def get_cryonuseg(self, idx):
+        data_directory = os.path.join(self.data_directory, 'CryoNuSeg/')
+        image_names = os.listdir(data_directory + 'tissue images/')
+        image_name = data_directory + 'tissue images/' + image_names[idx]
+        mask_name = data_directory + 'Annotator 1 (biologist second round of manual marks up)/Annotator 1 (biologist second round of manual marks up)/label masks modify/' + image_names[idx]
+        return image_name, mask_name
+
+    def get_glas(self, idx):
+        data_directory = os.path.join(self.data_directory, 'GlaS/Warwick_QU_Dataset/')
+        if idx < 60:
+            image_name = data_directory + 'testA_' + str(idx + 1) + '.bmp'
+            mask_name = data_directory + 'testA_' + str(idx + 1) + '_anno.bmp'
+        elif idx < 60 + 20:
+            image_name = data_directory + 'testB_' + str(idx - 59) + '.bmp'
+            mask_name = data_directory + 'testB_' + str (idx - 59) + '_anno.bmp'
+        else:
+            image_name = data_directory + 'train_' + str(idx - 79) + '.bmp'
+            mask_name = data_directory + 'train_' + str(idx - 79) + '_anno.bmp'
+        return image_name, mask_name
+
+    def get_icia2018(self, idx):
+        data_directory = os.path.join(self.data_directory, 'ICIA2018/ICIAR2018_BACH_Challenge/WSI/')
+        image_name = data_directory + 'A' + str(idx + 1).zfill(2) + '.svs'
+        mask_name = data_directory + 'A' + str(idx + 1).zfill(2) + '.npy'
+        return image_name, mask_name
+
+    def get_janowczyk(self, idx):
+        data_directory = os.path.join(self.data_directory, 'Janowczyk/')
+        if self.cluster != "denbi":
+            names = os.listdir(data_directory)
+            image_names = [n for n in names if "original" in n]
+            image_names.sort()
+            image_name = data_directory + image_names[idx]
+            mask_name = data_directory + image_names[idx][:-12] + 'mask.png'
+        else:
+            image_names = os.listdir(data_directory + "images/")
+            image_names.sort()
+            image_name = data_directory + "images/" + image_names[idx]
+            mask_name = data_directory + "masks/" + image_names[idx][:-12] + 'mask.png'
+        return image_name, mask_name
+    
+    def get_kpi(self, idx):
+        if self.cluster != "denbi":
+            data_directories = [os.path.join(self.data_directory, 'KPI/', 'Task1_patch_level/data'), os.path.join(self.data_directory, 'KPI/', 'val/Task1_patch_level/data')]
+        else:
+            data_directories = [os.path.join(self.data_directory, 'KPI/', "KPIs24 Training Data/Task1_patch_level/data"), os.path.join(self.data_directory, 'KPI/', "KPIs24 Validation Data/Task1_patch_level/data")]
+        position = 0
+        if idx < self.length[0]:
+            data_directory = data_directories[0]
+            position = self.pointer[0]
+        else:
+            data_directory = data_directories[1]
+            idx -= self.length[0]
+            position = self.pointer[1]
+        dir_list = sorted(os.listdir(data_directory))
+        for (i1, d1) in enumerate(dir_list):
+            if idx < self.length[position + i1]:
+                position = self.pointer[position + i1]
+                subdir_list = sorted(os.listdir(os.path.join(data_directory, d1)))
+                for (i2, d2) in enumerate(subdir_list):
+                    if idx < self.length[position + i2]:
+                        file_list = sorted(os.listdir(os.path.join(data_directory, d1, d2,'img')))
+                        image_name = os.path.join(data_directory, d1, d2, 'img', file_list[idx])
+                        mask_name = os.path.join(data_directory, d1, d2, 'mask', file_list[idx].replace('img', 'mask'))
+                        return image_name, mask_name
+                    else:
+                        idx -= self.length[position + i2]          
+            else:
+                idx -= self.length[position + i1]
+
+    def get_kumar(self, idx):
+        data_directory = os.path.join(self.data_directory, 'Kumar/')
+        if idx < self.kumar_len[0]:
+            mask_names = os.listdir(data_directory + 'train/Labels_png/')
+            image_name = data_directory + 'train/Images/' + mask_names[idx][:23] + ".tif"
+            mask_name = data_directory + 'train/Labels_png/' + mask_names[idx]
+        elif idx < self.kumar_len[0] + self.kumar_len[1]:
+            mask_names = os.listdir(data_directory + 'test_same/Labels_png/')
+            image_name = data_directory + 'test_same/Images/' + mask_names[idx-self.kumar_len[0]][:23] + ".tif"
+            mask_name = data_directory + 'test_same/Labels_png/' + mask_names[idx-self.kumar_len[0]]
+        else:
+            mask_names = os.listdir(data_directory + 'test_diff/Labels_png/')
+            image_name = data_directory + 'test_diff/Images/' + mask_names[idx-self.kumar_len[0]-self.kumar_len[1]][:23] + ".tif"
+            mask_name = data_directory + 'test_diff/Labels_png/' + mask_names[idx-self.kumar_len[0]-self.kumar_len[1]]
+        return image_name, mask_name
+
+    def get_monusac(self, idx):
+        data_directory = os.path.join(self.data_directory, 'MoNuSAC/')
+        mask_names = os.listdir(data_directory + 'masks/')
+        mask_name = data_directory + 'masks/' + mask_names[idx]
+        types = ["Epithelial", "Lymphocyte", "Macrophage", "Neutrophil"]
+        image_name = data_directory + 'images/' + mask_names[idx]
+        for t in types:
+            if mask_names[idx].endswith(t + '.png'):
+                image_name = data_directory + 'images/' + mask_names[idx][:-len(t)-5]
+                if os.path.exists(image_name + '.tif'):
+                    image_name += '.tif'
+                else:
+                    image_name += '.png'
+                break       
+        return image_name, mask_name
+
+    def get_monuseg(self, idx):
+        data_directory = os.path.join(self.data_directory, 'MoNuSeg/MoNuSeg 2018 Training Data/')
+        mask_names = os.listdir(data_directory + 'Masks/')
+        mask_name = data_directory + 'Masks/' + mask_names[idx]
+        image_name = data_directory + 'Tissue Images/' + mask_names[idx][:23] + '.tif'
+        return image_name, mask_name
+
+    def get_nuclick(self, idx):
+        data_directory = os.path.join(self.data_directory, 'NuClick/')
+        if idx < 1213:
+            mask_names = os.listdir(data_directory + 'Hemato_Data/Train/masks/')
+            image_name = data_directory + 'Hemato_Data/Train/images/' + mask_names[idx][:-9] + ".png"
+            mask_name = data_directory + 'Hemato_Data/Train/masks/' + mask_names[idx]
+        elif idx < 1213 + 250:
+            mask_names = os.listdir(data_directory + 'Hemato_Data/Validation/masks/')
+            image_name = data_directory + 'Hemato_Data/Validation/images/' + mask_names[idx-1213][:-9] + ".png"
+            mask_name = data_directory + 'Hemato_Data/Validation/masks/' + mask_names[idx-1213]
+        elif idx < 1213 + 250 + 462:
+            mask_names = os.listdir(data_directory + 'IHC_nuclick/IHC/masks_png/Train/')
+            image_name = data_directory + 'IHC_nuclick/IHC/images/Train/' + mask_names[idx-1213-250]
+            mask_name = data_directory + 'IHC_nuclick/IHC/masks_png/Train/' + mask_names[idx-1213-250]
+        else:
+            mask_names = os.listdir(data_directory + 'IHC_nuclick/IHC/masks_png/Validation/')
+            image_name = data_directory + 'IHC_nuclick/IHC/images/Validation/' + mask_names[idx-1213-250-462]
+            mask_name = data_directory + 'IHC_nuclick/IHC/masks_png/Validation/' + mask_names[idx-1213-250-462]
+        return image_name, mask_name
+
+    def get_paip2023(self, idx):
+        data_directory = os.path.join(self.data_directory, 'PAIP2023/')
+        if idx < 50:
+            image_name = data_directory + 'tr_p' + str(idx + 1).zfill(3) + '.png'
+            mask_name = data_directory + "non_tumor/" + 'tr_p' + str(idx + 1).zfill(3) + '_nontumor.png'
+        elif idx < 50 + 50:
+            image_name = data_directory + 'tr_p' + str(idx - 50 + 1).zfill(3) + '.png'
+            mask_name = data_directory + "tumor/" + 'tr_p' + str(idx - 50 + 1).zfill(3) + '_tumor.png'
+        elif idx < 50 + 50 + 3:
+            image_name = data_directory + 'tr_c' + str(idx - 100 + 1).zfill(3) + '.png'
+            mask_name = data_directory + "non_tumor/" + 'tr_c' + str(idx -100 + 1).zfill(3) + '_nontumor.png'
+        else:
+            image_name = data_directory + 'tr_c' + str(idx - 103 + 1).zfill(3) + '.png'
+            mask_name = data_directory + "tumor/" + 'tr_c' + str(idx -103 + 1).zfill(3) + '_tumor.png'
+        return image_name, mask_name
+
+    def get_pannuke(self, idx):
+        data_directory = os.path.join(self.data_directory, 'PanNuke/')
+        image_name = data_directory + 'images_png/' + str(idx).zfill(4) + ".png"
+        mask_name = data_directory + 'masks_png/' + str(idx).zfill(4) + ".png"
+        return image_name, mask_name
+
+    def get_segpath(self, idx):
+        data_directory = os.path.join(self.data_directory, 'SegPath/')
+        if idx < 10647:
+            image_and_mask_names = os.listdir(data_directory + 'endothelial_cells/ERG_Endothelium/')
+            image_names = [x for x in image_and_mask_names if "HE" in x]
+            image_name = data_directory + 'endothelial_cells/ERG_Endothelium/' + image_names[idx]
+            mask_name = data_directory + 'endothelial_cells/ERG_Endothelium/' + image_names[idx][:-6] + 'mask.png' 
+        elif idx < 10647 + 26509:
+            image_and_mask_names = os.listdir(data_directory + 'epithelial_cells/panCK_Epithelium/')
+            image_names = [x for x in image_and_mask_names if "HE" in x]
+            image_name = data_directory + 'epithelial_cells/panCK_Epithelium/' + image_names[idx-10647]
+            mask_name = data_directory + 'epithelial_cells/panCK_Epithelium/' + image_names[idx-10647][:-6] + 'mask.png'
+        elif idx < 10647 + 26509 + 24805:
+            image_and_mask_names = os.listdir(data_directory + 'leukocytes/CD45RB_Leukocyte/')
+            image_names = [x for x in image_and_mask_names if "HE" in x]
+            image_name = data_directory + 'leukocytes/CD45RB_Leukocyte/' + image_names[idx-10647-26509]
+            mask_name = data_directory + 'leukocytes/CD45RB_Leukocyte/' + image_names[idx-10647-26509][:-6] + 'mask.png'
+        elif idx < 10647 + 26509 + 24805 + 12273:
+            image_and_mask_names = os.listdir(data_directory + 'lymphocytes/CD3CD20_Lymphocyte/')
+            image_names = [x for x in image_and_mask_names if "HE" in x]
+            image_name = data_directory + 'lymphocytes/CD3CD20_Lymphocyte/' + image_names[idx-10647-26509-24805]
+            mask_name = data_directory + 'lymphocytes/CD3CD20_Lymphocyte/' + image_names[idx-10647-26509-24805][:-6] + 'mask.png'
+        elif idx < 10647 + 26509 + 24805 + 12273 + 14135:
+            image_and_mask_names = os.listdir(data_directory + 'myeloid_cells/MNDA_MyeloidCell/')
+            image_names = [x for x in image_and_mask_names if "HE" in x]
+            image_name = data_directory + 'myeloid_cells/MNDA_MyeloidCell/' + image_names[idx-10647-26509-24805-12273]
+            mask_name = data_directory + 'myeloid_cells/MNDA_MyeloidCell/' + image_names[idx-10647-26509-24805-12273][:-6] + 'mask.png'
+        elif idx < 10647 + 26509 + 24805 + 12273 + 14135 + 13231:
+            image_and_mask_names = os.listdir(data_directory + 'plasma_cells/MIST1_PlasmaCell/')
+            image_names = [x for x in image_and_mask_names if "HE" in x]
+            image_name = data_directory + 'plasma_cells/MIST1_PlasmaCell/' + image_names[idx-10647-26509-24805-12273-14135]
+            mask_name = data_directory + 'plasma_cells/MIST1_PlasmaCell/' + image_names[idx-10647-26509-24805-12273-14135][:-6] + 'mask.png'
+        elif idx < 10647 + 26509 + 24805 + 12273 + 14135 + 13231 + 25909:
+            image_and_mask_names = os.listdir(data_directory + 'red_blood_cells/CD235a_RBC/')
+            image_names = [x for x in image_and_mask_names if "HE" in x]
+            image_name = data_directory + 'red_blood_cells/CD235a_RBC/' + image_names[idx-10647-26509-24805-12273-14135-13231]
+            mask_name = data_directory + 'red_blood_cells/CD235a_RBC/' + image_names[idx-10647-26509-24805-12273-14135-13231][:-6] + 'mask.png'
+        else:
+            image_and_mask_names = os.listdir(data_directory + 'smooth_muscle_cells/aSMA_SmoothMuscle/')
+            image_names = [x for x in image_and_mask_names if "HE" in x]
+            image_name = data_directory + 'smooth_muscle_cells/aSMA_SmoothMuscle/' + image_names[idx-10647-26509-24805-12273-14135-13231-25909]
+            mask_name = data_directory + 'smooth_muscle_cells/aSMA_SmoothMuscle/' + image_names[idx-10647-26509-24805-12273-14135-13231-25909][:-6] + 'mask.png'
+        return image_name, mask_name
+
+    def get_segpc(self, idx):
+        data_directory = os.path.join(self.data_directory, 'SegPC/TCIA_SegPC_dataset/')
+        if idx < 298:
+            mask_names = os.listdir(data_directory + 'train/masks_png/')
+            image_name = data_directory + 'train/x/' + mask_names[idx][:-4] + ".bmp"
+            mask_name = data_directory + 'train/masks_png/' + mask_names[idx]
+        else:
+            mask_names = os.listdir(data_directory + 'validation/masks_png/')
+            image_name = data_directory + 'validation/x/' + mask_names[idx-298][:-4] + ".bmp"
+            mask_name = data_directory + 'validation/masks_png/' + mask_names[idx-298]
+        return image_name, mask_name
+
+    def get_tiger(self, idx):
+        data_directory = os.path.join(self.data_directory, 'TIGER/wsirois/roi-level-annotations/tissue-cells/')
+        image_names = os.listdir(data_directory + 'images/')
+        image_name = data_directory + 'images/' + image_names[idx]
+        mask_name = data_directory + 'masks/' + image_names[idx]
+        return image_name, mask_name
+
+    def get_tnbc(self, idx):
+        if self.cluster == "denbi":
+            data_directory = os.path.join(self.data_directory, 'TNBC/TNBC_NucleiSegmentation/')
+        else: 
+            data_directory = os.path.join(self.data_directory, 'TNBC/TNBC_dataset/')
+        bucket = 1
+        idx = idx + 1
+        if idx > 7:
+            bucket += 1
+            idx -= 7
+            if idx > 3:
+                bucket += 1
+                idx -= 3
+                if idx > 5:
+                    bucket += 1
+                    idx -= 5
+                    if idx > 8:
+                        bucket += 1
+                        idx -= 8
+                        if idx > 4:
+                            bucket += 1
+                            idx -= 4
+                            if idx > 3:
+                                bucket += 1
+                                idx -= 3
+                                if idx > 3:
+                                    bucket += 1
+                                    idx -= 3
+                                    if idx > 4:
+                                        bucket += 1
+                                        idx -= 4
+                                        if idx > 6:
+                                            bucket += 1
+                                            idx -= 6
+                                            if idx > 4:
+                                                bucket += 1
+                                                idx -= 4
+        image_name = data_directory + 'Slide_' + str(bucket).zfill(2) + '/' +  str(bucket).zfill(2) + '_' + str(idx) + '.png'
+        mask_name = data_directory + 'GT_' + str(bucket).zfill(2) + '/' +  str(bucket).zfill(2) + '_' + str(idx) + '.png'
+        return image_name, mask_name

CellPilot/cellpilot/data_processing/data_processing.md

@@ -0,0 +1,33 @@
+# Data Processing
+
+This folder contains the following files:
+- `dataset.py`: The Pytorch dataset used for training our model
+- `data_fetching.py`: `len`and `get` methods for each dataset
+- `data_utils.py`: DataProcessing and PromptProcessing methods
+
+Detailed Description:
+## Dataset
+
+## Data Fetching
+`data_fetching.py` implements `len`and `get` methods for all used datasets. For this we assume our own file structure. More details in the section [Our File Structure](#our-file-structure).
+However you can easily implement `len` and `get` for your own dataset. For this have a look at the section [Using your own Dataset](#using-your-own-dataset).
+
+### Our File Structure
+All of our data is stored in one `data_directory`. Each dataset has a subdirectory in this data_directory. The individual datasets are stored as follows:
+
+#### BCSS
+```
+BCSS
+|--- 0_Public-data-Amgad2019_0.25MPP
+    |--- masks
+    |--- rgbs_colorNormalized
+```
+#### CellSeg
+```
+
+```
+
+### Using your own Dataset
+Entry in DATASET_DICT
+Implement len and get
+## Data Utils

CellPilot/cellpilot/data_processing/data_utils.py

@@ -0,0 +1,368 @@
+import numpy as np
+import slideio
+import torch
+from PIL import Image
+import torch.nn.functional as F
+from skimage.transform import resize as sk_resize
+import random
+from segment_anything.utils.transforms import ResizeLongestSide
+from scipy.ndimage import label, convolve
+from torchvision.transforms.functional import resize, to_pil_image  # type: ignore
+import tifffile
+import albumentations as A
+
+class DataProcessing:
+    
+    def preprocess(image_name, mask_name, pixel_mean, pixel_std, wsi_image, image_encoder_size, mask_augmentation_tries, data_augmentations, threshold_connected_components, coords=None):
+        image, gt, patch_coordinates, resized_size = DataProcessing.patch(image_name, mask_name, wsi_image, image_encoder_size=image_encoder_size, coords=coords)
+        if data_augmentations != ["NoOp"]:
+            image, gt = DataProcessing.augment(image, gt, data_augmentations, mask_augmentation_tries)
+        image = DataProcessing.preprocess_image(image, pixel_mean, pixel_std, image_encoder_size=image_encoder_size)
+        image = image.float()
+        gt = DataProcessing.preprocess_gt(gt, image_encoder_size=image_encoder_size)
+        gt = DataProcessing.connected_component_analysis(gt, threshold_connected_components)
+        nr_labels = torch.max(gt)
+        return image, gt, nr_labels, (image_name, mask_name), patch_coordinates, resized_size
+    
+    def augment(image, gt, data_augmentations, mask_augmentation_tries=5):
+        image_augmentation_dict = {
+            "AdvancedBlur": A.AdvancedBlur(),
+            "Blur": A.Blur(),
+            "GaussianBlur": A.GaussianBlur(),
+            "ZoomBlur": A.ZoomBlur(),
+            "CLAHE": A.CLAHE(),
+            "Emboss": A.Emboss(),
+            "GaussNoise": A.GaussNoise(),
+            "IsoNoise": A.ISONoise(),
+            "ImageCompression": A.ImageCompression(),
+            "Posterize": A.Posterize(),
+            "RingingOvershoot": A.RingingOvershoot(),
+            "Sharpen": A.Sharpen(),
+            "ToGray": A.ToGray(),
+            "Downscale": A.Downscale(scale_range=(0.5, 0.9), p=0.5),
+            "ChannelShuffle": A.ChannelShuffle(),
+            "ChromaticAberration": A.ChromaticAberration(),
+            "ColorJitter": A.ColorJitter(),
+            "HueSaturationValue": A.HueSaturationValue(),
+            "MultiplicativeNoise": A.MultiplicativeNoise(),
+            "PlanckianJitter": A.PlanckianJitter(),
+            "RGBShift": A.RGBShift(),
+            "RandomBrightnessContrast": A.RandomBrightnessContrast(),
+            "RandomGamma": A.RandomGamma(),
+            "RandomToneCurve": A.RandomToneCurve(),
+            "FancyPCA": A.FancyPCA(),
+        }
+        mask_augmentation_dict = {
+            "Affine": A.Affine(),
+            "CropNonEmptyMaskIfExists": A.CropNonEmptyMaskIfExists(512, 512, p=0.5),
+            "ElasticTransform": A.ElasticTransform(),
+            "GridDistortion": A.GridDistortion(),
+            "OpticalDistortion": A.OpticalDistortion(),
+            "RandomCrop": A.RandomCrop(512, 512, p=0.5),
+            "RandomGridShuffle": A.RandomGridShuffle(),
+            "RandomResizedCrop": A.RandomResizedCrop(size=(1024, 1024),p=0.5),
+            "Rotate": A.Rotate(),
+            "ShiftScaleRotate": A.ShiftScaleRotate(),
+            "CropAndPad": A.CropAndPad(px=10, p=0.5),
+            "D4": A.D4(p=0.5),
+            "PadIfNeeded": A.PadIfNeeded(p=0.5),
+            "Perspective": A.Perspective(),
+            "RandomScale": A.RandomScale(),
+        }
+        image_augmentations = [image_augmentation_dict[da] for da in data_augmentations if da in image_augmentation_dict]
+        mask_augmentations = [mask_augmentation_dict[da] for da in data_augmentations if da in mask_augmentation_dict]
+        image_transform = A.Compose(image_augmentations)
+        mask_transform = A.Compose(mask_augmentations)
+        transformed = image_transform(image=image)
+        image = transformed["image"]
+        for i in range(mask_augmentation_tries):
+            transformed = mask_transform(image=image, mask=gt)
+            if np.unique(transformed["mask"]).shape[0] > 1:
+                image = transformed["image"]
+                gt = transformed["mask"]
+                break
+        return image, gt
+
+    def patch(image_name, gt_name, wsi_image=False, image_encoder_size=1024, coords=None):
+        if wsi_image:
+            image = slideio.open_slide(image_name)
+            image_scene = image.get_scene(0)
+            if gt_name.endswith(".npy"):
+                gt = np.load(gt_name).transpose()
+            else:
+                gt = slideio.open_slide(gt_name)
+                gt_scene = gt.get_scene(0)
+            h, w = image_scene.size
+        else: 
+            if image_name.endswith(".tiff") or image_name.endswith(".tif"):
+                image = tifffile.imread(image_name)
+            else:
+                image = Image.open(image_name)
+            if gt_name.endswith(".tiff") or gt_name.endswith(".tif"):
+                gt = tifffile.imread(gt_name)
+            else:
+                gt = Image.open(gt_name)
+            image = np.array(image)
+            gt = np.array(gt)
+            h, w = image.shape[:2]
+
+        def random_patches(h, w):
+            if coords is not None:
+                left, right, upper, lower = coords
+                return left, upper, right, lower
+            else:
+                left = random.randint(0, max(0, h - image_encoder_size))
+                upper = random.randint(0, max(0, w - image_encoder_size))
+                right = random.randint(min(h,left + image_encoder_size), h)
+                lower = random.randint(min(w, upper + image_encoder_size), w)
+                return left, upper, right, lower
+        
+        def grid_patches(i, h, w):
+            left = i % ((h // image_encoder_size) + 1) * image_encoder_size
+            upper = i // ((h // image_encoder_size) + 1) * image_encoder_size
+            right = min(h, left + image_encoder_size)
+            lower = min(w, upper + image_encoder_size)
+            return left, upper, right, lower
+        
+        nr_of_random_samples = 10
+        i = 0
+        while True:
+            if i < nr_of_random_samples:
+                left, upper, right, lower = random_patches(h, w)
+            else:
+                left, upper, right, lower = grid_patches(i - nr_of_random_samples, h, w)
+            i += 1
+
+            new_h, new_w = ResizeLongestSide.get_preprocess_shape(right - left, lower - upper, image_encoder_size)
+            if wsi_image:
+                image_resized = image_scene.read_block((left, upper, right-left, lower-upper), (new_h, new_w))
+                if gt_name.endswith(".npy"):
+                    gt_cropped = gt[left:right, upper:lower].astype(np.uint8)
+                    gt_resized = sk_resize(gt_cropped, (new_h,new_w), preserve_range=True, order = 0)
+                else:
+                    gt_resized = gt_scene.read_block((left, upper, right-left, lower-upper), (new_h, new_w))
+            else:
+                image_cropped = image[left:right, upper:lower]
+                try:
+                    if np.max(image_cropped) > 255:
+                        image_cropped = (255/np.max(image_cropped)) * image_cropped
+                except:
+                    pass
+                image_resized = np.array(resize(to_pil_image(image_cropped.astype(np.uint8)), (new_h, new_w)))
+                gt_cropped = gt[left:right, upper:lower].astype(np.uint8)
+                gt_resized = sk_resize(gt_cropped, (new_h,new_w), preserve_range=True, order = 0)
+
+            if np.unique(gt_resized).shape[0] > 1:
+                return image_resized, gt_resized, (left, upper, right, lower), (new_h, new_w)
+
+    def preprocess_image(x, pixel_mean, pixel_std, image_encoder_size=1024):
+        """Normalize pixel values and pad to a square input."""
+        # Normalize colors
+        if len(x.shape) == 2:
+            x = np.repeat(x[:, :, np.newaxis], 3, axis=2)
+        if x.shape[2] == 4:
+            x = x[:, :, :3]
+        x = x.transpose((2,0,1))
+        x = torch.tensor(x)
+        x = (x - pixel_mean) / pixel_std
+        # Pad
+        h, w = x.shape[-2:]
+        padh = image_encoder_size - h
+        padw = image_encoder_size - w
+        x = F.pad(x, (0, padw, 0, padh))     
+        return x
+    
+    def preprocess_gt(x, image_encoder_size=1024):
+        """Pad to a square input."""
+        # Pad
+        h, w = x.shape[-2:]
+        padh = image_encoder_size - h
+        padw = image_encoder_size - w
+        x = torch.tensor(x)
+        x = F.pad(x, (0, padw, 0, padh))
+        return x
+    
+    def connected_component_analysis(gt, threshold):
+        structure = np.ones((3, 3), dtype=np.int32)
+        mask_values= np.unique(gt)
+        mask_values= mask_values[1:]
+        counter = 0
+        cca_gt = np.zeros_like(gt, dtype=np.int32)
+        for v in mask_values:
+            binary_gt_mask = np.where(gt == v, 1.0, 0.0)
+            labeled_gt_mask, ncomponents = label(binary_gt_mask, structure)
+            counts = np.bincount(labeled_gt_mask.flatten())[1:]
+            j = 0
+            for (i, c) in enumerate(counts):
+                if c < threshold:
+                    labeled_gt_mask = np.where(labeled_gt_mask == i + 1, 0, labeled_gt_mask)
+                else:
+                    j += 1
+                    labeled_gt_mask = np.where(labeled_gt_mask == i + 1, j, labeled_gt_mask)
+            labeled_gt_mask = np.where(labeled_gt_mask > 0, labeled_gt_mask+counter, 0)
+            counter += j
+            cca_gt += labeled_gt_mask
+        cca_gt = torch.tensor(cca_gt)
+        return cca_gt
+
+    def unconnected_component_analysis(gt):
+        mask_values= np.unique(gt)
+        mask_values= mask_values[1:]
+        uca_gt = np.zeros_like(gt, dtype=np.int32)
+        for (i, v) in enumerate(mask_values):
+            uca_gt = np.where(gt == v, i+1, uca_gt)
+        uca_gt = torch.tensor(uca_gt)
+        return uca_gt
+
+
+
+class PromptProcessing: 
+
+    @staticmethod
+    def get_prompts_and_targets(nr, target, device, prompt_config):
+        "Get prompts to be used in the model"
+        prompt_batch_size = prompt_config["prompt_batch_size"]
+        prompt_type = prompt_config["prompt_type"]
+        nr_of_points = prompt_config["nr_of_points"]
+        nr_of_pos_points = prompt_config["nr_of_positive_points"]
+        bbox_shift = prompt_config["bbox_shift"]
+        components = [[random.randint(1, nr[j]) for i in range(prompt_batch_size)] for j in range(len(nr))]
+        targets = []
+        target_nr = []
+        for i in range(len(components)):
+            for j in range(len(components[i])):
+                targets.append(torch.where(target[i] == components[i][j], 1, 0)) 
+                target_nr.append(components[i][j])
+        target = torch.stack(targets, dim=0)
+
+        if prompt_type == "both" or prompt_type == "points":
+            nr_of_points_per_component = [nr_of_points for j in range(len(components))]
+            nr_of_pos_points_per_component = [nr_of_pos_points for j in range(len(components))]
+            prompts = PromptProcessing.get_point_prompts(target, nr, prompt_batch_size, nr_of_points_per_component, nr_of_pos_points_per_component, device)
+        else:
+            prompts = PromptProcessing.get_box_prompts(target, components, device, bbox_shift)
+        if prompt_type == "both":
+            box_prompts = PromptProcessing.get_box_prompts(target, components, device, bbox_shift)
+            prompts = prompts + box_prompts
+            target = torch.cat((target, target), 0)
+        return prompts, target, target_nr
+    
+    @staticmethod
+    def get_point_prompts(target, nr, prompt_batch_size, nr_of_points, nr_of_pos_points, device):
+        prompts = []
+        idx = 0
+        for i in range(len(nr)):
+            prompt = {}
+            point_coords = torch.zeros(prompt_batch_size, nr_of_points[i], 2)
+            point_labels = torch.ones(prompt_batch_size, nr_of_points[i]) 
+            point_labels[:, nr_of_pos_points[i]:] = 0
+            for j in range(prompt_batch_size):
+                x_indices, y_indices  = PromptProcessing.filter_out_edge(target[idx])
+                for k in range(nr_of_pos_points[i]):
+                    rand_idx = random.randrange(0, len(x_indices), 1)
+                    point_coords[j, k, 0] = y_indices[rand_idx]
+                    point_coords[j, k, 1] = x_indices[rand_idx]
+                x_indices, y_indices = PromptProcessing.filter_out_edge(1-target[idx])
+                for k in range(nr_of_points[i] - nr_of_pos_points[i]):
+                    rand_idx = random.randrange(0, len(x_indices), 1)
+                    point_coords[j, k + nr_of_pos_points[i], 0] = y_indices[rand_idx]
+                    point_coords[j, k + nr_of_pos_points[i], 1] = x_indices[rand_idx]
+                idx += 1
+            point_coords, point_labels = point_coords.to(device), point_labels.to(device)
+            prompt.update({
+                "point_coords": point_coords,
+                "point_labels": point_labels,
+            })
+            prompts.append(prompt)
+        return prompts
+
+    @staticmethod
+    def filter_out_edge(target):
+        kernel = np.ones((3,3))
+        target_np = target.cpu().numpy()
+        inside = convolve(target_np, kernel, mode='constant', cval=0.0)
+        if np.any(inside == 9):
+            return np.where(inside == 9)
+        else: 
+            return np.where(target_np == 1)
+
+    @staticmethod
+    def get_box_prompts(target, components, device, bbox_shift):
+        prompts = []
+        idx = 0    
+        for i in range(len(components)):
+            prompt = {}
+            bboxes = torch.zeros(len(components[i]), 4)
+            for j in range(len(components[i])):
+                y_indices, x_indices = torch.where(target[idx] == 1)
+                x_min, x_max = torch.min(x_indices), torch.max(x_indices)
+                y_min, y_max = torch.min(y_indices), torch.max(y_indices)
+                # add perturbation to bounding box coordinates
+                _,H, W = target.shape
+                x_min = max(0, x_min - random.randint(0, bbox_shift))
+                x_max = min(W, x_max + random.randint(0, bbox_shift))
+                y_min = max(0, y_min - random.randint(0, bbox_shift))
+                y_max = min(H, y_max + random.randint(0, bbox_shift))
+                bboxes[j,0] = x_min
+                bboxes[j,1] = y_min
+                bboxes[j,2] = x_max
+                bboxes[j,3] = y_max
+                idx += 1
+            bboxes = bboxes.to(device) 
+            prompt["boxes"] = bboxes
+            prompts.append(prompt)
+        return prompts
+
+    @staticmethod
+    def postprocess_masks(masks, input_size=(1024,1024), original_size=(1024,1024)):
+        masks = F.interpolate(
+            masks,
+            (1024, 1024),
+            mode="bilinear",
+            align_corners=False,
+        )
+        masks = masks[..., : input_size[0], : input_size[1]]
+        masks = F.interpolate(masks, original_size, mode="bilinear", align_corners=False)
+        return masks
+
+    @staticmethod
+    def refine_prompts(nr, target, previous_prompts, previous_prediction, device, prompt_batch_size):
+        binary_prediction = (previous_prediction > 0).float()
+        diff = target.unsqueeze(1) - binary_prediction
+        pos_diff = diff > 0
+        neg_diff = diff < 0
+        structure = np.ones((3, 3), dtype=np.int32)
+        
+        for i in range(len(previous_prompts)):
+            prompt_list = []
+            prompt_label_list = []
+            for j in range(prompt_batch_size):
+                conn_comp_pos, threshold = label(pos_diff[prompt_batch_size * i + j][0].cpu().numpy(), structure)
+                conn_comp_neg = label(neg_diff[prompt_batch_size * i + j][0].cpu().numpy(), structure)[0]
+                conn_comp = conn_comp_pos + np.where(conn_comp_neg > 0, conn_comp_neg + threshold, 0)
+                component_size = np.bincount(conn_comp.flatten())[1:]
+                if "point_coords" in previous_prompts[i]:
+                    prompt_list.append(previous_prompts[i]["point_coords"][j])
+                    prompt_label_list.append(previous_prompts[i]["point_labels"][j])
+                if component_size.size == 0:
+                    max_indices = [0] 
+                else:
+                    max_indices = [np.argmax(component_size) + 1]
+                for m in max_indices:
+                    target_m = torch.tensor(np.where(conn_comp == m, 1, 0))
+                    if m == 0:
+                        label_m = torch.tensor([0]).float().to(device)
+                    else:
+                        label_m = torch.tensor([(m - 1 < threshold)]).float().to(device)
+                    prompts = PromptProcessing.get_point_prompts(target_m.unsqueeze(0), [1], 1, [1], [1], device)
+                    if "point_coords" in previous_prompts[i]:
+                        prompt_list[j] = torch.cat((prompt_list[j], prompts[0]["point_coords"][0]), 0)
+                        prompt_label_list[j] = torch.cat((prompt_label_list[j], label_m), 0)
+                    else: 
+                        prompt_list.append(prompts[0]["point_coords"][0])
+                        prompt_label_list.append(label_m)
+            prompt_stack = torch.stack(prompt_list, dim=0)
+            prompt_label_stack = torch.stack(prompt_label_list, dim=0)
+            previous_prompts[i]["point_coords"] = prompt_stack
+            previous_prompts[i]["point_labels"] = prompt_label_stack
+        return previous_prompts

CellPilot/cellpilot/data_processing/dataset.py

@@ -0,0 +1,110 @@
+from torch.utils.data import Dataset, DataLoader, random_split
+import torch
+from segment_anything.utils.transforms import ResizeLongestSide
+from .data_fetching import DataFetcher
+from .data_utils import DataProcessing
+
+class HistologyDataset(Dataset):
+    def __init__(self, datasets, data_directory, cluster, image_encoder_size, mask_augmentation_tries, data_augmentations, threshold_connected_components):
+        self.datasets = datasets
+        self.data_fetcher = DataFetcher(data_directory, cluster)
+        self.image_encoder_size = image_encoder_size
+        self.mask_augmentation_tries = mask_augmentation_tries
+        self.data_augmentations = data_augmentations
+        self.threshold_connected_components = threshold_connected_components
+        self.pixel_mean = torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1)
+        self.pixel_std = torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1)
+        self.DATASET_DICT = {
+            "BCSS": [self.data_fetcher.len_bcss, self.data_fetcher.get_bcss, False, 0.0],
+            "CAMELYON": [self.data_fetcher.len_camelyon, self.data_fetcher.get_camelyon, True, 0.0],
+            "CellSeg": [self.data_fetcher.len_cellseg, self.data_fetcher.get_cellseg, False, 0.0],
+            "CoCaHis": [self.data_fetcher.len_cocahis, self.data_fetcher.get_cocahis, False, 0.0],
+            "CoNIC": [self.data_fetcher.len_conic, self.data_fetcher.get_conic, False, 0.0],
+            "CPM": [self.data_fetcher.len_cpm, self.data_fetcher.get_cpm, False, 0.0],
+            "CRAG": [self.data_fetcher.len_crag, self.data_fetcher.get_crag, False, 1.0],
+            "CryoNuSeg": [self.data_fetcher.len_cryonuseg, self.data_fetcher.get_cryonuseg, False, 0.0],
+            "GlaS": [self.data_fetcher.len_glas, self.data_fetcher.get_glas, False, 1.0], 
+            "ICIA2018": [self.data_fetcher.len_icia2018, self.data_fetcher.get_icia2018, True, 0.0],
+            "Janowczyk": [self.data_fetcher.len_janowczyk, self.data_fetcher.get_janowczyk, False, 0.0],
+            "KPI": [self.data_fetcher.len_kpi, self.data_fetcher.get_kpi, False, 0.0],
+            "Kumar": [self.data_fetcher.len_kumar, self.data_fetcher.get_kumar, False, 0.0],
+            "MoNuSAC": [self.data_fetcher.len_monusac, self.data_fetcher.get_monusac, False, 0.0],
+            "MoNuSeg": [self.data_fetcher.len_monuseg, self.data_fetcher.get_monuseg, False, 0.0],
+            "NuClick": [self.data_fetcher.len_nuclick, self.data_fetcher.get_nuclick, False, 0.0],
+            "PAIP2023": [self.data_fetcher.len_paip2023, self.data_fetcher.get_paip2023, False, 0.0],
+            "PanNuke": [self.data_fetcher.len_pannuke, self.data_fetcher.get_pannuke, False, 0.0],
+            "SegPath": [self.data_fetcher.len_segpath, self.data_fetcher.get_segpath, False, 0.0],
+            "SegPC": [self.data_fetcher.len_segpc, self.data_fetcher.get_segpc, False, 0.0],
+            "TIGER": [self.data_fetcher.len_tiger, self.data_fetcher.get_tiger, False, 0.0],
+            "TNBC": [self.data_fetcher.len_tnbc, self.data_fetcher.get_tnbc, False, 0.0],
+        }
+        self.len = 0
+        for dataset in self.datasets:
+            self.DATASET_DICT[dataset][0] = self.DATASET_DICT[dataset][0]()
+            self.len += self.DATASET_DICT[dataset][0]
+        self.resize = ResizeLongestSide(image_encoder_size)
+
+
+    def __len__(self):
+        return self.len
+
+    def __getitem__(self, idx):
+        for dataset in self.datasets:
+            if idx < self.DATASET_DICT[dataset][0]:
+                image_name, mask_name = self.DATASET_DICT[dataset][1](idx)
+                return idx, DataProcessing.preprocess(
+                    image_name, mask_name, self.pixel_mean, self.pixel_std, self.DATASET_DICT[dataset][2], 
+                    self.image_encoder_size, self.mask_augmentation_tries, self.data_augmentations, self.threshold_connected_components
+                    ), self.DATASET_DICT[dataset][3]
+            else:
+                idx -= self.DATASET_DICT[dataset][0]
+
+    def get_image(self, idx):
+        for dataset in self.datasets:
+            if idx < self.DATASET_DICT[dataset][0]:
+                image_name, mask_name = self.DATASET_DICT[dataset][1](idx)
+                return image_name, mask_name
+            else:
+                idx -= self.DATASET_DICT[dataset][0]
+
+def prepare_data(data_config):
+    """
+    
+    """
+    datasets = data_config["datasets"]
+    data_directory = data_config["data_directory"]
+    cluster = data_config["cluster"]
+    image_encoder_size = data_config["image_encoder_size"]
+    batch_size = data_config["batch_size"]
+    drop_last = data_config["drop_last"]
+    num_workers = data_config["num_workers"]
+    
+
+    # Parameters only for training
+    use_holdout_testset = data_config.get("use_holdout_testset", True)
+    holdout_testsets = data_config.get("test_datasets", datasets)
+    train_split = data_config.get("train_split", 0.8)
+    val_split = data_config.get("val_split", 0.1)
+    test_split = data_config.get("test_split", 0.1)
+    data_split = data_config.get("data_split", False)
+    shuffle = data_config.get("shuffle", False)
+    seed = data_config.get("seed", 1)
+    mask_augmentation_tries = data_config.get("mask_augmentation_tries", 5)
+    data_augmentations = data_config.get("data_augmentations", ["NoOp"])
+    threshold_connected_components = data_config.get("threshold_connected_components", 2)
+
+    dataset = HistologyDataset(datasets, data_directory, cluster, image_encoder_size, mask_augmentation_tries, data_augmentations, threshold_connected_components)
+    if data_split:
+        generator = torch.Generator().manual_seed(seed)
+        if use_holdout_testset:
+            train_set, val_set = random_split(dataset, [train_split + val_split, test_split],generator=generator)
+            test_set = HistologyDataset(holdout_testsets, data_directory, cluster, image_encoder_size, mask_augmentation_tries, ["NoOp"], threshold_connected_components)
+        else:
+            train_set, val_set, test_set = random_split(dataset, [train_split, val_split, test_split],generator=generator)
+        train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last, num_workers=num_workers)
+        val_loader = DataLoader(val_set, batch_size=batch_size, shuffle=False, drop_last=drop_last, num_workers=num_workers)
+        test_loader = DataLoader(test_set, batch_size=batch_size, shuffle=False, drop_last=drop_last, num_workers=num_workers)
+        return train_loader, val_loader, test_loader
+    else:
+        dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last, num_workers=num_workers)
+        return dataset, dataloader

CellPilot/cellpilot/inference/app_tools.py

@@ -0,0 +1,195 @@
+import numpy as np
+from torchvision.transforms.functional import resize, to_pil_image, InterpolationMode
+from segment_anything.utils.transforms import ResizeLongestSide
+import cv2
+from .inference import Inference
+import gradio as gr
+
+class App(Inference):
+    def __init__(self, config):
+        self.config = config
+        self.inference_config = config["inference_config"]
+        super().__init__(self.inference_config)
+        self.zoom_factor = 1
+        self.middle = (512,512)
+        self.upper = int(512 + self.zoom_factor*self.middle[0])
+        self.left = int(512 + self.zoom_factor*self.middle[1])
+        self.current_refinement = None
+        self.prompts = []
+        self.transform = ResizeLongestSide(1024)
+
+    def load_image(self, image, interpolation_mode=InterpolationMode.BILINEAR):
+        img = image["image"]    
+        self.orig_h, self.orig_w = img.shape[:2]
+        self.orig_masks = np.zeros((self.orig_h, self.orig_w), dtype=np.int16)
+        self.orig_img = img
+        self.h, self.w = ResizeLongestSide.get_preprocess_shape(img.shape[0], img.shape[1], 1024)
+        img = np.array(resize(to_pil_image(img), (self.h, self.w), interpolation_mode))
+        self.predictor.set_image(img)
+        self.img = img
+        self.masks = np.zeros((self.h, self.w), dtype=np.uint8)
+        self.zoom_factor = 1
+        self.middle = (512,512)
+        self.upper = int(512 + self.zoom_factor*self.middle[0])
+        self.left = int(512 + self.zoom_factor*self.middle[1])
+        self.current_refinement = None
+        self.prompts = []
+        self.transform = ResizeLongestSide(1024)
+        img = np.pad(img, ((0,1024-self.h), (0,1024-self.w), (0,0)))
+        self.current_image = np.pad(img, ((1024,1024), (1024,1024), (0,0)))
+        self.current_masks = np.zeros((3072,3072), dtype=np.int16)
+        self.masks = np.zeros((self.h, self.w))
+        self.orig_masks = np.array(resize(to_pil_image(self.masks.astype(np.int16)), (int(self.orig_h), int(self.orig_w)), InterpolationMode.NEAREST))
+        return {"image": img.astype(np.uint8)}
+
+    def zoom(self, zoom_factor, image):
+        points = image.get("points", [])
+        if points != []:
+            old_points = points
+            points = points.copy()
+            for i in range(len(points)):
+                orig_points = ((self.left -1024 + old_points[i][0])/self.zoom_factor, (self.upper - 1024 + old_points[i][1])/self.zoom_factor)
+                points[i][0] = int(orig_points[0] * zoom_factor)
+                points[i][1] = int(orig_points[1] * zoom_factor)
+        self.zoom_factor = zoom_factor
+        img = np.array(resize(to_pil_image(self.orig_img.astype(np.uint8)), (int(zoom_factor * self.h), int(zoom_factor*self.w)), InterpolationMode.BILINEAR))
+        self.current_image = np.pad(img, ((1024,1024), (1024,1024), (0,0)))
+        self.zoom_masks()
+        return self.display_current_image(points)
+    
+    def zoom_masks(self):
+        self.current_masks = np.array(resize(to_pil_image(self.orig_masks.astype(np.int16)), (int(self.zoom_factor * self.h), int(self.zoom_factor*self.w)), InterpolationMode.NEAREST))
+        self.current_masks = np.pad(self.current_masks, ((1024,1024), (1024,1024)))
+        
+    def color_mask(self, mask):
+        if self.current_refinement is not None:
+            mask = np.where(mask == self.current_refinement, -1, mask)
+        mask = np.where(mask > 0, 1, mask)
+        h, w = mask.shape[-2:]
+        color = np.array([30, 144, 255]).astype(np.uint8)
+        mask = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)
+        mask = mask.astype(np.uint8)
+        return mask
+
+    def move_up(self, amount=10):
+        self.middle = (self.middle[0] - amount/self.zoom_factor, self.middle[1])   
+        return self.display_current_image()
+    
+    def move_down(self, amount=10):
+        self.middle = (self.middle[0] + amount/self.zoom_factor, self.middle[1])
+        return self.display_current_image()
+    
+    def move_left(self, amount=10):
+        self.middle = (self.middle[0], self.middle[1] - amount/self.zoom_factor)
+        return self.display_current_image()
+    
+    def move_right(self, amount=10):
+        self.middle = (self.middle[0], self.middle[1] + amount/self.zoom_factor)
+        return self.display_current_image()
+    
+    def display_current_image(self, points=[]):
+        self.upper = int(512 + self.zoom_factor*self.middle[0])
+        self.left = int(512 + self.zoom_factor*self.middle[1])
+        img = self.current_image[self.upper:self.upper + 1024, self.left:self.left + 1024,:]
+        mask = self.current_masks[self.upper:self.upper + 1024, self.left:self.left + 1024]
+        color_mask = self.color_mask(mask)
+        img =  np.where(color_mask > 0, 0.4 * img, img)
+        img = img.astype(np.uint8)
+        img = cv2.addWeighted(img, 1.0, color_mask, 0.6, 0.0)
+        if points != []:
+            for i in range(len(points)):
+                points[i][0] = points[i][0] - self.upper + 1024
+                points[i][1] = points[i][1] - self.left + 1024
+        return {"image": img.astype(np.uint8),"points": points}
+
+
+    def segment_automatically_app(self):
+        masks, self.prompts = self.segment_automatically(self.img)
+        self.masks = masks[:self.h, :self.w]
+        self.orig_masks = np.array(resize(to_pil_image(self.masks.astype(np.int16)), (int(self.orig_h), int(self.orig_w)), InterpolationMode.NEAREST))
+        self.zoom_masks()
+        return self.display_current_image()
+         
+    def add_mask(self, input):
+        prompts = input.get("points", [])
+        if prompts == []:
+            return self.display_current_image()
+        new_prompt = self.new_prompt(prompts, [], [], [])
+        self.prompts.append(new_prompt)
+        new_mask = self.segment(new_prompt)
+        self.masks = np.where(new_mask == 1, len(self.prompts), self.masks)
+        self.orig_masks = np.array(resize(to_pil_image(self.masks.astype(np.int16)), (int(self.orig_h), int(self.orig_w)), InterpolationMode.NEAREST))
+        self.zoom_masks()
+        return self.display_current_image(points=[])
+    
+    def new_prompt(self, prompts, point_coords=[], point_labels=[], boxes=[]):
+        for prompt in prompts:
+            p0 = (prompt[0]+self.left-1024)/self.zoom_factor
+            p1 = (prompt[1]+self.upper-1024)/self.zoom_factor
+            if prompt[3] == 0:
+                point_coords.append([p0, p1])
+                point_labels.append(1)
+            else:
+                p2= (prompt[3]+self.left-1024)/self.zoom_factor
+                p3 = (prompt[4]+self.upper-1024)/self.zoom_factor
+                boxes = [p0, p1, p2, p3]     
+        if point_coords == []:
+            point_coords = None
+            point_labels = None
+        else:
+            point_coords = np.array(point_coords)
+            point_labels = np.array(point_labels)
+        if boxes == []:
+            boxes = None
+        else:
+            boxes = np.array(boxes)
+        new_prompt = {
+            "point_coords": point_coords,
+            "point_labels": point_labels,
+            "boxes": boxes
+        }
+        return new_prompt  
+
+    def start_refine_mask(self, input):
+        prompts = input.get("points", [])
+        if prompts == []:
+            return self.display_current_image()
+        point = (int(input["points"][0][1]), int(input["points"][0][0]))
+        value = self.current_masks[self.upper + point[0], self.left + point[1]]
+        self.current_refinement = value
+        return self.display_current_image(), gr.Column(visible=True), gr.Column(visible=False)
+
+    def remove_mask(self, input):
+        prompts = input.get("points", [])
+        if prompts == []:
+            return self.display_current_image()
+        point = (int(input["points"][0][1]), int(input["points"][0][0]))
+        value = self.current_masks[self.upper + point[0], self.left + point[1]]
+        self.orig_masks = np.where(self.orig_masks == value, 0, self.orig_masks)
+        self.current_masks = np.where(self.current_masks == value, 0, self.current_masks)
+        self.masks = np.where(self.masks == value, 0, self.masks)
+        return self.display_current_image()
+    
+    def refine_mask(self, input):
+        prompts = input.get("points", [])
+        if prompts == []:
+            return self.display_current_image()
+        prompts = input["points"]
+        point_coords = self.prompts[self.current_refinement-1].get("point_coords", np.array([]))
+        point_labels = self.prompts[self.current_refinement-1].get("point_labels", np.array([]))
+        boxes = self.prompts[self.current_refinement-1].get("boxes", np.array([]))
+        point_coords = point_coords.tolist() if point_coords is not None else []
+        point_labels = point_labels.tolist() if point_labels is not None else []
+        boxes = boxes.tolist() if boxes is not None else []
+        new_prompt = self.new_prompt(prompts, point_coords, point_labels, boxes)
+        self.prompts[self.current_refinement-1] = new_prompt
+        new_mask = self.segment(new_prompt, self.img)
+        self.masks = np.where(self.masks == self.current_refinement, 0, self.masks)
+        self.masks = np.where(new_mask == 1, self.current_refinement, self.masks)
+        self.orig_masks = np.array(resize(to_pil_image(self.masks.astype(np.int16)), (int(self.orig_h), int(self.orig_w)), InterpolationMode.NEAREST))
+        self.zoom_masks()
+        return self.display_current_image()
+    
+    def finish_mask(self):
+        self.current_refinement = None
+        return self.display_current_image(), gr.Column(visible=True), gr.Column(visible=False)

CellPilot/cellpilot/inference/display.py

@@ -0,0 +1,32 @@
+import numpy as np
+import torch
+import matplotlib.pyplot as plt
+
+def show_mask(mask, ax, random_color=False):
+    if random_color:
+        color = np.concatenate([np.random.random(3), np.array([0.6])], axis=0)
+    else:
+        color = np.array([30/255, 144/255, 255/255, 0.6])
+    h, w = mask.shape[-2:]
+    mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)
+    ax.imshow(mask_image)
+    
+def show_points(coords, labels, ax, marker_size=375):
+    pos_points = coords[labels==1]
+    neg_points = coords[labels==0]
+    ax.scatter(pos_points[:, 0], pos_points[:, 1], color='green', marker='*', s=marker_size, edgecolor='white', linewidth=1.25)
+    ax.scatter(neg_points[:, 0], neg_points[:, 1], color='red', marker='*', s=marker_size, edgecolor='white', linewidth=1.25)   
+    
+def show_box(box, ax):
+    x0, y0 = box[0], box[1]
+    w, h = box[2] - box[0], box[3] - box[1]
+    ax.add_patch(plt.Rectangle((x0, y0), w, h, edgecolor='green', facecolor=(0,0,0,0), lw=2)) 
+
+def show_boxes(boxes, ax):
+    for box in boxes:
+        show_box(box, ax)
+
+def get_original_image(image, mean=torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1), std=torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1)):
+    image = (image * std + mean) #* 255
+    image = image.permute(1, 2, 0).cpu().numpy()
+    return image.astype(np.uint8)

CellPilot/cellpilot/inference/evaluation_tools.py

@@ -0,0 +1,328 @@
+import torch
+from ..modeling.model import SamHI
+from ..data_processing.data_utils import PromptProcessing
+from ..data_processing.dataset import prepare_data
+from pathlib import Path
+import numpy as np
+from PIL import Image
+import os 
+import pandas as pd
+from monai.metrics import compute_iou, compute_dice
+import json
+import random
+from tqdm import tqdm
+from segment_anything import SamAutomaticMaskGenerator, SamPredictor
+import cv2
+import torch.nn.functional as F
+from isegm.inference.evaluation import evaluate_sample
+from isegm.inference.predictors import get_predictor
+from isegm.inference import utils
+from .inference import Inference
+from segment_anything.utils.transforms import ResizeLongestSide
+from skimage.transform import resize as sk_resize
+
+class Evaluation(Inference):
+    def __init__(self, config):
+        self.config = config
+        self.inference_config = config["inference_config"]
+        self.load_config(self.inference_config)
+        self.load_eval_config()
+        torch.manual_seed(self.seed)
+        np.random.seed(self.seed)
+        random.seed(self.seed)
+        if not os.path.exists(Path(self.model_dir + self.model_name + "_" + self.time + "_eval_config.json")):
+            os.mknod(Path(self.model_dir + self.model_name + "_" + self.time + "_eval_config.json"))
+        self.initialize_model_evaluation()
+
+    def load_eval_config(self):
+        self.data_config = self.config["data_config"]
+        self.eval_config = self.config["eval_config"]
+        self.prompt_config = self.config["prompt_config"]
+        self.interactive_or_auto = self.eval_config["interactive_or_auto"]
+        self.interactive_model = self.eval_config["interactive_model"]
+        self.auto_model = self.eval_config["auto_model"]
+        self.time = self.eval_config["time"]
+        self.seed = self.eval_config["seed"]
+        self.max_nr_of_points = self.eval_config["max_nr_of_points"]
+        self.batch_size = self.eval_config["batch_size"]
+        self.prompt_batch_size = self.eval_config.get("prompt_batch_size", 1)
+        self.nr_of_interactive_points_choices = self.eval_config.get("nr_of_interactive_points_choices", [0, 1, 2, 3, 4, 5])
+        self.prompt_choices = self.eval_config.get("prompt_choices", ["points", "boxes"])
+        self.dataset_choices = self.eval_config["dataset_choices"]
+        self.mask_threshold = self.eval_config.get("mask_threshold", 0.0)
+        self.model_type = self.eval_config["model_type"]
+        self.base_model = self.eval_config["base_model"]
+
+    def initialize_model_evaluation(self):
+        if self.interactive_or_auto == "interactive":
+            if self.interactive_model == "simpleclick":
+                self.initialize_simpleclick()
+            elif self.interactive_model == "sam":
+                self.initialize_sam()
+            else:
+                self.initialize_model()
+        else:
+            if self.auto_model == "cellvit":
+                self.initialize_cellvit()
+            elif self.auto_model == "sam":
+                self.initialize_sam()
+            else:
+                self.initialize_cellvit()
+                self.initialize_model()
+        
+    def initialize_simpleclick(self):
+        checkpoint_path = self.model_dir + "SimpleClick/cocolvis_vit_base.pth"
+        self.model = utils.load_is_model(checkpoint_path, self.device, False)
+        self.model.eval()
+        self.predictor = get_predictor(self.model, 'NoBRS', self.device, prob_thresh=0.49, zoom_in_params=None)
+        self.max_iou_thr = 1.0
+        self.pixel_mean = torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1).cuda()
+        self.pixel_std = torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1).cuda()
+        self.data_config["image_encoder_size"] = 448
+        self.prompt_batch_size = 1
+
+    def initialize_sam(self):
+        model_config = {
+            "model_type": self.model_type,
+            "base_model": self.base_model,
+            "lora_rank": 1,
+            "lora_layer": [-1],
+            "model_mode": "inference",
+            "model_dir": self.model_dir,
+        }
+        model = SamHI({"model_config": model_config}).to(self.device)
+        model.eval()
+        self.model = model
+        self.predictor = SamPredictor(model.model)
+    
+    @torch.no_grad()
+    def evaluate(self):
+        if self.interactive_or_auto == "auto":
+            self.evaluate_auto()
+        else:
+            self.initialize_eval_dict(self.max_nr_of_points)
+            self.predicted_masks = []
+            self.target_masks = []
+            for dataset_name in self.dataset_choices:
+                self.data_config["datasets"] = [dataset_name]
+                dataset, dataloader = prepare_data(self.data_config)
+                for (i1, p1) in enumerate(self.prompt_choices):
+                    self.prompt_config["prompt_type"] = p1
+                    for (i2, p2) in enumerate(self.nr_of_interactive_points_choices): 
+                        nr_of_interactive_points = p2
+                        for (idx, (data, target, nr, (image_name, mask_name), (left, upper, right, lower), (new_h, new_w)), p_tune) in tqdm(dataloader):
+                            self.update_eval_dict_first_part(dataset_name, nr_of_interactive_points, idx, image_name, mask_name, left, upper, right, lower) 
+                            data, target = data.to(self.device), target.to(self.device)
+                            prompts, target, target_nr = PromptProcessing.get_prompts_and_targets(nr,  target, self.device, self.prompt_config)
+                            if self.interactive_model == "simpleclick":
+                                self.evaluate_simpleclick(data, target, target_nr, nr_of_interactive_points + 1, new_h, new_w)
+                            else:
+                                output, _, image_embeddings = self.model.forward(data, prompts, p_tune=p_tune)
+                                for i in range(nr_of_interactive_points): 
+                                    prompts= PromptProcessing.refine_prompts(nr, target, prompts, output, self.device,self.prompt_batch_size)
+                                    output, _, _ = self.model.forward(data, prompts, image_embeddings=image_embeddings, p_tune=p_tune)
+                                output = output > self.mask_threshold
+                                self.predicted_masks.extend([m for m in output.squeeze().cpu().numpy()])
+                                self.target_masks.extend([m for m in target.cpu().numpy()])
+                                self.update_eval_dict_second_part( prompts, target_nr, output, target)
+        self.save_eval_results(self.predicted_masks, self.target_masks)
+
+
+    def evaluate_simpleclick(self, data, target, target_nr, max_clicks, new_h, new_w):
+        for i in range(len(data)):
+            image = data[i]
+            image = image[:, :new_h[i], :new_w[i]]
+            image = image * self.pixel_std + self.pixel_mean
+            image = F.pad(image, (0, 448 - new_w[i], 0, 448 - new_h[i]))
+            image = image.permute(1, 2, 0).cpu().numpy().astype(np.uint8)
+            gt = target[i].cpu().numpy()
+            clicks_list, ious, probs = evaluate_sample(image, gt, self.predictor, self.max_iou_thr, max_clicks=max_clicks)
+            prompt = {
+                "point_coords": torch.zeros(1, max_clicks, 2),
+                "point_labels": torch.zeros(1, max_clicks)
+            }
+            for (c, click) in enumerate(clicks_list):
+                prompt["point_coords"][0, c, 0] = click.coords[1]
+                prompt["point_coords"][0, c, 1] = click.coords[0]
+                prompt["point_labels"][0,c] = int(click.is_positive)
+            output = (probs > 0.49)
+            self.predicted_masks.extend([output])
+            self.target_masks.extend([target[i].cpu().numpy()])
+            self.update_eval_dict_second_part([prompt], [target_nr[i]], torch.tensor(output).unsqueeze(0).unsqueeze(1), target[i].cpu().unsqueeze(0))
+
+    def evaluate_auto(self):
+        transform = ResizeLongestSide(1024)
+        self.eval_dict = {
+            "Dataset": [],
+            "Index": [],
+            "Image name": [],
+            "Mask name": [],
+            "Left": [],
+            "Upper": [],
+            "Right": [],
+            "Lower": [],
+            "IoU": [],
+            "Dice": [],
+        }
+        self.predicted_masks = []
+        self.target_masks = []
+        pixel_mean = torch.Tensor([123.675, 116.28, 103.53]).view(-1, 1, 1).cuda()
+        pixel_std = torch.Tensor([58.395, 57.12, 57.375]).view(-1, 1, 1).cuda()
+        for dataset_name in self.dataset_choices:
+            self.data_config["datasets"] = [dataset_name]
+            dataset, dataloader = prepare_data(self.data_config)
+            for (idx, (data, target, _,(image_name, mask_name), (left, upper, right, lower), (new_h, new_w)), p_tune) in tqdm(dataloader):
+                with torch.no_grad():  
+                    data, target = data.to(self.device), target.to(self.device)
+                    for (i,d) in enumerate(data):
+                        img = np.array(Image.open(image_name[i]).convert("RGB"))
+                        img = transform.apply_image(img)
+                        h, w = img.shape[0], img.shape[1]
+                        # img = d[:, :new_h[i], :new_w[i]]
+                        # img = img * pixel_std + pixel_mean 
+                        # img = img.cpu().numpy().astype(np.uint8)   
+                        # img = img.transpose((1, 2, 0))     
+                        if self.auto_model=="cellvit":
+                            mask = self.segment_automatically_cellvit(img)
+                        elif self.auto_model=="sam":
+                            mask, _ = self.segment_automatically_sam(img)
+                            mask = np.pad(mask,((0, 1024-mask.shape[0]), (0, 1024-mask.shape[1])))
+                        else:
+                            mask,_ = self.segment_automatically(img)
+                            mask = np.pad(mask,((0, 1024-mask.shape[0]), (0, 1024-mask.shape[1])))
+                        mask = np.where(mask == 0, 0, 1)
+                        gt = np.array(Image.open(mask_name[i]))
+                        gt = np.where(gt == 0, 0, 1).astype(np.uint8)
+                        gt = sk_resize(gt, (h,w), preserve_range=True, order = 0)
+                        gt = np.pad(gt,((0, 1024-gt.shape[0]), (0, 1024-gt.shape[1])))
+                        # gt = torch.where(target[i].unsqueeze(0).unsqueeze(1).cpu() == 0, 0, 1)
+                        self.predicted_masks.extend([mask])
+                        self.target_masks.extend([gt])
+                        self.eval_dict["Dataset"].extend([dataset_name])
+                        self.eval_dict["Index"].extend([idx[i].item()])
+                        self.eval_dict["Image name"].extend([image_name[i]])
+                        self.eval_dict["Mask name"].extend([mask_name[i]])
+                        self.eval_dict["Left"].extend([left[i].item()])
+                        self.eval_dict["Upper"].extend([upper[i].item()])
+                        self.eval_dict["Right"].extend([right[i].item()])
+                        self.eval_dict["Lower"].extend([lower[i].item()])
+                        self.eval_dict["IoU"].extend([compute_iou(torch.tensor(mask[np.newaxis, np.newaxis, :, :]), torch.tensor(gt[np.newaxis, np.newaxis, :, :])).item()])
+                        self.eval_dict["Dice"].extend([compute_dice(torch.tensor(mask[np.newaxis, np.newaxis, :, :]), torch.tensor(gt[np.newaxis, np.newaxis, :, :])).item()])
+        #self.save_eval_results(self.predicted_masks, self.target_masks)
+
+    def initialize_eval_dict(self, max_nr_of_prompts=4):
+        eval_dict = {
+            "Dataset": [],
+            "Prompt type": [],
+            "Nr of interactive points": [],
+            "Index": [],
+            "Image name": [],
+            "Mask name": [],
+            "Left": [],
+            "Upper": [],
+            "Right": [],
+            "Lower": [],
+            "Target nr": [],
+            "IoU": [],
+            "Dice": [],
+        }
+        for i in range(4):
+            eval_dict["Box_coord_" + str(i+1)] = []
+        for i in range(max_nr_of_prompts):
+            eval_dict["Point_coord_" + str(2*i+1)] = []
+            eval_dict["Point_coord_" + str(2*i+2)] = []
+            eval_dict["Point_label_" + str(i+1)] = []
+        self.eval_dict = eval_dict
+
+    def update_eval_dict_first_part(self, dataset, nr_of_interactive_points, idx, image_name, mask_name, left, upper, right, lower):
+        self.eval_dict["Dataset"].extend([dataset] * self.batch_size * self.prompt_batch_size)
+        self.eval_dict["Nr of interactive points"].extend([nr_of_interactive_points] * self.batch_size * self.prompt_batch_size)
+        self.eval_dict["Prompt type"].extend([self.prompt_config["prompt_type"]] * self.batch_size * self.prompt_batch_size)
+        self.eval_dict["Index"].extend([i.item() for i in idx for j in range(self.prompt_batch_size)])
+        self.eval_dict["Image name"].extend([i for i in image_name for j in range(self.prompt_batch_size)])
+        self.eval_dict["Mask name"].extend([m for m in mask_name for j in range(self.prompt_batch_size)])
+        self.eval_dict["Left"].extend([l.item() for l in left for j in range(self.prompt_batch_size)])
+        self.eval_dict["Upper"].extend([u.item() for u in upper for j in range(self.prompt_batch_size)])
+        self.eval_dict["Right"].extend([r.item() for r in right for j in range(self.prompt_batch_size)])
+        self.eval_dict["Lower"].extend([l.item() for l in lower for j in range(self.prompt_batch_size)])
+
+
+    def update_eval_dict_second_part(self, prompts, target_nr, output, target):
+        self.eval_dict["Target nr"].extend(target_nr)
+        for (i, p) in enumerate(prompts):
+            if "point_coords" in p.keys():
+                points = p["point_coords"].cpu().numpy()
+                labels = p["point_labels"].cpu().numpy()
+                for j in range(self.max_nr_of_points):
+                    if j < points.shape[1]:
+                        self.eval_dict["Point_coord_" + str(2*j+1)].extend([pt[j,0] for pt in points])
+                        self.eval_dict["Point_coord_" + str(2*j+2)].extend([pt[j,1] for pt in points])
+                        self.eval_dict["Point_label_" + str(j+1)].extend([l[j] for l in labels])
+                    else:
+                        self.eval_dict["Point_coord_" + str(2*j+1)].extend([0 for p in points])
+                        self.eval_dict["Point_coord_" + str(2*j+2)].extend([0 for p in points])
+                        self.eval_dict["Point_label_" + str(j+1)].extend([0 for l in labels])
+            else:
+                for j in range(self.max_nr_of_points):
+                    self.eval_dict["Point_coord_" + str(2*j+1)].extend([0 for p in range(self.prompt_batch_size)])
+                    self.eval_dict["Point_coord_" + str(2*j+2)].extend([0 for p in range(self.prompt_batch_size)])
+                    self.eval_dict["Point_label_" + str(j+1)].extend([0 for l in range(self.prompt_batch_size)])
+            
+            if "boxes" in p.keys():
+                boxes = p["boxes"].cpu().numpy()
+                for i in range(4):   
+                    self.eval_dict["Box_coord_" + str(i+1)].extend([b[i] for b in boxes])
+            else:
+                for i in range(4):
+                    self.eval_dict["Box_coord_" + str(i+1)].extend([0 for b in range(self.prompt_batch_size)])
+        self.eval_dict["IoU"].extend(compute_iou(output, target.unsqueeze(1)).squeeze(1).cpu().numpy().tolist())
+        self.eval_dict["Dice"].extend(compute_dice(output, target.unsqueeze(1)).squeeze(1).cpu().numpy().tolist())
+
+    def save_eval_results(self, predicted_masks, target_masks):
+        os.makedirs(self.model_dir + self.model_name + "_" + self.time + "_eval_results", exist_ok=True)
+        for (i,p) in enumerate(predicted_masks):
+            p = np.where(p == 1, 255, 0).astype(np.uint8)
+            p = Image.fromarray(p)
+            p.save(self.model_dir + self.model_name + "_" + self.time + "_eval_results/" + str(i) +  "_pred_mask.png")
+
+        for (i,t) in enumerate(target_masks):
+            t = np.where(t == 1, 255, 0).astype(np.uint8)
+            t = Image.fromarray(t)
+            t.save(self.model_dir + self.model_name + "_" + self.time + "_eval_results/" + str(i) + "_target_mask.png")
+
+        df = pd.DataFrame(self.eval_dict)
+        df.to_csv(Path(self.model_dir + self.model_name + "_" + self.time + "_eval_results.csv"))
+        with open(Path(self.model_dir + self.model_name + "_" + self.time + "_eval_config.json"), "w") as file:
+            self.device = str(self.device)
+            json.dump(self.config, file)
+
+    def segment_automatically_sam(self, image):
+        mask_generator = SamAutomaticMaskGenerator(self.model.model)
+        masks = mask_generator.generate(image)
+        sorted_masks = sorted(masks, key=lambda x: x['stability_score'], reverse=False)
+        # Get the masks and prompts
+        prel_masks = np.zeros(image.shape[:2])
+        prel_points = []
+        for (i, m) in enumerate(sorted_masks):
+            prel_masks[m["segmentation"]] = i
+            prel_points.append(m['point_coords'])
+        # filter out masks that are covered up by other masks
+        prompts = []
+        masks = np.zeros(image.shape[:2])
+        for (i, v) in enumerate(np.unique(prel_masks)):
+            masks = np.where(prel_masks == v, i, masks)
+            prompts.append({"point_coords":torch.tensor(prel_points[int(v)]).to(self.device), "point_labels":torch.tensor([1]).to(self.device)})
+        return masks, prompts
+    
+    def segment_automatically_cellvit(self, image):
+        instance_types = self.detect_cellvit(image)
+        base = np.zeros((1024,1024))
+        masks = []
+        for k in instance_types[0].keys():
+            masks.append(cv2.drawContours(base, [instance_types[0][k]['contour'][:, np.newaxis,:]], 0, 1, cv2.FILLED))
+        try:
+            mask = np.array(masks).max(axis=0)
+        except:
+            mask = np.zeros((1024,1024))
+        return mask
+    

CellPilot/cellpilot/inference/inference.py

@@ -0,0 +1,140 @@
+import torch
+from ..modeling.model import SamHI
+import wandb
+from pathlib import Path
+import numpy as np
+import os 
+from segment_anything.utils.transforms import ResizeLongestSide
+from torchvision.transforms.functional import resize, to_pil_image, InterpolationMode
+from ..modeling.predictor import SamHIPredictor
+from models.segmentation.cell_segmentation.cellvit import CellViT256
+from torchvision import transforms as T
+import torch.nn.functional as F
+from typing import Dict, List, Tuple
+
+class Inference:
+    def __init__(self, config):
+        self.load_config(config)
+        self.initialize_model()
+        self.initialize_cellvit()
+    
+    def load_config(self, config):
+        self.device = config["device"]
+        self.model_dir = config["model_dir"]
+        self.model_name = config["model_name"]
+
+    def initialize_model(self):
+        model_path = os.path.join(self.model_dir, self.model_name + ".ckpt")
+        if not Path(model_path).is_file():
+            api = wandb.Api()
+            artifact = api.artifact("philippresearch/SAMHI/" + self.model_name, type="model")
+            artifact_dir = artifact.download(root=self.model_dir)
+            os.rename(artifact_dir + "model.ckpt", model_path)
+        model_config = torch.load(model_path, map_location=lambda storage, loc: storage)["hyper_parameters"]["config"]["model_config"]
+        model_config["model_mode"] = "inference"
+        model_config["model_dir"] = self.model_dir
+        config = {"model_config": model_config}
+        model = SamHI.load_from_checkpoint(model_path, config=config).to(self.device)
+        model.eval()
+        self.model = model
+        self.predictor = SamHIPredictor(model.model, p_tuning=model.p_tuning)
+
+    def initialize_cellvit(self, model_name="CellViT-256-x40.pth"):
+        checkpoint = torch.load(self.model_dir + model_name)
+        config = checkpoint['config']
+        model = CellViT256(model256_path=None,
+                num_nuclei_classes=config["data.num_nuclei_classes"],
+                num_tissue_classes=config["data.num_tissue_classes"],
+                regression_loss=False)
+        model.load_state_dict(checkpoint['model_state_dict'])
+        mean = (0.5, 0.5, 0.5)
+        std = (0.5, 0.5, 0.5)
+        inference_transforms = T.Compose(
+            [T.ToTensor(), T.Normalize(mean=mean, std=std)]
+        )
+        self.cellvit = model
+        self.cellvit_transforms = inference_transforms
+        self.cellvit_mean = mean
+        self.cellvit_std = std
+        self.cellvit.eval()
+
+
+    def segment(
+        self, 
+        prompt, 
+        image = None
+    ) -> np.ndarray:
+        """
+        Segment the image based on the prompt
+
+        Arguments:
+            prompt (dict): The prompt for the segmentation. It should contain the following keys:
+                - point_coords (np.ndarray): The coordinates of the points. 
+                    A Nx2 array of point prompts to the model.
+                    Each point is in (X,Y) in pixels.
+                - point_labels (np.ndarray): The labels of the points.
+                    A length N array of labels for the point prompts. 
+                    1 indicates a foreground point and 0 indicates background point.
+                - boxes (np.ndarray): The bounding boxes.
+                    A length 4 array given a box prompt to the model, in XYXY format.
+            image (np.ndarray): The image to segment. If None, it is assumed that the image is already set.
+                Expects an image in HWC uint8 format, with pixel values in [0, 255].
+        
+        Returns:
+            np.ndarray: The mask of the segmentation. A binary mask of the same shape as the input image.
+        """
+        if image is not None:
+            try:
+                self.predictor.set_image(image)
+            except:
+                raise("Error: Please provide an image")
+        mask, _, _ = self.predictor.predict(point_coords=prompt.get("point_coords", None), point_labels=prompt.get("point_labels", None), box=prompt.get("boxes", None), multimask_output=False)
+        return mask[0]
+    
+    def segment_automatically(
+        self, 
+        image
+    ) -> Tuple[np.ndarray, List[Dict]]:
+        """
+        Segment the image automatically using CellViT
+
+        Arguments:
+            image (np.ndarray): The image to segment. 
+                Expects an image in HWC uint8 format, with pixel values in [0, 255].
+
+        Returns:
+            Tuple[np.ndarray, List[Dict]]: The segmented image and the prompts for the segmentation.   
+        """
+        if image.shape[0] > 1024 or image.shape[1] > 1024:
+            h, w = ResizeLongestSide.get_preprocess_shape(img.shape[0], img.shape[1], 1024)
+            img = resize(to_pil_image(img), (h, w), InterpolationMode.BILINEAR)
+        instance_types = self.detect_cellvit(image)
+        self.predictor.set_image(image)
+        masks = np.zeros(image.shape[:2])
+        prompts = []
+        for k in instance_types[0].keys():
+            box = instance_types[0][k]["bbox"]
+            box = np.array([box[0,1], box[0,0], box[1,1], box[1,0]])
+            mask, _, _ = self.predictor.predict(box=np.array(box), multimask_output=False)
+            masks[mask[0] == 1] = int(k)
+            prompts.append({"boxes":np.array(box)})
+        return masks, prompts
+
+    def detect_cellvit(self, image):
+        """
+        Detect the nuclei in the image using CellViT
+
+        Arguments:
+            image (np.ndarray): The image to segment. 
+                Expects an image in HWC uint8 format, with pixel values in [0, 255].
+        """
+        img = torch.tensor(image).unsqueeze(0).float()
+        img = img.permute(0, 3, 1, 2)
+        img = torch.nn.functional.pad(img, (0, 1024-img.shape[3], 0, 1024-img.shape[2]), value=0)
+        img_norm = (img/256-torch.tensor(self.cellvit_mean).view(3, 1, 1)/torch.tensor(self.cellvit_std).view(3, 1, 1))
+        with torch.no_grad():
+            predictions = self.cellvit(img_norm)
+            predictions["nuclei_binary_map"] = F.softmax(predictions["nuclei_binary_map"], dim=1)  # shape: (batch_size, 2, H, W)
+            predictions["nuclei_type_map"] = F.softmax(predictions["nuclei_type_map"], dim=1)  # shape: (batch_size, num_nuclei_classes, H, W)
+            (_, instance_types,) = self.cellvit.calculate_instance_map(predictions)
+        return instance_types

CellPilot/cellpilot/modeling/LoRA_Sam.py

@@ -0,0 +1,224 @@
+# Adapted from: https://github.com/hitachinsk/SAMed/blob/main/sam_lora_image_encoder.py
+import torch
+import lightning.pytorch as pl
+from segment_anything.modeling import Sam
+from torch import nn
+import math
+from torch.nn.parameter import Parameter
+
+class _LoRA_qkv(nn.Module):
+    """In Sam it is implemented as
+    self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+    B, N, C = x.shape
+    qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
+    q, k, v = qkv.unbind(0)
+    """
+
+    def __init__(
+            self,
+            qkv: nn.Module,
+            linear_a_q: nn.Module,
+            linear_b_q: nn.Module,
+            linear_a_v: nn.Module,
+            linear_b_v: nn.Module,
+    ):
+        super().__init__()
+        self.qkv = qkv
+        self.linear_a_q = linear_a_q
+        self.linear_b_q = linear_b_q
+        self.linear_a_v = linear_a_v
+        self.linear_b_v = linear_b_v
+        self.dim = qkv.in_features
+        self.w_identity = torch.eye(qkv.in_features)
+
+    def forward(self, x):
+        qkv = self.qkv(x)  # B,N,N,3*org_C
+        new_q = self.linear_b_q(self.linear_a_q(x))
+        new_v = self.linear_b_v(self.linear_a_v(x))
+        qkv[:, :, :, : self.dim] += new_q
+        qkv[:, :, :, -self.dim:] += new_v
+        return qkv
+
+
+class LoRA_Sam(pl.LightningModule):
+    """
+    Applies low-rank adaptation to a Sam model's image encoder.
+    """
+
+    def __init__(self, sam_model: Sam, r: int, lora_layer=None, p_tuning = False):
+        super(LoRA_Sam, self).__init__()
+        self.p_tuning = p_tuning
+        assert r > 0
+        # base_vit_dim = sam_model.image_encoder.patch_embed.proj.out_channels
+        # dim = base_vit_dim
+        if lora_layer:
+            self.lora_layer = lora_layer
+        else:
+            self.lora_layer = list(
+                range(len(sam_model.image_encoder.blocks)))  # Only apply lora to the image encoder by default
+        # create for storage, then we can init them or load weights
+        self.w_As = []  # These are linear layers
+        self.w_Bs = []
+
+        # lets freeze first
+        for param in sam_model.image_encoder.parameters():
+            param.requires_grad = False
+
+        # Here, we do the surgery
+        for t_layer_i, blk in enumerate(sam_model.image_encoder.blocks):
+            # If we only want few lora layer instead of all
+            if t_layer_i not in self.lora_layer:
+                continue
+            w_qkv_linear = blk.attn.qkv
+            self.dim = w_qkv_linear.in_features
+            w_a_linear_q = nn.Linear(self.dim, r, bias=False)
+            w_b_linear_q = nn.Linear(r, self.dim, bias=False)
+            w_a_linear_v = nn.Linear(self.dim, r, bias=False)
+            w_b_linear_v = nn.Linear(r, self.dim, bias=False)
+            self.w_As.append(w_a_linear_q)
+            self.w_Bs.append(w_b_linear_q)
+            self.w_As.append(w_a_linear_v)
+            self.w_Bs.append(w_b_linear_v)
+            blk.attn.qkv = _LoRA_qkv(
+                w_qkv_linear,
+                w_a_linear_q,
+                w_b_linear_q,
+                w_a_linear_v,
+                w_b_linear_v,
+            )
+        if p_tuning:
+            self.embed_dim = sam_model.prompt_encoder.embed_dim
+            self.image_embedding_size = sam_model.prompt_encoder.image_embedding_size
+            p = torch.nn.Linear(1, self.embed_dim * self.image_embedding_size[0] * self.image_embedding_size[1], bias=False)
+            sam_model.p = p
+        self.reset_parameters()
+        self.sam = sam_model
+
+
+
+    def save_lora_parameters(self, filename: str) -> None:
+        r"""Only safetensors is supported now.
+
+        pip install safetensor if you do not have one installed yet.
+
+        save both lora and fc parameters.
+        """
+
+        assert filename.endswith(".pt") or filename.endswith('.pth')
+
+        num_layer = len(self.w_As)  # actually, it is half
+        a_tensors = {f"w_a_{i:03d}": self.w_As[i].weight for i in range(num_layer)}
+        b_tensors = {f"w_b_{i:03d}": self.w_Bs[i].weight for i in range(num_layer)}
+        prompt_encoder_tensors = {}
+        mask_decoder_tensors = {}
+
+        # save prompt encoder, only `state_dict`, the `named_parameter` is not permitted
+        if isinstance(self.sam, torch.nn.DataParallel) or isinstance(self.sam, torch.nn.parallel.DistributedDataParallel):
+            state_dict = self.sam.module.state_dict()
+        else:
+            state_dict = self.sam.state_dict()
+        for key, value in state_dict.items():
+            if 'prompt_encoder' in key:
+                prompt_encoder_tensors[key] = value
+            if 'mask_decoder' in key:
+                mask_decoder_tensors[key] = value
+
+        merged_dict = {**a_tensors, **b_tensors, **prompt_encoder_tensors, **mask_decoder_tensors}
+        torch.save(merged_dict, filename)
+
+    def load_lora_parameters(self, filename: str) -> None:
+        r"""Only safetensors is supported now.
+
+        pip install safetensor if you do not have one installed yet.\
+
+        load both lora and fc parameters.
+        """
+
+        assert filename.endswith(".pt") or filename.endswith('.pth')
+
+        state_dict = torch.load(filename)
+
+        for i, w_A_linear in enumerate(self.w_As):
+            saved_key = f"w_a_{i:03d}"
+            saved_tensor = state_dict[saved_key]
+            w_A_linear.weight = Parameter(saved_tensor)
+
+        for i, w_B_linear in enumerate(self.w_Bs):
+            saved_key = f"w_b_{i:03d}"
+            saved_tensor = state_dict[saved_key]
+            w_B_linear.weight = Parameter(saved_tensor)
+
+        sam_dict = self.sam.state_dict()
+        sam_keys = sam_dict.keys()
+
+        # load prompt encoder
+        prompt_encoder_keys = [k for k in sam_keys if 'prompt_encoder' in k]
+        prompt_encoder_values = [state_dict[k] for k in prompt_encoder_keys]
+        prompt_encoder_new_state_dict = {k: v for k, v in zip(prompt_encoder_keys, prompt_encoder_values)}
+        sam_dict.update(prompt_encoder_new_state_dict)
+
+        # load mask decoder
+        mask_decoder_keys = [k for k in sam_keys if 'mask_decoder' in k]
+        mask_decoder_values = [state_dict[k] for k in mask_decoder_keys]
+        mask_decoder_new_state_dict = {k: v for k, v in zip(mask_decoder_keys, mask_decoder_values)}
+        sam_dict.update(mask_decoder_new_state_dict)
+        self.sam.load_state_dict(sam_dict)
+
+
+
+    def reset_parameters(self) -> None:
+        for w_A in self.w_As:
+            nn.init.kaiming_uniform_(w_A.weight, a=math.sqrt(5))
+        for w_B in self.w_Bs:
+            nn.init.zeros_(w_B.weight)
+
+    def forward(self, batched_input, batched_prompts, multimask_output, image_size, image_embeddings=None, p_tune=None):
+        sam = self.sam
+        if image_embeddings is None:
+            image_embeddings = sam.image_encoder(batched_input)
+        outputs = []
+        low_res_outputs = []
+        if p_tune is None:
+            p_tune = torch.zeros(len(batched_prompts)).to(self.device)
+        for i, (image_record, curr_embedding) in enumerate(zip(batched_prompts, image_embeddings)):
+            masks, low_res_masks = self.forward_prompts(image_record, curr_embedding, multimask_output, image_size, p_tune[i])
+            outputs.append(
+                masks
+            )
+            low_res_outputs.append(
+                low_res_masks
+            )
+        outputs = torch.cat(outputs, dim = 0)
+        low_res_outputs = torch.cat(low_res_outputs, dim = 0)
+        return outputs, low_res_outputs, image_embeddings
+    
+    def forward_prompts(self, image_record, curr_embedding, multimask_output, image_size, p_tune):
+        sam = self.sam
+        if "point_coords" in image_record:
+            points = (image_record["point_coords"], image_record["point_labels"])
+        else:
+            points = None
+        sparse_embeddings, dense_embeddings = sam.prompt_encoder(
+            points=points,
+            boxes=image_record.get("boxes", None),
+            masks=image_record.get("mask_inputs", None),
+        )
+        if self.p_tuning:
+            bs = sam.prompt_encoder._get_batch_size(points, image_record.get("boxes", None), image_record.get("mask_inputs", None))
+            tuning = sam.p(p_tune.unsqueeze(0).float()).reshape(
+                1, self.embed_dim, self.image_embedding_size[0], self.image_embedding_size[1]
+            ).expand(bs, -1, -1, -1)
+            dense_embeddings = dense_embeddings + tuning
+        low_res_masks, iou_predictions = sam.mask_decoder(
+            image_embeddings=curr_embedding.unsqueeze(0),
+            image_pe=sam.prompt_encoder.get_dense_pe(),
+            sparse_prompt_embeddings=sparse_embeddings,
+            dense_prompt_embeddings=dense_embeddings,
+            multimask_output=multimask_output,
+        )
+        masks = sam.postprocess_masks(
+            low_res_masks,
+            input_size=(image_size, image_size),
+            original_size=(image_size, image_size)
+        )
+        return masks, low_res_masks

CellPilot/cellpilot/modeling/model.py

@@ -0,0 +1,237 @@
+from .LoRA_Sam import LoRA_Sam
+from segment_anything import sam_model_registry
+import schedulefree
+from ..data_processing.data_utils import PromptProcessing
+import monai
+import random
+import torch
+import numpy as np
+import wandb
+from monai.metrics import compute_iou, compute_dice
+
+class SamHI (LoRA_Sam):
+    def __init__(self, config):
+        self.config = config
+        self.model_config = config["model_config"]
+        self.load_config()
+        super().__init__(sam_model_registry[self.model_type](checkpoint=self.model_dir + self.base_model), self.lora_rank, self.lora_layer, self.p_tuning)
+        self.model = self.sam
+        if self.model_mode == "train":
+            self.training_config = config["training_config"]
+            self.prompt_config = config["prompt_config"]
+            self.random_prompt_config = config["random_prompt_config"]
+            self.training_init()
+        
+
+    def load_config(self):
+        self.base_model = self.model_config["base_model"]
+        self.model_type = self.model_config["model_type"]
+        self.lora_layer = self.model_config["lora_layer"]
+        self.lora_rank = self.model_config["lora_rank"]
+        self.model_mode = self.model_config["model_mode"] 
+        self.model_dir = self.model_config["model_dir"] 
+        self.p_tuning = self.model_config.get("p_tuning", False)    
+    
+    def training_init(self):
+        self.load_training_config()
+        loss_dict = {
+            "diceCE": monai.losses.DiceCELoss(sigmoid=True),
+            "diceFocal": monai.losses.DiceFocalLoss(sigmoid=True),
+            "dice": monai.losses.DiceLoss(sigmoid=True),
+            "generalized_dice": monai.losses.GeneralizedDiceLoss(sigmoid=True),
+            "generalized_diceFocal": monai.losses.GeneralizedDiceFocalLoss(sigmoid=True),
+            "tversky": monai.losses.TverskyLoss(sigmoid=True),         
+        }
+        self.loss = loss_dict[self.loss]
+        self.save_hyperparameters()
+        self.freeze_parameters()
+
+
+    def load_training_config(self):
+        self.loss = self.training_config["loss"]
+        self.freeze = self.training_config["freeze"]
+        self.learning_rate = self.training_config["learning_rate"]
+        self.batch_size = self.training_config["batch_size"]
+        self.prompt_batch_size = self.training_config["prompt_batch_size"]
+        self.mode = self.training_config["mode"]
+        self.prompt_type = self.training_config["prompt_type"]
+        self.nr_of_interactive_points = self.training_config["nr_of_interactive_points"]
+        self.compile_model = self.training_config["compile_model"]
+        self.mask_threshold = self.training_config["mask_threshold"]
+    
+    def randomization(self):
+        if self.random_prompt_config["random_mode"]:
+            self.mode = random.choice(["random", "interactive"])
+        if self.random_prompt_config["random_prompt_type"]:
+            self.prompt_type = random.choice(["points", "boxes"])
+        if self.random_prompt_config["random_nr_of_interactive_points"]:
+            self.nr_of_interactive_points = random.randint(1, self.random_prompt_config["max_nr_of_interactive_points"])
+        if self.random_prompt_config["random_nr_of_points"]:
+            self.prompt_config["nr_of_points"] = random.randint(1, self.random_prompt_config["max_nr_of_points"])
+        if self.random_prompt_config["random_nr_of_positive_points"]:
+            self.prompt_config["nr_of_positive_points"] = random.randint(1, self.random_prompt_config["max_nr_of_positive_points"])
+        if self.random_prompt_config["only_positive_points"]:
+            self.prompt_config["nr_of_positive_points"] = self.prompt_config["nr_of_points"]
+
+
+    def freeze_parameters(self):
+        if "image" in self.freeze:
+            for param in self.model.image_encoder.parameters():
+                param.requires_grad = False
+        if "prompt" in self.freeze:
+            for param in self.model.prompt_encoder.parameters():
+                param.requires_grad = False
+        if "mask" in self.freeze:
+            for param in self.model.mask_decoder.parameters():
+                param.requires_grad = False
+
+    def forward(self, data, prompts, image_embeddings=None, p_tune=None):
+        return super().forward(data, prompts, False, 1024, image_embeddings, p_tune)
+    
+    def configure_optimizers(self):
+        optimizer = schedulefree.AdamWScheduleFree(filter(lambda p: p.requires_grad, self.model.parameters()), lr=self.learning_rate)
+        return optimizer  
+
+    def working_step(self, batch):
+        self.randomization()
+        idx, (data, target, nr, _, _, size), p_tune = batch 
+        prompts, target, _ = PromptProcessing.get_prompts_and_targets(nr, target, self.device, self.prompt_config) 
+        if self.mode == "interactive":
+            with torch.no_grad():
+                output, low_res_output, image_embeddings = self.forward(data, prompts, p_tune=p_tune)
+                if self.prompt_type == "both":
+                    box_output, _, _ = self.forward(data, prompts[int(len(prompts)/2):], image_embeddings, p_tune=p_tune)
+                    output = torch.cat([output, box_output], dim=0)
+                for i in range(self.nr_of_interactive_points): 
+                    prompts = PromptProcessing.refine_prompts(nr, target, prompts, output, self.device, self.prompt_batch_size)
+                    output, low_res_output, _ = self.forward(data, prompts, image_embeddings, p_tune=p_tune) 
+                    if self.prompt_type == "both":
+                        box_output, _, _ = self.forward(data, prompts[int(len(prompts)/2):], image_embeddings, p_tune=p_tune)
+                        output = torch.cat([output, box_output], dim=0)
+        output, _, image_embeddings = self.forward(data, prompts, p_tune=p_tune)
+        if self.prompt_type == "both":
+            box_output, _, _ = self.forward(data, prompts[int(len(prompts)/2):], image_embeddings, p_tune=p_tune)
+            output = torch.cat([output, box_output], dim=0)
+        return data, target, prompts, output
+
+    #@torch.compile
+    def compiled_working_step(self, batch):
+        return self.working_step(batch)
+
+    def unified_step(self, batch):
+        if self.compile_model:
+            return self.compiled_working_step(batch)
+        else: 
+            return self.working_step(batch)
+
+    def training_step(self, batch, batch_idx):
+        data, target, prompts, output = self.unified_step(batch)
+        loss = self.loss(output.squeeze().unsqueeze(1), target.float().squeeze().unsqueeze(1))
+        self.log("train_loss", loss, on_epoch=True, batch_size=self.batch_size)
+        return loss
+    
+    def validation_step(self, batch, batch_idx):
+        data, target, prompts, output = self.unified_step(batch)
+        loss = self.loss(output.squeeze().unsqueeze(1), target.float().squeeze().unsqueeze(1))
+        if batch_idx == 0:
+            self.display_samples(data, target, prompts, output, "val")
+        self.log("val_loss", loss, on_epoch=True, batch_size=self.batch_size)
+
+    def display_samples(self, data, target, prompts, output, mode):
+        img = []
+        output = output > self.mask_threshold
+        d = data.shape[0]
+        for i in range(d):
+            image = data[i].squeeze().cpu().numpy().transpose(1, 2, 0)
+            if self.prompt_type == "both":
+                image_masks = {
+                    "target": {"mask_data": target[(i) * self.prompt_batch_size].squeeze().cpu().numpy()},
+                    "output_box": {"mask_data": 2*output[(i) * self.prompt_batch_size].squeeze().cpu().numpy()},
+                    "output_point": {"mask_data": 3*output[(d + i) * self.prompt_batch_size].squeeze().cpu().numpy()}
+                }
+            else:
+                image_masks = {
+                    "target": {"mask_data": target[i * self.prompt_batch_size].squeeze().cpu().numpy()},
+                    "output": {"mask_data": 2*output[i * self.prompt_batch_size].squeeze().cpu().numpy()}
+                }
+            k = i
+            if self.prompt_type in ["points", "both"]:
+                points = prompts[k]['point_coords'][0]
+                point_mask = np.zeros((image.shape[0], image.shape[1]))
+                for (j, point) in enumerate(points):
+                    point = point.cpu().numpy()
+                    point_mask[int(point[1]) - 2:int(point[1]) + 2, int(point[0]) - 2:int(point[0]) + 2] = 4 + int(prompts[i]['point_labels'][0][j].item())
+                image_masks["point"] = {"mask_data": point_mask}
+            if self.prompt_type == "both":
+                k = d + i
+            if self.prompt_type in ["boxes", "both"]: 
+                boxes = prompts[k]['boxes'][0].squeeze().cpu().numpy()
+                box_mask = np.zeros((image.shape[0], image.shape[1]))
+                box_mask[int(boxes[1]):int(boxes[3]), int(boxes[0]):int(boxes[2])] = 6
+                image_masks["box"] = {"mask_data": box_mask}
+                if "point_coords" in prompts[k]:
+                    points = prompts[k]['point_coords'][0]
+                    point_mask = np.zeros((image.shape[0], image.shape[1]))
+                    for (j, point) in enumerate(points):
+                        point = point.cpu().numpy()
+                        point_mask[int(point[1]) - 2:int(point[1]) + 2, int(point[0]) - 2:int(point[0]) + 2] = 7 + int(prompts[k]['point_labels'][0][j].item())
+                    image_masks["box_point"] = {"mask_data": point_mask}
+            img.append(wandb.Image(image, masks=image_masks))
+        self.trainer.logger.experiment.log({mode + "_samples": img})
+
+    def test_step(self, batch, batch_idx):
+        data, target, prompts, output = self.unified_step(batch)
+        if batch_idx == 0:
+            self.display_samples(data, target, prompts, output, "test")
+        output = output > self.mask_threshold
+        if self.prompt_type == "both":
+            point_iou = compute_iou(output[:int(len(output)/2)], target[:int(len(output)/2)].unsqueeze(1))
+            point_dice = compute_dice(output[:int(len(output)/2)], target[:int(len(output)/2)].unsqueeze(1))
+            box_iou = compute_iou(output[int(len(output)/2):], target[int(len(output)/2):].unsqueeze(1))
+            box_dice = compute_dice(output[int(len(output)/2):], target[int(len(output)/2):].unsqueeze(1))
+            self.log("test_point_iou", torch.mean(point_iou), on_epoch=True, batch_size=self.batch_size)
+            self.log("test_point_dice", torch.mean(point_dice), on_epoch=True, batch_size=self.batch_size)
+            self.log("test_point_iou_std", torch.std(point_iou), on_epoch=True, batch_size=self.batch_size)
+            self.log("test_point_dice_std", torch.std(point_dice), on_epoch=True, batch_size=self.batch_size)
+            self.log("test_box_iou", torch.mean(box_iou), on_epoch=True, batch_size=self.batch_size)
+            self.log("test_box_dice", torch.mean(box_dice), on_epoch=True, batch_size=self.batch_size)
+            self.log("test_box_iou_std", torch.std(box_iou), on_epoch=True, batch_size=self.batch_size)
+            self.log("test_box_dice_std", torch.std(box_dice), on_epoch=True, batch_size=self.batch_size)
+        else:
+            iou = compute_iou(output, target.unsqueeze(1))
+            dice = compute_dice(output, target.unsqueeze(1))
+            self.log("test_iou", torch.mean(iou), on_epoch=True, batch_size=self.batch_size)
+            self.log("test_dice", torch.mean(dice), on_epoch=True, batch_size=self.batch_size)
+            self.log("test_iou_std", torch.std(iou), on_epoch=True, batch_size=self.batch_size)
+            self.log("test_dice_std", torch.std(dice), on_epoch=True, batch_size=self.batch_size)
+    
+
+
+
+
+    # make schedulefree work
+    def on_fit_start(self) -> None:
+        self.optimizers().train()
+
+    def on_predict_start(self) -> None:
+        self.optimizers().eval()
+        
+    def on_validation_model_eval(self) -> None:
+        self.model.eval()
+        self.optimizers().eval()
+
+    def on_validation_model_train(self) -> None:
+        self.model.train()
+        self.optimizers().train()
+
+    def on_test_model_eval(self) -> None:
+        self.model.eval()
+        #self.optimizers().eval()
+
+    def on_test_model_train(self) -> None:
+        self.model.train()
+        self.optimizers().train()
+
+    def on_predict_model_eval(self) -> None:  # redundant with on_predict_start()
+        self.model.eval()
+        self.optimizers().eval()

CellPilot/cellpilot/modeling/predictor.py

@@ -0,0 +1,174 @@
+from segment_anything import SamPredictor
+import torch
+import numpy as np
+from typing import Optional, Tuple
+
+class SamHIPredictor(SamPredictor):
+    def __init__(self, model, p_tuning=False):
+        self.p_tuning = p_tuning
+        super().__init__(model)
+
+    def predict(
+        self,
+        point_coords: Optional[np.ndarray] = None,
+        point_labels: Optional[np.ndarray] = None,
+        box: Optional[np.ndarray] = None,
+        p_tune: Optional[np.ndarray] = None,
+        mask_input: Optional[np.ndarray] = None,
+        multimask_output: bool = True,
+        return_logits: bool = False,
+    ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
+        """
+        Predict masks for the given input prompts, using the currently set image.
+
+        Arguments:
+          point_coords (np.ndarray or None): A Nx2 array of point prompts to the
+            model. Each point is in (X,Y) in pixels.
+          point_labels (np.ndarray or None): A length N array of labels for the
+            point prompts. 1 indicates a foreground point and 0 indicates a
+            background point.
+          box (np.ndarray or None): A length 4 array given a box prompt to the
+            model, in XYXY format.
+          mask_input (np.ndarray): A low resolution mask input to the model, typically
+            coming from a previous prediction iteration. Has form 1xHxW, where
+            for SAM, H=W=256.
+          multimask_output (bool): If true, the model will return three masks.
+            For ambiguous input prompts (such as a single click), this will often
+            produce better masks than a single prediction. If only a single
+            mask is needed, the model's predicted quality score can be used
+            to select the best mask. For non-ambiguous prompts, such as multiple
+            input prompts, multimask_output=False can give better results.
+          return_logits (bool): If true, returns un-thresholded masks logits
+            instead of a binary mask.
+
+        Returns:
+          (np.ndarray): The output masks in CxHxW format, where C is the
+            number of masks, and (H, W) is the original image size.
+          (np.ndarray): An array of length C containing the model's
+            predictions for the quality of each mask.
+          (np.ndarray): An array of shape CxHxW, where C is the number
+            of masks and H=W=256. These low resolution logits can be passed to
+            a subsequent iteration as mask input.
+        """
+        if not self.is_image_set:
+            raise RuntimeError("An image must be set with .set_image(...) before mask prediction.")
+
+        # Transform input prompts
+        coords_torch, labels_torch, box_torch, mask_input_torch = None, None, None, None
+        if point_coords is not None:
+            assert (
+                point_labels is not None
+            ), "point_labels must be supplied if point_coords is supplied."
+            point_coords = self.transform.apply_coords(point_coords, self.original_size)
+            coords_torch = torch.as_tensor(point_coords, dtype=torch.float, device=self.device)
+            labels_torch = torch.as_tensor(point_labels, dtype=torch.int, device=self.device)
+            coords_torch, labels_torch = coords_torch[None, :, :], labels_torch[None, :]
+        if box is not None:
+            box = self.transform.apply_boxes(box, self.original_size)
+            box_torch = torch.as_tensor(box, dtype=torch.float, device=self.device)
+            box_torch = box_torch[None, :]
+        if mask_input is not None:
+            mask_input_torch = torch.as_tensor(mask_input, dtype=torch.float, device=self.device)
+            mask_input_torch = mask_input_torch[None, :, :, :]
+        if p_tune is not None:
+            p_tune = torch.as_tensor(p_tune, dtype=torch.float, device=self.device)
+            p_tune = p_tune[None, :]
+
+        masks, iou_predictions, low_res_masks = self.predict_torch(
+            coords_torch,
+            labels_torch,
+            box_torch,
+            p_tune,
+            mask_input_torch,
+            multimask_output,
+            return_logits=return_logits,
+        )
+
+        masks_np = masks[0].detach().cpu().numpy()
+        iou_predictions_np = iou_predictions[0].detach().cpu().numpy()
+        low_res_masks_np = low_res_masks[0].detach().cpu().numpy()
+        return masks_np, iou_predictions_np, low_res_masks_np
+    
+    @torch.no_grad()
+    def predict_torch(
+        self,
+        point_coords: Optional[torch.Tensor],
+        point_labels: Optional[torch.Tensor],
+        boxes: Optional[torch.Tensor] = None,
+        p_tune: Optional[torch.Tensor] = None,
+        mask_input: Optional[torch.Tensor] = None,
+        multimask_output: bool = True,
+        return_logits: bool = False,
+    ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+        """
+        Predict masks for the given input prompts, using the currently set image.
+        Input prompts are batched torch tensors and are expected to already be
+        transformed to the input frame using ResizeLongestSide.
+
+        Arguments:
+          point_coords (torch.Tensor or None): A BxNx2 array of point prompts to the
+            model. Each point is in (X,Y) in pixels.
+          point_labels (torch.Tensor or None): A BxN array of labels for the
+            point prompts. 1 indicates a foreground point and 0 indicates a
+            background point.
+          boxes (np.ndarray or None): A Bx4 array given a box prompt to the
+            model, in XYXY format.
+          mask_input (np.ndarray): A low resolution mask input to the model, typically
+            coming from a previous prediction iteration. Has form Bx1xHxW, where
+            for SAM, H=W=256. Masks returned by a previous iteration of the
+            predict method do not need further transformation.
+          multimask_output (bool): If true, the model will return three masks.
+            For ambiguous input prompts (such as a single click), this will often
+            produce better masks than a single prediction. If only a single
+            mask is needed, the model's predicted quality score can be used
+            to select the best mask. For non-ambiguous prompts, such as multiple
+            input prompts, multimask_output=False can give better results.
+          return_logits (bool): If true, returns un-thresholded masks logits
+            instead of a binary mask.
+
+        Returns:
+          (torch.Tensor): The output masks in BxCxHxW format, where C is the
+            number of masks, and (H, W) is the original image size.
+          (torch.Tensor): An array of shape BxC containing the model's
+            predictions for the quality of each mask.
+          (torch.Tensor): An array of shape BxCxHxW, where C is the number
+            of masks and H=W=256. These low res logits can be passed to
+            a subsequent iteration as mask input.
+        """
+        if not self.is_image_set:
+            raise RuntimeError("An image must be set with .set_image(...) before mask prediction.")
+
+        if point_coords is not None:
+            points = (point_coords, point_labels)
+        else:
+            points = None
+
+        # Embed prompts
+        sparse_embeddings, dense_embeddings = self.model.prompt_encoder(
+            points=points,
+            boxes=boxes,
+            masks=mask_input,
+        )
+        if self.p_tuning:
+            bs = self.model.prompt_encoder._get_batch_size(points, boxes, mask_input)
+            tuning = self.model.p(p_tune.unsqueeze(0).float()).reshape(
+                1, self.embed_dim, self.image_embedding_size[0], self.image_embedding_size[1]
+            ).expand(bs, -1, -1, -1)
+            dense_embeddings = dense_embeddings + tuning
+
+        # Predict masks
+        low_res_masks, iou_predictions = self.model.mask_decoder(
+            image_embeddings=self.features,
+            image_pe=self.model.prompt_encoder.get_dense_pe(),
+            sparse_prompt_embeddings=sparse_embeddings,
+            dense_prompt_embeddings=dense_embeddings,
+            multimask_output=multimask_output,
+        )
+
+        # Upscale the masks to the original image resolution
+        masks = self.model.postprocess_masks(low_res_masks, self.input_size, self.original_size)
+
+        if not return_logits:
+            masks = masks > self.model.mask_threshold
+
+        return masks, iou_predictions, low_res_masks

CellPilot/environment.yml

@@ -0,0 +1,344 @@
+name: histo3.10
+channels:
+  - pytorch
+  - nvidia
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - anyio=4.3.0=pyhd8ed1ab_0
+  - argon2-cffi=23.1.0=pyhd8ed1ab_0
+  - argon2-cffi-bindings=21.2.0=py310h2372a71_4
+  - arrow=1.3.0=pyhd8ed1ab_0
+  - asttokens=2.4.1=pyhd8ed1ab_0
+  - async-lru=2.0.4=pyhd8ed1ab_0
+  - attrs=23.2.0=pyh71513ae_0
+  - babel=2.14.0=pyhd8ed1ab_0
+  - beautifulsoup4=4.12.3=pyha770c72_0
+  - blas=1.0=mkl
+  - bleach=6.1.0=pyhd8ed1ab_0
+  - brotli-python=1.0.9=py310hd8f1fbe_7
+  - bzip2=1.0.8=hd590300_5
+  - ca-certificates=2024.3.11=h06a4308_0
+  - cached-property=1.5.2=hd8ed1ab_1
+  - cached_property=1.5.2=pyha770c72_1
+  - cairo=1.18.0=h3faef2a_0
+  - certifi=2024.2.2=pyhd8ed1ab_0
+  - cffi=1.16.0=py310h2fee648_0
+  - charset-normalizer=3.3.2=pyhd8ed1ab_0
+  - colorama=0.4.6=pyhd8ed1ab_0
+  - comm=0.2.2=pyhd8ed1ab_0
+  - cuda-cudart=12.1.105=0
+  - cuda-cupti=12.1.105=0
+  - cuda-libraries=12.1.0=0
+  - cuda-nvrtc=12.1.105=0
+  - cuda-nvtx=12.1.105=0
+  - cuda-opencl=12.4.127=0
+  - cuda-runtime=12.1.0=0
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - defusedxml=0.7.1=pyhd8ed1ab_0
+  - entrypoints=0.4=pyhd8ed1ab_0
+  - exceptiongroup=1.2.0=pyhd8ed1ab_2
+  - executing=2.0.1=pyhd8ed1ab_0
+  - expat=2.6.2=h6a678d5_0
+  - ffmpeg=4.3=hf484d3e_0
+  - font-ttf-dejavu-sans-mono=2.37=hd3eb1b0_0
+  - font-ttf-inconsolata=2.001=hcb22688_0
+  - font-ttf-source-code-pro=2.030=hd3eb1b0_0
+  - font-ttf-ubuntu=0.83=h8b1ccd4_0
+  - fontconfig=2.14.2=h14ed4e7_0
+  - fonts-anaconda=1=h8fa9717_0
+  - fonts-conda-ecosystem=1=hd3eb1b0_0
+  - fqdn=1.5.1=pyhd8ed1ab_0
+  - freetype=2.12.1=h4a9f257_0
+  - gdk-pixbuf=2.42.11=hb9ae30d_0
+  - gmp=6.2.1=h295c915_3
+  - gmpy2=2.1.2=py310heeb90bb_0
+  - gnutls=3.6.15=he1e5248_0
+  - h11=0.14.0=pyhd8ed1ab_0
+  - h2=4.1.0=pyhd8ed1ab_0
+  - hpack=4.0.0=pyh9f0ad1d_0
+  - httpcore=1.0.5=pyhd8ed1ab_0
+  - httpx=0.27.0=pyhd8ed1ab_0
+  - hyperframe=6.0.1=pyhd8ed1ab_0
+  - icu=73.2=h59595ed_0
+  - idna=3.7=pyhd8ed1ab_0
+  - importlib_metadata=7.1.0=hd8ed1ab_0
+  - importlib_resources=6.4.0=pyhd8ed1ab_0
+  - intel-openmp=2023.1.0=hdb19cb5_46306
+  - ipykernel=6.29.3=pyhd33586a_0
+  - ipython=8.22.2=pyh707e725_0
+  - isoduration=20.11.0=pyhd8ed1ab_0
+  - jedi=0.19.1=pyhd8ed1ab_0
+  - jinja2=3.1.3=pyhd8ed1ab_0
+  - json5=0.9.25=pyhd8ed1ab_0
+  - jsonpointer=2.4=py310hff52083_3
+  - jsonschema=4.21.1=pyhd8ed1ab_0
+  - jsonschema-specifications=2023.12.1=pyhd8ed1ab_0
+  - jsonschema-with-format-nongpl=4.21.1=pyhd8ed1ab_0
+  - jupyter-lsp=2.2.5=pyhd8ed1ab_0
+  - jupyter_client=8.6.1=pyhd8ed1ab_0
+  - jupyter_core=5.7.2=py310hff52083_0
+  - jupyter_events=0.10.0=pyhd8ed1ab_0
+  - jupyter_server=2.14.0=pyhd8ed1ab_0
+  - jupyter_server_terminals=0.5.3=pyhd8ed1ab_0
+  - jupyterlab=4.1.6=pyhd8ed1ab_0
+  - jupyterlab_pygments=0.3.0=pyhd8ed1ab_1
+  - jupyterlab_server=2.27.1=pyhd8ed1ab_0
+  - lame=3.100=h7b6447c_0
+  - lcms2=2.16=hb7c19ff_0
+  - ld_impl_linux-64=2.40=h55db66e_0
+  - lerc=4.0.0=h27087fc_0
+  - libcublas=12.1.0.26=0
+  - libcufft=11.0.2.4=0
+  - libcufile=1.9.1.3=0
+  - libcurand=10.3.5.147=0
+  - libcusolver=11.4.4.55=0
+  - libcusparse=12.0.2.55=0
+  - libdeflate=1.20=hd590300_0
+  - libdicom=1.0.5=hd590300_1
+  - libffi=3.4.4=h6a678d5_0
+  - libgcc-ng=13.2.0=hc881cc4_6
+  - libglib=2.80.0=hf2295e7_6
+  - libgomp=13.2.0=hc881cc4_6
+  - libiconv=1.17=hd590300_2
+  - libidn2=2.3.4=h5eee18b_0
+  - libjpeg-turbo=3.0.0=hd590300_1
+  - libnpp=12.0.2.50=0
+  - libnsl=2.0.1=hd590300_0
+  - libnvjitlink=12.1.105=0
+  - libnvjpeg=12.1.1.14=0
+  - libpng=1.6.43=h2797004_0
+  - libsodium=1.0.18=h36c2ea0_1
+  - libsqlite=3.45.3=h2797004_0
+  - libstdcxx-ng=13.2.0=h95c4c6d_6
+  - libtasn1=4.19.0=h5eee18b_0
+  - libtiff=4.6.0=h1dd3fc0_3
+  - libunistring=0.9.10=h27cfd23_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libwebp-base=1.3.2=h5eee18b_0
+  - libxcb=1.15=h7f8727e_0
+  - libxcrypt=4.4.36=hd590300_1
+  - libxml2=2.12.6=h232c23b_2
+  - libzlib=1.2.13=hd590300_5
+  - lightning=2.2.5=pyhd8ed1ab_0
+  - lightning-utilities=0.11.2=pyhd8ed1ab_0
+  - llvm-openmp=14.0.6=h9e868ea_0
+  - lz4-c=1.9.4=h6a678d5_0
+  - markupsafe=2.1.5=py310h2372a71_0
+  - matplotlib-inline=0.1.7=pyhd8ed1ab_0
+  - mistune=3.0.2=pyhd8ed1ab_0
+  - mkl=2023.1.0=h213fc3f_46344
+  - mkl-service=2.4.0=py310h5eee18b_1
+  - mkl_fft=1.3.8=py310h5eee18b_0
+  - mkl_random=1.2.4=py310hdb19cb5_0
+  - mpc=1.1.0=h10f8cd9_1
+  - mpfr=4.0.2=hb69a4c5_1
+  - mpmath=1.3.0=py310h06a4308_0
+  - nbclient=0.10.0=pyhd8ed1ab_0
+  - nbconvert-core=7.16.3=pyhd8ed1ab_1
+  - nbformat=5.10.4=pyhd8ed1ab_0
+  - ncurses=6.4.20240210=h59595ed_0
+  - nest-asyncio=1.6.0=pyhd8ed1ab_0
+  - nettle=3.7.3=hbbd107a_1
+  - networkx=3.1=py310h06a4308_0
+  - notebook-shim=0.2.4=pyhd8ed1ab_0
+  - numpy=1.26.4=py310h5f9d8c6_0
+  - numpy-base=1.26.4=py310hb5e798b_0
+  - openh264=2.1.1=h4ff587b_0
+  - openjpeg=2.5.2=h488ebb8_0
+  - openslide=4.0.0=h58ba908_1
+  - openslide-python=1.3.1=py310h091d076_2
+  - openssl=3.3.0=h4ab18f5_3
+  - overrides=7.7.0=pyhd8ed1ab_0
+  - pandocfilters=1.5.0=pyhd8ed1ab_0
+  - parso=0.8.4=pyhd8ed1ab_0
+  - pcre2=10.43=hcad00b1_0
+  - pexpect=4.9.0=pyhd8ed1ab_0
+  - pickleshare=0.7.5=py_1003
+  - pillow=10.3.0=py310hf73ecf8_0
+  - pip=24.0=pyhd8ed1ab_0
+  - pixman=0.43.2=h59595ed_0
+  - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1
+  - platformdirs=4.2.1=pyhd8ed1ab_0
+  - prometheus_client=0.20.0=pyhd8ed1ab_0
+  - prompt-toolkit=3.0.42=pyha770c72_0
+  - psutil=5.9.8=py310h2372a71_0
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pure_eval=0.2.2=pyhd8ed1ab_0
+  - pycparser=2.22=pyhd8ed1ab_0
+  - pygments=2.17.2=pyhd8ed1ab_0
+  - pysocks=1.7.1=pyha2e5f31_6
+  - python=3.10.14=hd12c33a_0_cpython
+  - python-dateutil=2.9.0=pyhd8ed1ab_0
+  - python-fastjsonschema=2.19.1=pyhd8ed1ab_0
+  - python-json-logger=2.0.7=pyhd8ed1ab_0
+  - python_abi=3.10=2_cp310
+  - pytorch=2.3.0=py3.10_cuda12.1_cudnn8.9.2_0
+  - pytorch-cuda=12.1=ha16c6d3_5
+  - pytorch-lightning=2.2.2=pyhd8ed1ab_0
+  - pytorch-mutex=1.0=cuda
+  - pyyaml=6.0.1=py310h2372a71_1
+  - readline=8.2=h8228510_1
+  - referencing=0.34.0=pyhd8ed1ab_0
+  - rfc3339-validator=0.1.4=pyhd8ed1ab_0
+  - rfc3986-validator=0.1.1=pyh9f0ad1d_0
+  - rpds-py=0.18.0=py310hcb5633a_0
+  - send2trash=1.8.3=pyh0d859eb_0
+  - six=1.16.0=pyh6c4a22f_0
+  - sniffio=1.3.1=pyhd8ed1ab_0
+  - soupsieve=2.5=pyhd8ed1ab_1
+  - sqlite=3.45.3=h2c6b66d_0
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - sympy=1.12=py310h06a4308_0
+  - tbb=2021.8.0=hdb19cb5_0
+  - terminado=0.18.1=pyh0d859eb_0
+  - tinycss2=1.2.1=pyhd8ed1ab_0
+  - tk=8.6.13=noxft_h4845f30_101
+  - tomli=2.0.1=pyhd8ed1ab_0
+  - torchaudio=2.3.0=py310_cu121
+  - torchmetrics=1.4.0.post0=pyhd8ed1ab_0
+  - torchtriton=2.3.0=py310
+  - torchvision=0.18.0=py310_cu121
+  - tornado=6.4=py310h2372a71_0
+  - traitlets=5.14.3=pyhd8ed1ab_0
+  - types-python-dateutil=2.9.0.20240316=pyhd8ed1ab_0
+  - typing_extensions=4.11.0=pyha770c72_0
+  - typing_utils=0.1.0=pyhd8ed1ab_0
+  - uri-template=1.3.0=pyhd8ed1ab_0
+  - wcwidth=0.2.13=pyhd8ed1ab_0
+  - webcolors=1.13=pyhd8ed1ab_0
+  - webencodings=0.5.1=pyhd8ed1ab_2
+  - websocket-client=1.8.0=pyhd8ed1ab_0
+  - wheel=0.43.0=pyhd8ed1ab_1
+  - xorg-kbproto=1.0.7=h7f98852_1002
+  - xorg-libice=1.1.1=hd590300_0
+  - xorg-libsm=1.2.4=h7391055_0
+  - xorg-libx11=1.8.9=h8ee46fc_0
+  - xorg-libxext=1.3.4=h0b41bf4_2
+  - xorg-libxrender=0.9.11=hd590300_0
+  - xorg-renderproto=0.11.1=h7f98852_1002
+  - xorg-xextproto=7.3.0=h0b41bf4_1003
+  - xorg-xproto=7.0.31=h27cfd23_1007
+  - xz=5.4.6=h5eee18b_0
+  - yaml=0.2.5=h7f98852_2
+  - zeromq=4.3.5=h6a678d5_0
+  - zipp=3.17.0=pyhd8ed1ab_0
+  - zlib=1.2.13=hd590300_5
+  - zstd=1.5.5=hc292b87_0
+  - pip:
+      - absl-py==2.1.0
+      - addict==2.4.0
+      - aiofiles==23.2.1
+      - albucore==0.0.12
+      - albumentations==1.4.11
+      - aliyun-python-sdk-core==2.15.1
+      - aliyun-python-sdk-kms==2.16.3
+      - altair==5.3.0
+      - annotated-types==0.7.0
+      - asciitree==0.3.3
+      - asyncio-atexit==1.0.1
+      - backoff==2.2.1
+      - click==8.1.7
+      - cloudpickle==3.0.0
+      - crcmod==1.7
+      - cryptography==43.0.0
+      - debugpy==1.8.1
+      - deprecated==1.2.14
+      - dnspython==2.6.1
+      - docker-pycreds==0.4.0
+      - email-validator==2.1.1
+      - eval-type-backport==0.2.0
+      - fastapi==0.111.0
+      - fastapi-cli==0.0.4
+      - fasteners==0.19
+      - ffmpy==0.3.2
+      - filelock==3.14.0
+      - fsspec==2024.3.1
+      - gdown==5.1.0
+      - googleapis-common-protos==1.63.0
+      - gradio==4.36.1
+      - gradio-client==1.0.1
+      - gradio-image-prompter==0.1.0
+      - grpcio==1.65.1
+      - h5py==3.11.0
+      - httptools==0.6.1
+      - huggingface-hub==0.23.3
+      - imagecodecs==2024.1.1
+      - importlib-metadata==6.11.0
+      - jmespath==0.10.0
+      - jupyter-capture-output==0.0.11
+      - loky==3.0.0
+      - markdown==3.6
+      - markdown-it-py==3.0.0
+      - mdurl==0.1.2
+      - mmcv==1.6.2
+      - mmengine==0.10.4
+      - model-index==0.1.11
+      - monai==1.3.1
+      - numcodecs==0.12.1
+      - nvidia-nccl-cu12==2.20.5
+      - opencv-python==4.9.0.80
+      - opencv-python-headless==4.10.0.84
+      - opendatalab==0.0.10
+      - openmim==0.3.9
+      - opentelemetry-api==1.21.0
+      - opentelemetry-exporter-otlp-proto-common==1.21.0
+      - opentelemetry-exporter-otlp-proto-http==1.21.0
+      - opentelemetry-proto==1.21.0
+      - opentelemetry-sdk==1.21.0
+      - opentelemetry-semantic-conventions==0.42b0
+      - openxlab==0.1.1
+      - ordered-set==4.1.0
+      - orjson==3.10.4
+      - oss2==2.17.0
+      - packaging==24.1
+      - pandas==2.2.2
+      - protobuf==4.25.3
+      - pycocotools==2.0.7
+      - pycryptodome==3.20.0
+      - pydantic==2.7.4
+      - pydantic-core==2.18.4
+      - pydub==0.25.1
+      - pyparsing==3.1.2
+      - python-multipart==0.0.9
+      - pytz==2023.4
+      - pyzmq==26.0.2
+      - requests==2.28.2
+      - rich==13.4.2
+      - ruff==0.4.8
+      - schedulefree==1.2.5
+      - segment-anything==1.0
+      - semantic-version==2.10.0
+      - sentry-sdk==2.2.1
+      - setproctitle==1.3.3
+      - setuptools==60.2.0
+      - shapely==2.0.4
+      - shellingham==1.5.4
+      - slideio==2.5.0
+      - starlette==0.37.2
+      - synapseclient==4.2.0
+      - tensorboard==2.17.0
+      - tensorboard-data-server==0.7.2
+      - termcolor==2.4.0
+      - tifffile==2024.4.24
+      - timm==0.6.11
+      - tomlkit==0.12.0
+      - toolz==0.12.1
+      - tqdm==4.65.2
+      - triton==2.3.0
+      - typer==0.12.3
+      - typing-extensions==4.12.2
+      - tzdata==2024.1
+      - ujson==5.10.0
+      - urllib3==1.26.19
+      - uvicorn==0.30.1
+      - uvloop==0.19.0
+      - wandb==0.17.0
+      - watchfiles==0.22.0
+      - websockets==11.0.3
+      - werkzeug==3.0.3
+      - yapf==0.40.2
+      - zarr==2.17.2
+prefix: /vol/data/miniconda3/envs/histo3.10

CellPilot/notebooks/wandb/debug.log

@@ -0,0 +1,57 @@
+2024-06-13 12:16:20,040 INFO    MainThread:63330 [wandb_setup.py:_flush():76] Current SDK version is 0.17.0
+2024-06-13 12:16:20,041 INFO    MainThread:63330 [wandb_setup.py:_flush():76] Configure stats pid to 63330
+2024-06-13 12:16:20,041 INFO    MainThread:63330 [wandb_setup.py:_flush():76] Loading settings from /home/ubuntu/.config/wandb/settings
+2024-06-13 12:16:20,041 INFO    MainThread:63330 [wandb_setup.py:_flush():76] Loading settings from /home/ubuntu/thesis/SAMHI/notebooks/wandb/settings
+2024-06-13 12:16:20,041 INFO    MainThread:63330 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-06-13 12:16:20,042 INFO    MainThread:63330 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-06-13 12:16:20,042 INFO    MainThread:63330 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program': '<python with no main file>'}
+2024-06-13 12:16:20,042 INFO    MainThread:63330 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-06-13 12:16:20,042 INFO    MainThread:63330 [wandb_init.py:_log_setup():520] Logging user logs to /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240613_121620-ytzflyds/logs/debug.log
+2024-06-13 12:16:20,043 INFO    MainThread:63330 [wandb_init.py:_log_setup():521] Logging internal logs to /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240613_121620-ytzflyds/logs/debug-internal.log
+2024-06-13 12:16:20,043 INFO    MainThread:63330 [wandb_init.py:_jupyter_setup():466] configuring jupyter hooks <wandb.sdk.wandb_init._WandbInit object at 0x7f0e76940100>
+2024-06-13 12:16:20,044 INFO    MainThread:63330 [wandb_init.py:init():560] calling init triggers
+2024-06-13 12:16:20,044 INFO    MainThread:63330 [wandb_init.py:init():567] wandb.init called with sweep_config: {}
+config: {}
+2024-06-13 12:16:20,044 INFO    MainThread:63330 [wandb_init.py:init():610] starting backend
+2024-06-13 12:16:20,045 INFO    MainThread:63330 [wandb_init.py:init():614] setting up manager
+2024-06-13 12:16:20,047 INFO    MainThread:63330 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-06-13 12:16:20,049 INFO    MainThread:63330 [wandb_init.py:init():622] backend started and connected
+2024-06-13 12:16:20,062 INFO    MainThread:63330 [wandb_run.py:_label_probe_notebook():1328] probe notebook
+2024-06-13 12:16:20,063 INFO    MainThread:63330 [wandb_run.py:_label_probe_notebook():1338] Unable to probe notebook: 'NoneType' object has no attribute 'get'
+2024-06-13 12:16:20,064 INFO    MainThread:63330 [wandb_init.py:init():711] updated telemetry
+2024-06-13 12:16:20,073 INFO    MainThread:63330 [wandb_init.py:init():744] communicating run to backend with 90.0 second timeout
+2024-06-13 12:16:20,512 INFO    MainThread:63330 [wandb_run.py:_on_init():2396] communicating current version
+2024-06-13 12:16:20,604 INFO    MainThread:63330 [wandb_run.py:_on_init():2405] got version response upgrade_message: "wandb version 0.17.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-06-13 12:16:20,605 INFO    MainThread:63330 [wandb_init.py:init():795] starting run threads in backend
+2024-06-13 12:16:23,677 INFO    MainThread:63330 [wandb_run.py:_console_start():2374] atexit reg
+2024-06-13 12:16:23,677 INFO    MainThread:63330 [wandb_run.py:_redirect():2229] redirect: wrap_raw
+2024-06-13 12:16:23,679 INFO    MainThread:63330 [wandb_run.py:_redirect():2294] Wrapping output streams.
+2024-06-13 12:16:23,679 INFO    MainThread:63330 [wandb_run.py:_redirect():2319] Redirects installed.
+2024-06-13 12:16:23,680 INFO    MainThread:63330 [wandb_init.py:init():838] run started, returning control to user process
+2024-06-13 12:16:27,183 INFO    MainThread:63330 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-13 12:16:27,184 INFO    MainThread:63330 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-13 12:17:17,377 INFO    MainThread:63330 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-13 12:17:17,380 INFO    MainThread:63330 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-13 12:17:17,381 INFO    MainThread:63330 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-13 12:17:28,676 INFO    MainThread:63330 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-13 12:17:33,381 INFO    MainThread:63330 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-13 12:17:33,381 INFO    MainThread:63330 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-13 14:18:52,866 INFO    MainThread:63330 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-13 14:18:52,876 INFO    MainThread:63330 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-13 14:18:52,881 INFO    MainThread:63330 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-13 14:18:53,005 INFO    MainThread:63330 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-13 14:18:53,012 INFO    MainThread:63330 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-13 14:18:53,012 INFO    MainThread:63330 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-13 14:18:53,103 INFO    MainThread:63330 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-13 14:18:53,108 INFO    MainThread:63330 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-13 14:18:53,108 INFO    MainThread:63330 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-13 14:18:53,305 INFO    MainThread:63330 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-13 14:18:53,312 INFO    MainThread:63330 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-13 14:18:53,313 INFO    MainThread:63330 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-13 14:18:54,944 INFO    MainThread:63330 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-13 14:18:54,959 INFO    MainThread:63330 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-13 14:18:54,962 INFO    MainThread:63330 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-13 14:18:55,063 INFO    MainThread:63330 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-13 14:18:55,068 INFO    MainThread:63330 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-13 14:18:55,068 INFO    MainThread:63330 [wandb_init.py:_pause_backend():431] pausing backend

CellPilot/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/conda-environment.yaml

@@ -0,0 +1,316 @@
+name: histo3.10
+channels:
+  - pytorch
+  - nvidia
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - anyio=4.3.0=pyhd8ed1ab_0
+  - argon2-cffi=23.1.0=pyhd8ed1ab_0
+  - argon2-cffi-bindings=21.2.0=py310h2372a71_4
+  - arrow=1.3.0=pyhd8ed1ab_0
+  - asttokens=2.4.1=pyhd8ed1ab_0
+  - async-lru=2.0.4=pyhd8ed1ab_0
+  - attrs=23.2.0=pyh71513ae_0
+  - babel=2.14.0=pyhd8ed1ab_0
+  - beautifulsoup4=4.12.3=pyha770c72_0
+  - blas=1.0=mkl
+  - bleach=6.1.0=pyhd8ed1ab_0
+  - brotli-python=1.0.9=py310hd8f1fbe_7
+  - bzip2=1.0.8=hd590300_5
+  - ca-certificates=2024.3.11=h06a4308_0
+  - cached-property=1.5.2=hd8ed1ab_1
+  - cached_property=1.5.2=pyha770c72_1
+  - cairo=1.18.0=h3faef2a_0
+  - certifi=2024.2.2=pyhd8ed1ab_0
+  - cffi=1.16.0=py310h2fee648_0
+  - charset-normalizer=3.3.2=pyhd8ed1ab_0
+  - colorama=0.4.6=pyhd8ed1ab_0
+  - comm=0.2.2=pyhd8ed1ab_0
+  - cuda-cudart=12.1.105=0
+  - cuda-cupti=12.1.105=0
+  - cuda-libraries=12.1.0=0
+  - cuda-nvrtc=12.1.105=0
+  - cuda-nvtx=12.1.105=0
+  - cuda-opencl=12.4.127=0
+  - cuda-runtime=12.1.0=0
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - defusedxml=0.7.1=pyhd8ed1ab_0
+  - entrypoints=0.4=pyhd8ed1ab_0
+  - exceptiongroup=1.2.0=pyhd8ed1ab_2
+  - executing=2.0.1=pyhd8ed1ab_0
+  - expat=2.6.2=h6a678d5_0
+  - ffmpeg=4.3=hf484d3e_0
+  - filelock=3.13.1=py310h06a4308_0
+  - font-ttf-dejavu-sans-mono=2.37=hd3eb1b0_0
+  - font-ttf-inconsolata=2.001=hcb22688_0
+  - font-ttf-source-code-pro=2.030=hd3eb1b0_0
+  - font-ttf-ubuntu=0.83=h8b1ccd4_0
+  - fontconfig=2.14.2=h14ed4e7_0
+  - fonts-anaconda=1=h8fa9717_0
+  - fonts-conda-ecosystem=1=hd3eb1b0_0
+  - fqdn=1.5.1=pyhd8ed1ab_0
+  - freetype=2.12.1=h4a9f257_0
+  - gdk-pixbuf=2.42.11=hb9ae30d_0
+  - gmp=6.2.1=h295c915_3
+  - gmpy2=2.1.2=py310heeb90bb_0
+  - gnutls=3.6.15=he1e5248_0
+  - h11=0.14.0=pyhd8ed1ab_0
+  - h2=4.1.0=pyhd8ed1ab_0
+  - hpack=4.0.0=pyh9f0ad1d_0
+  - httpcore=1.0.5=pyhd8ed1ab_0
+  - httpx=0.27.0=pyhd8ed1ab_0
+  - hyperframe=6.0.1=pyhd8ed1ab_0
+  - icu=73.2=h59595ed_0
+  - idna=3.7=pyhd8ed1ab_0
+  - importlib_metadata=7.1.0=hd8ed1ab_0
+  - importlib_resources=6.4.0=pyhd8ed1ab_0
+  - intel-openmp=2023.1.0=hdb19cb5_46306
+  - ipykernel=6.29.3=pyhd33586a_0
+  - ipython=8.22.2=pyh707e725_0
+  - isoduration=20.11.0=pyhd8ed1ab_0
+  - jedi=0.19.1=pyhd8ed1ab_0
+  - jinja2=3.1.3=pyhd8ed1ab_0
+  - json5=0.9.25=pyhd8ed1ab_0
+  - jsonpointer=2.4=py310hff52083_3
+  - jsonschema=4.21.1=pyhd8ed1ab_0
+  - jsonschema-specifications=2023.12.1=pyhd8ed1ab_0
+  - jsonschema-with-format-nongpl=4.21.1=pyhd8ed1ab_0
+  - jupyter-lsp=2.2.5=pyhd8ed1ab_0
+  - jupyter_client=8.6.1=pyhd8ed1ab_0
+  - jupyter_core=5.7.2=py310hff52083_0
+  - jupyter_events=0.10.0=pyhd8ed1ab_0
+  - jupyter_server=2.14.0=pyhd8ed1ab_0
+  - jupyter_server_terminals=0.5.3=pyhd8ed1ab_0
+  - jupyterlab=4.1.6=pyhd8ed1ab_0
+  - jupyterlab_pygments=0.3.0=pyhd8ed1ab_1
+  - jupyterlab_server=2.27.1=pyhd8ed1ab_0
+  - lame=3.100=h7b6447c_0
+  - lcms2=2.16=hb7c19ff_0
+  - ld_impl_linux-64=2.40=h55db66e_0
+  - lerc=4.0.0=h27087fc_0
+  - libcublas=12.1.0.26=0
+  - libcufft=11.0.2.4=0
+  - libcufile=1.9.1.3=0
+  - libcurand=10.3.5.147=0
+  - libcusolver=11.4.4.55=0
+  - libcusparse=12.0.2.55=0
+  - libdeflate=1.20=hd590300_0
+  - libdicom=1.0.5=hd590300_1
+  - libffi=3.4.4=h6a678d5_0
+  - libgcc-ng=13.2.0=hc881cc4_6
+  - libglib=2.80.0=hf2295e7_6
+  - libgomp=13.2.0=hc881cc4_6
+  - libiconv=1.17=hd590300_2
+  - libidn2=2.3.4=h5eee18b_0
+  - libjpeg-turbo=3.0.0=hd590300_1
+  - libnpp=12.0.2.50=0
+  - libnsl=2.0.1=hd590300_0
+  - libnvjitlink=12.1.105=0
+  - libnvjpeg=12.1.1.14=0
+  - libpng=1.6.43=h2797004_0
+  - libsodium=1.0.18=h36c2ea0_1
+  - libsqlite=3.45.3=h2797004_0
+  - libstdcxx-ng=13.2.0=h95c4c6d_6
+  - libtasn1=4.19.0=h5eee18b_0
+  - libtiff=4.6.0=h1dd3fc0_3
+  - libunistring=0.9.10=h27cfd23_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libwebp-base=1.3.2=h5eee18b_0
+  - libxcb=1.15=h7f8727e_0
+  - libxcrypt=4.4.36=hd590300_1
+  - libxml2=2.12.6=h232c23b_2
+  - libzlib=1.2.13=hd590300_5
+  - lightning=2.2.5=pyhd8ed1ab_0
+  - lightning-utilities=0.11.2=pyhd8ed1ab_0
+  - llvm-openmp=14.0.6=h9e868ea_0
+  - lz4-c=1.9.4=h6a678d5_0
+  - markupsafe=2.1.5=py310h2372a71_0
+  - matplotlib-inline=0.1.7=pyhd8ed1ab_0
+  - mistune=3.0.2=pyhd8ed1ab_0
+  - mkl=2023.1.0=h213fc3f_46344
+  - mkl-service=2.4.0=py310h5eee18b_1
+  - mkl_fft=1.3.8=py310h5eee18b_0
+  - mkl_random=1.2.4=py310hdb19cb5_0
+  - mpc=1.1.0=h10f8cd9_1
+  - mpfr=4.0.2=hb69a4c5_1
+  - mpmath=1.3.0=py310h06a4308_0
+  - nbclient=0.10.0=pyhd8ed1ab_0
+  - nbconvert-core=7.16.3=pyhd8ed1ab_1
+  - nbformat=5.10.4=pyhd8ed1ab_0
+  - ncurses=6.4.20240210=h59595ed_0
+  - nest-asyncio=1.6.0=pyhd8ed1ab_0
+  - nettle=3.7.3=hbbd107a_1
+  - networkx=3.1=py310h06a4308_0
+  - notebook-shim=0.2.4=pyhd8ed1ab_0
+  - numpy=1.26.4=py310h5f9d8c6_0
+  - numpy-base=1.26.4=py310hb5e798b_0
+  - openh264=2.1.1=h4ff587b_0
+  - openjpeg=2.5.2=h488ebb8_0
+  - openslide=4.0.0=h58ba908_1
+  - openslide-python=1.3.1=py310h091d076_2
+  - openssl=3.3.0=h4ab18f5_3
+  - overrides=7.7.0=pyhd8ed1ab_0
+  - packaging=24.0=pyhd8ed1ab_0
+  - pandocfilters=1.5.0=pyhd8ed1ab_0
+  - parso=0.8.4=pyhd8ed1ab_0
+  - pcre2=10.43=hcad00b1_0
+  - pexpect=4.9.0=pyhd8ed1ab_0
+  - pickleshare=0.7.5=py_1003
+  - pillow=10.3.0=py310hf73ecf8_0
+  - pip=24.0=pyhd8ed1ab_0
+  - pixman=0.43.2=h59595ed_0
+  - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_1
+  - platformdirs=4.2.1=pyhd8ed1ab_0
+  - prometheus_client=0.20.0=pyhd8ed1ab_0
+  - prompt-toolkit=3.0.42=pyha770c72_0
+  - psutil=5.9.8=py310h2372a71_0
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pure_eval=0.2.2=pyhd8ed1ab_0
+  - pycparser=2.22=pyhd8ed1ab_0
+  - pygments=2.17.2=pyhd8ed1ab_0
+  - pysocks=1.7.1=pyha2e5f31_6
+  - python=3.10.14=hd12c33a_0_cpython
+  - python-dateutil=2.9.0=pyhd8ed1ab_0
+  - python-fastjsonschema=2.19.1=pyhd8ed1ab_0
+  - python-json-logger=2.0.7=pyhd8ed1ab_0
+  - python_abi=3.10=2_cp310
+  - pytorch=2.3.0=py3.10_cuda12.1_cudnn8.9.2_0
+  - pytorch-cuda=12.1=ha16c6d3_5
+  - pytorch-lightning=2.2.2=pyhd8ed1ab_0
+  - pytorch-mutex=1.0=cuda
+  - pytz=2024.1=pyhd8ed1ab_0
+  - pyyaml=6.0.1=py310h2372a71_1
+  - readline=8.2=h8228510_1
+  - referencing=0.34.0=pyhd8ed1ab_0
+  - requests=2.31.0=pyhd8ed1ab_0
+  - rfc3339-validator=0.1.4=pyhd8ed1ab_0
+  - rfc3986-validator=0.1.1=pyh9f0ad1d_0
+  - rpds-py=0.18.0=py310hcb5633a_0
+  - send2trash=1.8.3=pyh0d859eb_0
+  - setuptools=69.5.1=pyhd8ed1ab_0
+  - six=1.16.0=pyh6c4a22f_0
+  - sniffio=1.3.1=pyhd8ed1ab_0
+  - soupsieve=2.5=pyhd8ed1ab_1
+  - sqlite=3.45.3=h2c6b66d_0
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - sympy=1.12=py310h06a4308_0
+  - tbb=2021.8.0=hdb19cb5_0
+  - terminado=0.18.1=pyh0d859eb_0
+  - tinycss2=1.2.1=pyhd8ed1ab_0
+  - tk=8.6.13=noxft_h4845f30_101
+  - tomli=2.0.1=pyhd8ed1ab_0
+  - torchaudio=2.3.0=py310_cu121
+  - torchmetrics=1.4.0.post0=pyhd8ed1ab_0
+  - torchtriton=2.3.0=py310
+  - torchvision=0.18.0=py310_cu121
+  - tornado=6.4=py310h2372a71_0
+  - tqdm=4.66.4=pyhd8ed1ab_0
+  - traitlets=5.14.3=pyhd8ed1ab_0
+  - types-python-dateutil=2.9.0.20240316=pyhd8ed1ab_0
+  - typing-extensions=4.11.0=hd8ed1ab_0
+  - typing_extensions=4.11.0=pyha770c72_0
+  - typing_utils=0.1.0=pyhd8ed1ab_0
+  - uri-template=1.3.0=pyhd8ed1ab_0
+  - wcwidth=0.2.13=pyhd8ed1ab_0
+  - webcolors=1.13=pyhd8ed1ab_0
+  - webencodings=0.5.1=pyhd8ed1ab_2
+  - websocket-client=1.8.0=pyhd8ed1ab_0
+  - wheel=0.43.0=pyhd8ed1ab_1
+  - xorg-kbproto=1.0.7=h7f98852_1002
+  - xorg-libice=1.1.1=hd590300_0
+  - xorg-libsm=1.2.4=h7391055_0
+  - xorg-libx11=1.8.9=h8ee46fc_0
+  - xorg-libxext=1.3.4=h0b41bf4_2
+  - xorg-libxrender=0.9.11=hd590300_0
+  - xorg-renderproto=0.11.1=h7f98852_1002
+  - xorg-xextproto=7.3.0=h0b41bf4_1003
+  - xorg-xproto=7.0.31=h27cfd23_1007
+  - xz=5.4.6=h5eee18b_0
+  - yaml=0.2.5=h7f98852_2
+  - zeromq=4.3.5=h6a678d5_0
+  - zipp=3.17.0=pyhd8ed1ab_0
+  - zlib=1.2.13=hd590300_5
+  - zstd=1.5.5=hc292b87_0
+  - pip:
+      - aiofiles==23.2.1
+      - altair==5.3.0
+      - annotated-types==0.7.0
+      - asciitree==0.3.3
+      - asyncio-atexit==1.0.1
+      - backoff==2.2.1
+      - click==8.1.7
+      - cloudpickle==3.0.0
+      - debugpy==1.8.1
+      - deprecated==1.2.14
+      - dnspython==2.6.1
+      - docker-pycreds==0.4.0
+      - email-validator==2.1.1
+      - fastapi==0.111.0
+      - fastapi-cli==0.0.4
+      - fasteners==0.19
+      - ffmpy==0.3.2
+      - fsspec==2024.3.1
+      - gdown==5.1.0
+      - googleapis-common-protos==1.63.0
+      - gradio==4.36.1
+      - gradio-client==1.0.1
+      - gradio-image-prompter==0.1.0
+      - h5py==3.11.0
+      - httptools==0.6.1
+      - huggingface-hub==0.23.3
+      - imagecodecs==2024.1.1
+      - importlib-metadata==6.11.0
+      - jupyter-capture-output==0.0.11
+      - loky==3.0.0
+      - markdown-it-py==3.0.0
+      - mdurl==0.1.2
+      - monai==1.3.1
+      - numcodecs==0.12.1
+      - nvidia-nccl-cu12==2.20.5
+      - opencv-python==4.9.0.80
+      - opentelemetry-api==1.21.0
+      - opentelemetry-exporter-otlp-proto-common==1.21.0
+      - opentelemetry-exporter-otlp-proto-http==1.21.0
+      - opentelemetry-proto==1.21.0
+      - opentelemetry-sdk==1.21.0
+      - opentelemetry-semantic-conventions==0.42b0
+      - orjson==3.10.4
+      - pandas==2.2.2
+      - protobuf==4.25.3
+      - pycocotools==2.0.7
+      - pydantic==2.7.4
+      - pydantic-core==2.18.4
+      - pydub==0.25.1
+      - pyparsing==3.1.2
+      - python-multipart==0.0.9
+      - pyzmq==26.0.2
+      - rich==13.7.1
+      - ruff==0.4.8
+      - schedulefree==1.2.5
+      - segment-anything==1.0
+      - semantic-version==2.10.0
+      - sentry-sdk==2.2.1
+      - setproctitle==1.3.3
+      - shapely==2.0.4
+      - shellingham==1.5.4
+      - slideio==2.5.0
+      - starlette==0.37.2
+      - synapseclient==4.2.0
+      - tifffile==2024.4.24
+      - tomlkit==0.12.0
+      - toolz==0.12.1
+      - triton==2.3.0
+      - typer==0.12.3
+      - tzdata==2024.1
+      - ujson==5.10.0
+      - urllib3==2.2.1
+      - uvicorn==0.30.1
+      - uvloop==0.19.0
+      - wandb==0.17.0
+      - watchfiles==0.22.0
+      - websockets==11.0.3
+      - zarr==2.17.2
+prefix: /vol/data/miniconda3/envs/histo3.10

CellPilot/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/config.yaml

@@ -0,0 +1,35 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.14
+    cli_version: 0.17.0
+    framework: torch
+    is_jupyter_run: true
+    is_kaggle_kernel: true
+    start_time: 1718218198
+    t:
+      1:
+      - 1
+      - 41
+      - 48
+      - 55
+      - 105
+      2:
+      - 1
+      - 41
+      - 48
+      - 49
+      - 55
+      - 105
+      3:
+      - 2
+      - 23
+      4: 3.10.14
+      5: 0.17.0
+      8:
+      - 1
+      - 2
+      - 5
+      13: linux-x86_64

CellPilot/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/output.log

@@ -0,0 +1,33 @@
+wandb: Downloading large artifact model-xh4xphgx:v2, 389.52MB. 1 files...
+wandb:   1 of 1 files downloaded.
+Done. 0:0:0.5
+Running on local URL:  http://127.0.0.1:7860
+Running on public URL: https://a26d899ec4f9522031.gradio.live
+This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)
+[348, 185, 0, 0]
+Traceback (most recent call last):
+  File "/vol/data/miniconda3/envs/histo3.10/lib/python3.10/site-packages/gradio/queueing.py", line 532, in process_events
+    response = await route_utils.call_process_api(
+  File "/vol/data/miniconda3/envs/histo3.10/lib/python3.10/site-packages/gradio/route_utils.py", line 276, in call_process_api
+    output = await app.get_blocks().process_api(
+  File "/vol/data/miniconda3/envs/histo3.10/lib/python3.10/site-packages/gradio/blocks.py", line 1928, in process_api
+    result = await self.call_function(
+  File "/vol/data/miniconda3/envs/histo3.10/lib/python3.10/site-packages/gradio/blocks.py", line 1514, in call_function
+    prediction = await anyio.to_thread.run_sync(
+  File "/home/ubuntu/.local/lib/python3.10/site-packages/anyio/to_thread.py", line 49, in run_sync
+    return await get_async_backend().run_sync_in_worker_thread(
+  File "/home/ubuntu/.local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 2103, in run_sync_in_worker_thread
+    return await future
+  File "/home/ubuntu/.local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 823, in run
+    result = context.run(func, *args)
+  File "/vol/data/miniconda3/envs/histo3.10/lib/python3.10/site-packages/gradio/utils.py", line 832, in wrapper
+    response = f(*args, **kwargs)
+  File "/tmp/ipykernel_58594/4007838057.py", line 13, in segment
+    mask = inference(img, [prompt[0], prompt[1]], "points")
+  File "/tmp/ipykernel_58594/518913709.py", line 6, in inference
+    mask, _, _ = predictor.predict(point_coords= [[pixel]], point_labels= [[1]], multimask_output=False)
+  File "/vol/data/miniconda3/envs/histo3.10/lib/python3.10/site-packages/segment_anything/predictor.py", line 142, in predict
+    point_coords = self.transform.apply_coords(point_coords, self.original_size)
+  File "/vol/data/miniconda3/envs/histo3.10/lib/python3.10/site-packages/segment_anything/utils/transforms.py", line 42, in apply_coords
+    coords = deepcopy(coords).astype(float)
+AttributeError: 'list' object has no attribute 'astype'

CellPilot/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/requirements.txt

@@ -0,0 +1,378 @@
+Babel==2.13.1
+Babel==2.14.0
+Brotli==1.0.9
+CacheControl==0.13.1
+ConfigArgParse==1.7
+Deprecated==1.2.14
+GitPython==3.1.40
+Jinja2==3.1.3
+MarkupSafe==2.1.5
+POT==0.8.2
+Pillow==10.1.0
+PuLP==2.7.0
+PyNaCl==1.5.0
+PySocks==1.7.1
+PyYAML==6.0.1
+PyYAML==6.0.1
+Pygments==2.17.2
+Pygments==2.17.2
+QtPy==2.4.1
+Send2Trash==1.8.2
+Send2Trash==1.8.3
+aiofiles==23.2.1
+altair==5.3.0
+annotated-types==0.7.0
+anyio==4.1.0
+anyio==4.3.0
+appdirs==1.4.4
+argcomplete==3.1.6
+argon2-cffi-bindings==21.2.0
+argon2-cffi-bindings==21.2.0
+argon2-cffi==23.1.0
+argon2-cffi==23.1.0
+arrow==1.3.0
+arrow==1.3.0
+asciitree==0.3.3
+asttokens==2.4.1
+asttokens==2.4.1
+async-lru==2.0.4
+async-lru==2.0.4
+asyncio-atexit==1.0.1
+attrs==23.1.0
+attrs==23.2.0
+backoff==2.2.1
+beautifulsoup4==4.12.2
+beautifulsoup4==4.12.3
+bleach==6.1.0
+bleach==6.1.0
+cached-property==1.5.2
+certifi==2024.2.2
+cffi==1.16.0
+cffi==1.16.0
+charset-normalizer==3.3.2
+charset-normalizer==3.3.2
+click==8.1.7
+cloudpickle==3.0.0
+colorama==0.4.6
+coloredlogs==15.0.1
+comm==0.2.0
+comm==0.2.2
+connection-pool==0.0.3
+contourpy==1.2.0
+cwl-upgrader==1.2.10
+cwl-utils==0.31
+cwltool==3.1.20231114134824
+cycler==0.12.1
+datrie==0.8.2
+debugpy==1.6.7
+debugpy==1.8.0
+debugpy==1.8.1
+decorator==5.1.1
+decorator==5.1.1
+defusedxml==0.7.1
+defusedxml==0.7.1
+diskcache==5.6.3
+dnspython==2.6.1
+docker-compose==1.29.2
+docker-pycreds==0.4.0
+docker==6.1.3
+dockerpty==0.4.1
+docopt==0.6.2
+docutils==0.20.1
+dpath==2.1.6
+email_validator==2.1.1
+entrypoints==0.4
+exceptiongroup==1.2.0
+exceptiongroup==1.2.0
+executing==2.0.1
+executing==2.0.1
+fastapi-cli==0.0.4
+fastapi==0.111.0
+fasteners==0.19
+fastjsonschema==2.19.0
+fastjsonschema==2.19.1
+ffmpy==0.3.2
+filelock==3.13.1
+filelock==3.13.1
+fonttools==4.47.0
+fqdn==1.5.1
+fqdn==1.5.1
+fsspec==2024.3.1
+fsspec==2024.5.0
+gdown==5.1.0
+giotto-ph==0.2.2
+girder-client==3.2.2
+gitdb==4.0.11
+gmpy2==2.1.2
+googleapis-common-protos==1.63.0
+gradio==4.36.1
+gradio_client==1.0.1
+gradio_image_prompter==0.1.0
+gudhi==3.9.0
+h11==0.14.0
+h2==4.1.0
+h5py==3.11.0
+hpack==4.0.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+huggingface-hub==0.23.3
+humanfriendly==10.0
+hyperframe==6.0.1
+idna==3.7
+imagecodecs==2024.1.1
+imageio==2.34.0
+importlib-metadata==6.11.0
+importlib-resources==6.1.1
+importlib_resources==6.4.0
+ipykernel==6.27.1
+ipykernel==6.29.3
+ipython==8.18.1
+ipython==8.22.2
+ipywidgets==8.1.1
+isodate==0.6.1
+isoduration==20.11.0
+isoduration==20.11.0
+jedi==0.19.1
+jedi==0.19.1
+joblib==1.3.2
+json5==0.9.14
+json5==0.9.25
+jsondiff==2.0.0
+jsonpointer==2.4
+jsonschema-specifications==2023.11.2
+jsonschema-specifications==2023.12.1
+jsonschema==4.20.0
+jsonschema==4.21.1
+jupyter-capture-output==0.0.11
+jupyter-console==6.6.3
+jupyter-events==0.10.0
+jupyter-events==0.9.0
+jupyter-lsp==2.2.1
+jupyter-lsp==2.2.5
+jupyter==1.0.0
+jupyter_client==8.6.0
+jupyter_client==8.6.1
+jupyter_core==5.5.0
+jupyter_core==5.7.2
+jupyter_server==2.11.2
+jupyter_server==2.14.0
+jupyter_server_terminals==0.4.4
+jupyter_server_terminals==0.5.3
+jupyterlab-widgets==3.0.9
+jupyterlab==4.0.9
+jupyterlab==4.1.6
+jupyterlab_pygments==0.3.0
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.25.2
+jupyterlab_server==2.27.1
+kaggle==1.5.16
+kiwisolver==1.4.5
+lazy_loader==0.3
+lightning-utilities==0.11.2
+lightning==2.2.5
+loky==3.0.0
+lxml==4.9.3
+markdown-it-py==3.0.0
+matplotlib-inline==0.1.6
+matplotlib-inline==0.1.7
+matplotlib==3.8.2
+mdurl==0.1.2
+mistune==2.0.5
+mistune==3.0.2
+mkl-fft==1.3.8
+mkl-random==1.2.4
+mkl-service==2.4.0
+monai==1.3.1
+mpmath==1.3.0
+mpmath==1.3.0
+msgpack==1.0.7
+mypy-extensions==1.0.0
+nbclient==0.10.0
+nbclient==0.9.0
+nbconvert==7.12.0
+nbconvert==7.16.3
+nbformat==5.10.4
+nbformat==5.9.2
+nest-asyncio==1.6.0
+networkx==3.1
+networkx==3.2.1
+nextflow==23.10.0
+notebook==7.0.6
+notebook_shim==0.2.3
+notebook_shim==0.2.4
+numcodecs==0.12.1
+numpy==1.26.4
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.3.101
+nvidia-nvtx-cu12==12.1.105
+opencv-python==4.9.0.80
+openslide-python==1.3.1
+opentelemetry-api==1.21.0
+opentelemetry-exporter-otlp-proto-common==1.21.0
+opentelemetry-exporter-otlp-proto-http==1.21.0
+opentelemetry-proto==1.21.0
+opentelemetry-sdk==1.21.0
+opentelemetry-semantic-conventions==0.42b0
+orjson==3.10.4
+overrides==7.4.0
+overrides==7.7.0
+packaging==23.2
+packaging==24.0
+pandas==2.2.2
+pandocfilters==1.5.0
+pandocfilters==1.5.0
+paramiko==3.3.1
+parso==0.8.3
+parso==0.8.4
+pexpect==4.9.0
+pickleshare==0.7.5
+pillow==10.3.0
+pip==24.0
+pkgutil_resolve_name==1.3.10
+plac==1.4.1
+platformdirs==4.2.1
+prometheus-client==0.19.0
+prometheus_client==0.20.0
+prompt-toolkit==3.0.41
+prompt-toolkit==3.0.42
+protobuf==4.25.3
+prov==1.5.1
+psutil==5.9.8
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pure-eval==0.2.2
+pycocotools==2.0.7
+pycparser==2.21
+pycparser==2.22
+pydantic==2.7.4
+pydantic_core==2.18.4
+pydot==1.4.2
+pydub==0.25.1
+pyparsing==3.1.2
+python-dateutil==2.8.2
+python-dateutil==2.9.0
+python-dotenv==0.21.1
+python-json-logger==2.0.7
+python-json-logger==2.0.7
+python-multipart==0.0.9
+python-slugify==8.0.1
+pytorch-lightning==2.2.2
+pytz==2024.1
+pyzmq==25.1.1
+pyzmq==25.1.2
+pyzmq==26.0.2
+qtconsole==5.5.1
+rdflib==7.0.0
+referencing==0.31.1
+referencing==0.34.0
+requests-toolbelt==1.0.0
+requests==2.31.0
+requests==2.31.0
+reretry==0.11.8
+rfc3339-validator==0.1.4
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rfc3986-validator==0.1.1
+rich==13.7.1
+rpds-py==0.13.2
+rpds-py==0.18.0
+ruamel.yaml.clib==0.2.8
+ruamel.yaml==0.18.5
+ruff==0.4.8
+schedulefree==1.2.5
+schema-salad==8.4.20231117150958
+scikit-image==0.22.0
+scikit-learn==1.3.2
+scipy==1.11.4
+segment_anything==1.0
+semantic-version==2.10.0
+sentry-sdk==2.2.1
+setproctitle==1.3.3
+setuptools==69.5.1
+shapely==2.0.4
+shellescape==3.8.1
+shellingham==1.5.4
+six==1.16.0
+slideio==2.5.0
+smart-open==6.4.0
+smmap==5.0.1
+snakemake==7.32.4
+sniffio==1.3.0
+sniffio==1.3.1
+soupsieve==2.5
+soupsieve==2.5
+stack-data==0.6.2
+stack-data==0.6.3
+starlette==0.37.2
+stopit==1.1.2
+sympy==1.12
+sympy==1.12
+synapseclient==4.2.0
+tabulate==0.9.0
+terminado==0.18.0
+terminado==0.18.1
+text-unidecode==1.3
+texttable==1.7.0
+threadpoolctl==3.2.0
+throttler==1.2.2
+tifffile==2024.4.24
+tinycss2==1.2.1
+tinycss2==1.2.1
+tomli==2.0.1
+tomli==2.0.1
+tomlkit==0.12.0
+toolz==0.12.1
+toposort==1.10
+torch-topological==0.1.7
+torch==2.3.0
+torchaudio==2.3.0
+torchmetrics==1.4.0.post0
+torchvision==0.18.0
+tornado==6.4
+tornado==6.4
+tqdm==4.66.4
+traitlets==5.14.0
+traitlets==5.14.3
+triton==2.3.0
+triton==2.3.0
+typer==0.12.3
+types-python-dateutil==2.8.19.14
+types-python-dateutil==2.9.0.20240316
+typing-utils==0.1.0
+typing_extensions==4.11.0
+typing_extensions==4.8.0
+tzdata==2024.1
+ujson==5.10.0
+uri-template==1.3.0
+uri-template==1.3.0
+urllib3==2.2.1
+uvicorn==0.30.1
+uvloop==0.19.0
+wandb==0.17.0
+watchfiles==0.22.0
+wcwidth==0.2.12
+wcwidth==0.2.13
+webcolors==1.13
+webcolors==1.13
+webencodings==0.5.1
+webencodings==0.5.1
+websocket-client==0.59.0
+websocket-client==1.8.0
+websockets==11.0.3
+wheel==0.43.0
+widgetsnbextension==4.0.9
+wrapt==1.16.0
+yte==1.5.1
+zarr==2.17.2
+zipp==3.17.0

CellPilot/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/wandb-metadata.json

@@ -0,0 +1,87 @@
+{
+    "os": "Linux-5.15.0-112-generic-x86_64-with-glibc2.35",
+    "python": "3.10.14",
+    "heartbeatAt": "2024-06-12T18:49:59.300444",
+    "startedAt": "2024-06-12T18:49:58.656916",
+    "docker": null,
+    "cuda": null,
+    "args": [],
+    "state": "running",
+    "program": "<python with no main file>",
+    "codePathLocal": null,
+    "git": {
+        "remote": "https://github.com/philippendres/SAMHI.git",
+        "commit": "b72f97d47305e8d3ae04d0fd1d8bbba51deced40"
+    },
+    "email": "ge93qif@mytum.de",
+    "root": "/home/ubuntu/thesis/SAMHI",
+    "host": "dilaboct-5193a",
+    "username": "ubuntu",
+    "executable": "/vol/data/miniconda3/envs/histo3.10/bin/python",
+    "cpu_count": 8,
+    "cpu_count_logical": 8,
+    "cpu_freq": {
+        "current": 2800.0,
+        "min": 0.0,
+        "max": 0.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2800.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2800.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2800.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2800.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2800.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2800.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2800.0,
+            "min": 0.0,
+            "max": 0.0
+        },
+        {
+            "current": 2800.0,
+            "min": 0.0,
+            "max": 0.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 48.27519989013672,
+            "used": 41.396217346191406
+        }
+    },
+    "gpu": "NVIDIA A40",
+    "gpu_count": 1,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A40",
+            "memory_total": 48305799168
+        }
+    ],
+    "memory": {
+        "total": 62.792388916015625
+    }
+}

CellPilot/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 13}}

CellPilot/notebooks/wandb/run-20240612_184958-2vmzqu3d/logs/debug-internal.log

@@ -0,0 +1,486 @@
+2024-06-12 18:49:58,672 INFO    StreamThr :60040 [internal.py:wandb_internal():85] W&B internal server running at pid: 60040, started at: 2024-06-12 18:49:58.671044
+2024-06-12 18:49:58,676 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status
+2024-06-12 18:49:58,687 INFO    WriterThread:60040 [datastore.py:open_for_write():87] open: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/run-2vmzqu3d.wandb
+2024-06-12 18:49:58,688 DEBUG   SenderThread:60040 [sender.py:send():378] send: header
+2024-06-12 18:49:58,698 DEBUG   SenderThread:60040 [sender.py:send():378] send: run
+2024-06-12 18:49:59,143 INFO    SenderThread:60040 [dir_watcher.py:__init__():211] watching files in: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files
+2024-06-12 18:49:59,144 INFO    SenderThread:60040 [sender.py:_start_run_threads():1123] run started: 2vmzqu3d with start time 1718218198.671151
+2024-06-12 18:49:59,163 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: check_version
+2024-06-12 18:49:59,164 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: check_version
+2024-06-12 18:49:59,260 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: run_start
+2024-06-12 18:49:59,279 DEBUG   HandlerThread:60040 [system_info.py:__init__():26] System info init
+2024-06-12 18:49:59,279 DEBUG   HandlerThread:60040 [system_info.py:__init__():41] System info init done
+2024-06-12 18:49:59,280 INFO    HandlerThread:60040 [system_monitor.py:start():194] Starting system monitor
+2024-06-12 18:49:59,280 INFO    SystemMonitor:60040 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-06-12 18:49:59,280 INFO    HandlerThread:60040 [system_monitor.py:probe():214] Collecting system info
+2024-06-12 18:49:59,281 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started cpu monitoring
+2024-06-12 18:49:59,285 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started disk monitoring
+2024-06-12 18:49:59,287 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started gpu monitoring
+2024-06-12 18:49:59,290 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started memory monitoring
+2024-06-12 18:49:59,292 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started network monitoring
+2024-06-12 18:49:59,300 DEBUG   HandlerThread:60040 [system_info.py:probe():150] Probing system
+2024-06-12 18:49:59,304 DEBUG   HandlerThread:60040 [system_info.py:_probe_git():135] Probing git
+2024-06-12 18:49:59,314 DEBUG   HandlerThread:60040 [system_info.py:_probe_git():143] Probing git done
+2024-06-12 18:49:59,314 DEBUG   HandlerThread:60040 [system_info.py:probe():198] Probing system done
+2024-06-12 18:49:59,314 DEBUG   HandlerThread:60040 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-112-generic-x86_64-with-glibc2.35', 'python': '3.10.14', 'heartbeatAt': '2024-06-12T18:49:59.300444', 'startedAt': '2024-06-12T18:49:58.656916', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': '<python with no main file>', 'codePathLocal': None, 'git': {'remote': 'https://github.com/philippendres/SAMHI.git', 'commit': 'b72f97d47305e8d3ae04d0fd1d8bbba51deced40'}, 'email': 'ge93qif@mytum.de', 'root': '/home/ubuntu/thesis/SAMHI', 'host': 'dilaboct-5193a', 'username': 'ubuntu', 'executable': '/vol/data/miniconda3/envs/histo3.10/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 2800.0, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}, {'current': 2800.0, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 48.27519989013672, 'used': 41.396217346191406}}, 'gpu': 'NVIDIA A40', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A40', 'memory_total': 48305799168}], 'memory': {'total': 62.792388916015625}}
+2024-06-12 18:49:59,314 INFO    HandlerThread:60040 [system_monitor.py:probe():224] Finished collecting system info
+2024-06-12 18:49:59,314 INFO    HandlerThread:60040 [system_monitor.py:probe():227] Publishing system info
+2024-06-12 18:49:59,314 DEBUG   HandlerThread:60040 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment
+2024-06-12 18:50:00,147 INFO    Thread-12 :60040 [dir_watcher.py:_on_file_created():271] file/dir created: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/conda-environment.yaml
+2024-06-12 18:50:01,989 DEBUG   HandlerThread:60040 [system_info.py:_save_conda():222] Saving conda packages done
+2024-06-12 18:50:01,990 INFO    HandlerThread:60040 [system_monitor.py:probe():229] Finished publishing system info
+2024-06-12 18:50:01,995 DEBUG   SenderThread:60040 [sender.py:send():378] send: files
+2024-06-12 18:50:01,995 INFO    SenderThread:60040 [sender.py:_save_file():1389] saving file wandb-metadata.json with policy now
+2024-06-12 18:50:02,144 INFO    Thread-12 :60040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/conda-environment.yaml
+2024-06-12 18:50:02,145 INFO    Thread-12 :60040 [dir_watcher.py:_on_file_created():271] file/dir created: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/wandb-metadata.json
+2024-06-12 18:50:02,294 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: python_packages
+2024-06-12 18:50:02,295 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: python_packages
+2024-06-12 18:50:02,295 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: stop_status
+2024-06-12 18:50:02,298 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: stop_status
+2024-06-12 18:50:02,470 DEBUG   SenderThread:60040 [sender.py:send():378] send: telemetry
+2024-06-12 18:50:02,585 INFO    wandb-upload_0:60040 [upload_job.py:push():130] Uploaded file /tmp/tmp9moz5yy2wandb/3ssye1n1-wandb-metadata.json
+2024-06-12 18:50:03,025 DEBUG   SenderThread:60040 [sender.py:send():378] send: use_artifact
+2024-06-12 18:50:03,145 INFO    Thread-12 :60040 [dir_watcher.py:_on_file_created():271] file/dir created: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/requirements.txt
+2024-06-12 18:50:03,853 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:50:04,146 INFO    Thread-12 :60040 [dir_watcher.py:_on_file_created():271] file/dir created: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/output.log
+2024-06-12 18:50:05,538 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: pause
+2024-06-12 18:50:05,539 INFO    HandlerThread:60040 [handler.py:handle_request_pause():724] stopping system metrics thread
+2024-06-12 18:50:05,539 INFO    HandlerThread:60040 [system_monitor.py:finish():203] Stopping system monitor
+2024-06-12 18:50:05,539 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-06-12 18:50:05,539 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-06-12 18:50:05,539 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-06-12 18:50:05,540 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined cpu monitor
+2024-06-12 18:50:05,540 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined disk monitor
+2024-06-12 18:50:05,570 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined gpu monitor
+2024-06-12 18:50:05,571 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined memory monitor
+2024-06-12 18:50:05,571 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined network monitor
+2024-06-12 18:50:05,602 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: resume
+2024-06-12 18:50:05,602 INFO    HandlerThread:60040 [handler.py:handle_request_resume():715] starting system metrics thread
+2024-06-12 18:50:05,602 INFO    HandlerThread:60040 [system_monitor.py:start():194] Starting system monitor
+2024-06-12 18:50:05,603 INFO    SystemMonitor:60040 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-06-12 18:50:05,603 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started cpu monitoring
+2024-06-12 18:50:05,604 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started disk monitoring
+2024-06-12 18:50:05,605 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: pause
+2024-06-12 18:50:05,605 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started gpu monitoring
+2024-06-12 18:50:05,605 INFO    HandlerThread:60040 [handler.py:handle_request_pause():724] stopping system metrics thread
+2024-06-12 18:50:05,607 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started memory monitoring
+2024-06-12 18:50:05,608 INFO    HandlerThread:60040 [system_monitor.py:finish():203] Stopping system monitor
+2024-06-12 18:50:05,611 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started network monitoring
+2024-06-12 18:50:05,612 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined cpu monitor
+2024-06-12 18:50:05,613 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-06-12 18:50:05,617 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined disk monitor
+2024-06-12 18:50:05,617 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-06-12 18:50:05,618 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-06-12 18:50:05,658 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined gpu monitor
+2024-06-12 18:50:05,658 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined memory monitor
+2024-06-12 18:50:05,658 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined network monitor
+2024-06-12 18:50:05,659 DEBUG   SenderThread:60040 [sender.py:send():378] send: stats
+2024-06-12 18:50:06,147 INFO    Thread-12 :60040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/output.log
+2024-06-12 18:50:09,660 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:50:14,662 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:50:17,295 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: stop_status
+2024-06-12 18:50:17,296 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: stop_status
+2024-06-12 18:50:20,572 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:50:25,573 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:50:27,344 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: resume
+2024-06-12 18:50:27,345 INFO    HandlerThread:60040 [handler.py:handle_request_resume():715] starting system metrics thread
+2024-06-12 18:50:27,345 INFO    HandlerThread:60040 [system_monitor.py:start():194] Starting system monitor
+2024-06-12 18:50:27,345 INFO    SystemMonitor:60040 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-06-12 18:50:27,346 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started cpu monitoring
+2024-06-12 18:50:27,347 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started disk monitoring
+2024-06-12 18:50:27,348 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started gpu monitoring
+2024-06-12 18:50:27,353 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started memory monitoring
+2024-06-12 18:50:27,355 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started network monitoring
+2024-06-12 18:50:27,835 DEBUG   SenderThread:60040 [sender.py:send():378] send: telemetry
+2024-06-12 18:50:30,825 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:50:31,162 INFO    Thread-12 :60040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/config.yaml
+2024-06-12 18:50:32,296 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: stop_status
+2024-06-12 18:50:32,297 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: stop_status
+2024-06-12 18:50:33,163 INFO    Thread-12 :60040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/output.log
+2024-06-12 18:50:33,881 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: pause
+2024-06-12 18:50:33,881 INFO    HandlerThread:60040 [handler.py:handle_request_pause():724] stopping system metrics thread
+2024-06-12 18:50:33,881 INFO    HandlerThread:60040 [system_monitor.py:finish():203] Stopping system monitor
+2024-06-12 18:50:33,881 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-06-12 18:50:33,882 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-06-12 18:50:33,883 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-06-12 18:50:33,884 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined cpu monitor
+2024-06-12 18:50:33,884 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined disk monitor
+2024-06-12 18:50:33,925 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined gpu monitor
+2024-06-12 18:50:33,926 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined memory monitor
+2024-06-12 18:50:33,926 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined network monitor
+2024-06-12 18:50:33,927 DEBUG   SenderThread:60040 [sender.py:send():378] send: stats
+2024-06-12 18:50:35,164 INFO    Thread-12 :60040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/output.log
+2024-06-12 18:50:36,928 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:50:41,929 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:50:46,930 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:50:47,296 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: stop_status
+2024-06-12 18:50:47,297 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: stop_status
+2024-06-12 18:50:52,485 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:50:57,486 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:51:02,296 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: stop_status
+2024-06-12 18:51:02,297 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: stop_status
+2024-06-12 18:51:02,519 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:51:07,520 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:51:12,521 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:51:17,297 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: stop_status
+2024-06-12 18:51:17,298 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: stop_status
+2024-06-12 18:51:17,691 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:51:22,692 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:51:27,693 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:51:32,297 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: stop_status
+2024-06-12 18:51:32,299 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: stop_status
+2024-06-12 18:51:33,453 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:51:38,455 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:51:43,457 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:51:47,298 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: stop_status
+2024-06-12 18:51:47,299 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: stop_status
+2024-06-12 18:51:48,524 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:51:53,526 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:51:58,528 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:52:02,298 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: stop_status
+2024-06-12 18:52:02,298 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: stop_status
+2024-06-12 18:52:04,494 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:52:09,495 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:52:14,497 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:52:17,298 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: stop_status
+2024-06-12 18:52:17,299 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: stop_status
+2024-06-12 18:52:19,544 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:52:24,546 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:52:29,547 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:52:32,298 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: stop_status
+2024-06-12 18:52:32,300 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: stop_status
+2024-06-12 18:52:35,527 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:52:40,624 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:52:41,232 INFO    Thread-12 :60040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/output.log
+2024-06-12 18:52:42,232 INFO    Thread-12 :60040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/output.log
+2024-06-12 18:52:45,625 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:52:47,299 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: stop_status
+2024-06-12 18:52:47,300 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: stop_status
+2024-06-12 18:52:51,465 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:52:56,467 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:53:01,468 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:53:02,299 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: stop_status
+2024-06-12 18:53:02,301 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: stop_status
+2024-06-12 18:53:06,532 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:53:11,533 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:53:16,534 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:53:17,300 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: stop_status
+2024-06-12 18:53:17,301 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: stop_status
+2024-06-12 18:53:21,539 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:53:26,540 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:53:31,542 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:53:32,300 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: stop_status
+2024-06-12 18:53:32,301 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: stop_status
+2024-06-12 18:53:35,081 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: resume
+2024-06-12 18:53:35,081 INFO    HandlerThread:60040 [handler.py:handle_request_resume():715] starting system metrics thread
+2024-06-12 18:53:35,081 INFO    HandlerThread:60040 [system_monitor.py:start():194] Starting system monitor
+2024-06-12 18:53:35,081 INFO    SystemMonitor:60040 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-06-12 18:53:35,083 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started cpu monitoring
+2024-06-12 18:53:35,084 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started disk monitoring
+2024-06-12 18:53:35,086 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started gpu monitoring
+2024-06-12 18:53:35,088 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started memory monitoring
+2024-06-12 18:53:35,090 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started network monitoring
+2024-06-12 18:53:35,091 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: pause
+2024-06-12 18:53:35,093 INFO    HandlerThread:60040 [handler.py:handle_request_pause():724] stopping system metrics thread
+2024-06-12 18:53:35,093 INFO    HandlerThread:60040 [system_monitor.py:finish():203] Stopping system monitor
+2024-06-12 18:53:35,094 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined cpu monitor
+2024-06-12 18:53:35,094 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-06-12 18:53:35,095 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-06-12 18:53:35,095 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined disk monitor
+2024-06-12 18:53:35,095 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-06-12 18:53:35,137 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined gpu monitor
+2024-06-12 18:53:35,138 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined memory monitor
+2024-06-12 18:53:35,138 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined network monitor
+2024-06-12 18:53:35,139 DEBUG   SenderThread:60040 [sender.py:send():378] send: stats
+2024-06-12 18:53:36,135 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: resume
+2024-06-12 18:53:36,135 INFO    HandlerThread:60040 [handler.py:handle_request_resume():715] starting system metrics thread
+2024-06-12 18:53:36,135 INFO    HandlerThread:60040 [system_monitor.py:start():194] Starting system monitor
+2024-06-12 18:53:36,136 INFO    SystemMonitor:60040 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-06-12 18:53:36,137 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started cpu monitoring
+2024-06-12 18:53:36,139 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started disk monitoring
+2024-06-12 18:53:36,140 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started gpu monitoring
+2024-06-12 18:53:36,146 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started memory monitoring
+2024-06-12 18:53:36,147 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: pause
+2024-06-12 18:53:36,149 INFO    HandlerThread:60040 [handler.py:handle_request_pause():724] stopping system metrics thread
+2024-06-12 18:53:36,149 INFO    HandlerThread:60040 [system_monitor.py:finish():203] Stopping system monitor
+2024-06-12 18:53:36,148 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started network monitoring
+2024-06-12 18:53:36,150 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined cpu monitor
+2024-06-12 18:53:36,153 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-06-12 18:53:36,154 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined disk monitor
+2024-06-12 18:53:36,155 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-06-12 18:53:36,156 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-06-12 18:53:36,195 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined gpu monitor
+2024-06-12 18:53:36,195 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined memory monitor
+2024-06-12 18:53:36,196 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined network monitor
+2024-06-12 18:53:36,196 DEBUG   SenderThread:60040 [sender.py:send():378] send: stats
+2024-06-12 18:53:37,197 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:53:38,899 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: resume
+2024-06-12 18:53:38,899 INFO    HandlerThread:60040 [handler.py:handle_request_resume():715] starting system metrics thread
+2024-06-12 18:53:38,899 INFO    HandlerThread:60040 [system_monitor.py:start():194] Starting system monitor
+2024-06-12 18:53:38,900 INFO    SystemMonitor:60040 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-06-12 18:53:38,902 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started cpu monitoring
+2024-06-12 18:53:38,903 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started disk monitoring
+2024-06-12 18:53:38,904 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started gpu monitoring
+2024-06-12 18:53:38,906 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: pause
+2024-06-12 18:53:38,906 INFO    HandlerThread:60040 [handler.py:handle_request_pause():724] stopping system metrics thread
+2024-06-12 18:53:38,906 INFO    HandlerThread:60040 [system_monitor.py:finish():203] Stopping system monitor
+2024-06-12 18:53:38,908 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined cpu monitor
+2024-06-12 18:53:38,910 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined disk monitor
+2024-06-12 18:53:38,910 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started memory monitoring
+2024-06-12 18:53:38,911 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-06-12 18:53:38,911 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-06-12 18:53:38,911 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-06-12 18:53:38,954 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined gpu monitor
+2024-06-12 18:53:38,955 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined memory monitor
+2024-06-12 18:53:38,956 DEBUG   SenderThread:60040 [sender.py:send():378] send: stats
+2024-06-12 18:53:39,571 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: resume
+2024-06-12 18:53:39,571 INFO    HandlerThread:60040 [handler.py:handle_request_resume():715] starting system metrics thread
+2024-06-12 18:53:39,571 INFO    HandlerThread:60040 [system_monitor.py:start():194] Starting system monitor
+2024-06-12 18:53:39,572 INFO    SystemMonitor:60040 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-06-12 18:53:39,573 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started cpu monitoring
+2024-06-12 18:53:39,574 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started disk monitoring
+2024-06-12 18:53:39,575 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started gpu monitoring
+2024-06-12 18:53:39,576 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started memory monitoring
+2024-06-12 18:53:39,578 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started network monitoring
+2024-06-12 18:53:39,578 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: pause
+2024-06-12 18:53:39,580 INFO    HandlerThread:60040 [handler.py:handle_request_pause():724] stopping system metrics thread
+2024-06-12 18:53:39,583 INFO    HandlerThread:60040 [system_monitor.py:finish():203] Stopping system monitor
+2024-06-12 18:53:39,583 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined cpu monitor
+2024-06-12 18:53:39,585 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-06-12 18:53:39,585 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-06-12 18:53:39,586 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-06-12 18:53:39,588 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined disk monitor
+2024-06-12 18:53:39,631 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined gpu monitor
+2024-06-12 18:53:39,631 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined memory monitor
+2024-06-12 18:53:39,631 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined network monitor
+2024-06-12 18:53:39,632 DEBUG   SenderThread:60040 [sender.py:send():378] send: stats
+2024-06-12 18:53:39,743 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: resume
+2024-06-12 18:53:39,743 INFO    HandlerThread:60040 [handler.py:handle_request_resume():715] starting system metrics thread
+2024-06-12 18:53:39,743 INFO    HandlerThread:60040 [system_monitor.py:start():194] Starting system monitor
+2024-06-12 18:53:39,744 INFO    SystemMonitor:60040 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-06-12 18:53:39,744 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started cpu monitoring
+2024-06-12 18:53:39,746 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started disk monitoring
+2024-06-12 18:53:39,747 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started gpu monitoring
+2024-06-12 18:53:39,748 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started memory monitoring
+2024-06-12 18:53:39,750 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started network monitoring
+2024-06-12 18:53:39,751 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: pause
+2024-06-12 18:53:39,756 INFO    HandlerThread:60040 [handler.py:handle_request_pause():724] stopping system metrics thread
+2024-06-12 18:53:39,756 INFO    HandlerThread:60040 [system_monitor.py:finish():203] Stopping system monitor
+2024-06-12 18:53:39,756 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined cpu monitor
+2024-06-12 18:53:39,759 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-06-12 18:53:39,759 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-06-12 18:53:39,759 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-06-12 18:53:39,760 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined disk monitor
+2024-06-12 18:53:39,802 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined gpu monitor
+2024-06-12 18:53:39,802 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined memory monitor
+2024-06-12 18:53:39,803 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined network monitor
+2024-06-12 18:53:39,803 DEBUG   SenderThread:60040 [sender.py:send():378] send: stats
+2024-06-12 18:53:40,108 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: resume
+2024-06-12 18:53:40,108 INFO    HandlerThread:60040 [handler.py:handle_request_resume():715] starting system metrics thread
+2024-06-12 18:53:40,109 INFO    HandlerThread:60040 [system_monitor.py:start():194] Starting system monitor
+2024-06-12 18:53:40,109 INFO    SystemMonitor:60040 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-06-12 18:53:40,110 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started cpu monitoring
+2024-06-12 18:53:40,111 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started disk monitoring
+2024-06-12 18:53:40,112 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started gpu monitoring
+2024-06-12 18:53:40,115 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started memory monitoring
+2024-06-12 18:53:40,116 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started network monitoring
+2024-06-12 18:53:40,118 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: pause
+2024-06-12 18:53:40,118 INFO    HandlerThread:60040 [handler.py:handle_request_pause():724] stopping system metrics thread
+2024-06-12 18:53:40,118 INFO    HandlerThread:60040 [system_monitor.py:finish():203] Stopping system monitor
+2024-06-12 18:53:40,119 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-06-12 18:53:40,119 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined cpu monitor
+2024-06-12 18:53:40,119 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-06-12 18:53:40,120 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined disk monitor
+2024-06-12 18:53:40,120 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-06-12 18:53:40,164 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined gpu monitor
+2024-06-12 18:53:40,165 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined memory monitor
+2024-06-12 18:53:40,165 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined network monitor
+2024-06-12 18:53:40,166 DEBUG   SenderThread:60040 [sender.py:send():378] send: stats
+2024-06-12 18:53:41,413 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: resume
+2024-06-12 18:53:41,413 INFO    HandlerThread:60040 [handler.py:handle_request_resume():715] starting system metrics thread
+2024-06-12 18:53:41,413 INFO    HandlerThread:60040 [system_monitor.py:start():194] Starting system monitor
+2024-06-12 18:53:41,414 INFO    SystemMonitor:60040 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-06-12 18:53:41,416 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started cpu monitoring
+2024-06-12 18:53:41,417 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started disk monitoring
+2024-06-12 18:53:41,419 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started gpu monitoring
+2024-06-12 18:53:41,420 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started memory monitoring
+2024-06-12 18:53:41,422 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started network monitoring
+2024-06-12 18:53:41,424 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: pause
+2024-06-12 18:53:41,425 INFO    HandlerThread:60040 [handler.py:handle_request_pause():724] stopping system metrics thread
+2024-06-12 18:53:41,425 INFO    HandlerThread:60040 [system_monitor.py:finish():203] Stopping system monitor
+2024-06-12 18:53:41,426 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined cpu monitor
+2024-06-12 18:53:41,426 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined disk monitor
+2024-06-12 18:53:41,427 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-06-12 18:53:41,427 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-06-12 18:53:41,427 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-06-12 18:53:41,471 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined gpu monitor
+2024-06-12 18:53:41,471 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined memory monitor
+2024-06-12 18:53:41,472 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined network monitor
+2024-06-12 18:53:41,472 DEBUG   SenderThread:60040 [sender.py:send():378] send: stats
+2024-06-12 18:53:41,597 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: resume
+2024-06-12 18:53:41,598 INFO    HandlerThread:60040 [handler.py:handle_request_resume():715] starting system metrics thread
+2024-06-12 18:53:41,598 INFO    HandlerThread:60040 [system_monitor.py:start():194] Starting system monitor
+2024-06-12 18:53:41,598 INFO    SystemMonitor:60040 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-06-12 18:53:41,599 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started cpu monitoring
+2024-06-12 18:53:41,601 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started disk monitoring
+2024-06-12 18:53:41,603 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started gpu monitoring
+2024-06-12 18:53:41,605 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started memory monitoring
+2024-06-12 18:53:41,607 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: pause
+2024-06-12 18:53:41,608 INFO    HandlerThread:60040 [handler.py:handle_request_pause():724] stopping system metrics thread
+2024-06-12 18:53:41,608 INFO    HandlerThread:60040 [system_monitor.py:finish():203] Stopping system monitor
+2024-06-12 18:53:41,610 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started network monitoring
+2024-06-12 18:53:41,610 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-06-12 18:53:41,611 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-06-12 18:53:41,611 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-06-12 18:53:41,612 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined cpu monitor
+2024-06-12 18:53:41,613 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined disk monitor
+2024-06-12 18:53:41,662 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined gpu monitor
+2024-06-12 18:53:41,662 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined memory monitor
+2024-06-12 18:53:41,663 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined network monitor
+2024-06-12 18:53:41,663 DEBUG   SenderThread:60040 [sender.py:send():378] send: stats
+2024-06-12 18:53:42,665 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:53:43,999 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: resume
+2024-06-12 18:53:44,000 INFO    HandlerThread:60040 [handler.py:handle_request_resume():715] starting system metrics thread
+2024-06-12 18:53:44,000 INFO    HandlerThread:60040 [system_monitor.py:start():194] Starting system monitor
+2024-06-12 18:53:44,000 INFO    SystemMonitor:60040 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-06-12 18:53:44,002 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started cpu monitoring
+2024-06-12 18:53:44,003 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started disk monitoring
+2024-06-12 18:53:44,004 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started gpu monitoring
+2024-06-12 18:53:44,006 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started memory monitoring
+2024-06-12 18:53:44,007 INFO    SystemMonitor:60040 [interfaces.py:start():188] Started network monitoring
+2024-06-12 18:53:44,045 DEBUG   SenderThread:60040 [sender.py:send():378] send: telemetry
+2024-06-12 18:53:44,048 DEBUG   SenderThread:60040 [sender.py:send():378] send: exit
+2024-06-12 18:53:44,048 INFO    SenderThread:60040 [sender.py:send_exit():585] handling exit code: 0
+2024-06-12 18:53:44,049 INFO    SenderThread:60040 [sender.py:send_exit():587] handling runtime: 13
+2024-06-12 18:53:44,049 INFO    SenderThread:60040 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end
+2024-06-12 18:53:44,050 INFO    SenderThread:60040 [sender.py:send_exit():593] send defer
+2024-06-12 18:53:44,050 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: defer
+2024-06-12 18:53:44,051 INFO    HandlerThread:60040 [handler.py:handle_request_defer():184] handle defer: 0
+2024-06-12 18:53:44,051 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: defer
+2024-06-12 18:53:44,051 INFO    SenderThread:60040 [sender.py:send_request_defer():609] handle sender defer: 0
+2024-06-12 18:53:44,052 INFO    SenderThread:60040 [sender.py:transition_state():613] send defer: 1
+2024-06-12 18:53:44,052 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: defer
+2024-06-12 18:53:44,052 INFO    HandlerThread:60040 [handler.py:handle_request_defer():184] handle defer: 1
+2024-06-12 18:53:44,053 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: defer
+2024-06-12 18:53:44,053 INFO    SenderThread:60040 [sender.py:send_request_defer():609] handle sender defer: 1
+2024-06-12 18:53:44,053 INFO    SenderThread:60040 [sender.py:transition_state():613] send defer: 2
+2024-06-12 18:53:44,053 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: defer
+2024-06-12 18:53:44,053 INFO    HandlerThread:60040 [handler.py:handle_request_defer():184] handle defer: 2
+2024-06-12 18:53:44,053 INFO    HandlerThread:60040 [system_monitor.py:finish():203] Stopping system monitor
+2024-06-12 18:53:44,054 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-06-12 18:53:44,055 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2024-06-12 18:53:44,055 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined cpu monitor
+2024-06-12 18:53:44,056 DEBUG   SystemMonitor:60040 [system_monitor.py:_start():183] Publishing last batch of metrics
+2024-06-12 18:53:44,056 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined disk monitor
+2024-06-12 18:53:44,101 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined gpu monitor
+2024-06-12 18:53:44,101 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined memory monitor
+2024-06-12 18:53:44,102 INFO    HandlerThread:60040 [interfaces.py:finish():200] Joined network monitor
+2024-06-12 18:53:44,102 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: defer
+2024-06-12 18:53:44,103 INFO    SenderThread:60040 [sender.py:send_request_defer():609] handle sender defer: 2
+2024-06-12 18:53:44,103 INFO    SenderThread:60040 [sender.py:transition_state():613] send defer: 3
+2024-06-12 18:53:44,103 DEBUG   SenderThread:60040 [sender.py:send():378] send: stats
+2024-06-12 18:53:44,104 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: defer
+2024-06-12 18:53:44,104 INFO    HandlerThread:60040 [handler.py:handle_request_defer():184] handle defer: 3
+2024-06-12 18:53:44,105 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: defer
+2024-06-12 18:53:44,105 INFO    SenderThread:60040 [sender.py:send_request_defer():609] handle sender defer: 3
+2024-06-12 18:53:44,105 INFO    SenderThread:60040 [sender.py:transition_state():613] send defer: 4
+2024-06-12 18:53:44,105 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: defer
+2024-06-12 18:53:44,106 INFO    HandlerThread:60040 [handler.py:handle_request_defer():184] handle defer: 4
+2024-06-12 18:53:44,106 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: defer
+2024-06-12 18:53:44,106 INFO    SenderThread:60040 [sender.py:send_request_defer():609] handle sender defer: 4
+2024-06-12 18:53:44,106 INFO    SenderThread:60040 [sender.py:transition_state():613] send defer: 5
+2024-06-12 18:53:44,107 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: defer
+2024-06-12 18:53:44,107 INFO    HandlerThread:60040 [handler.py:handle_request_defer():184] handle defer: 5
+2024-06-12 18:53:44,107 DEBUG   SenderThread:60040 [sender.py:send():378] send: summary
+2024-06-12 18:53:44,108 INFO    SenderThread:60040 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end
+2024-06-12 18:53:44,109 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: defer
+2024-06-12 18:53:44,109 INFO    SenderThread:60040 [sender.py:send_request_defer():609] handle sender defer: 5
+2024-06-12 18:53:44,109 INFO    SenderThread:60040 [sender.py:transition_state():613] send defer: 6
+2024-06-12 18:53:44,109 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: defer
+2024-06-12 18:53:44,109 INFO    HandlerThread:60040 [handler.py:handle_request_defer():184] handle defer: 6
+2024-06-12 18:53:44,110 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: defer
+2024-06-12 18:53:44,110 INFO    SenderThread:60040 [sender.py:send_request_defer():609] handle sender defer: 6
+2024-06-12 18:53:44,115 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: status_report
+2024-06-12 18:53:44,265 INFO    Thread-12 :60040 [dir_watcher.py:_on_file_created():271] file/dir created: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/wandb-summary.json
+2024-06-12 18:53:44,280 INFO    SenderThread:60040 [sender.py:transition_state():613] send defer: 7
+2024-06-12 18:53:44,280 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: defer
+2024-06-12 18:53:44,281 INFO    HandlerThread:60040 [handler.py:handle_request_defer():184] handle defer: 7
+2024-06-12 18:53:44,281 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: defer
+2024-06-12 18:53:44,281 INFO    SenderThread:60040 [sender.py:send_request_defer():609] handle sender defer: 7
+2024-06-12 18:53:45,048 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: poll_exit
+2024-06-12 18:53:45,266 INFO    Thread-12 :60040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/config.yaml
+2024-06-12 18:53:47,100 INFO    SenderThread:60040 [sender.py:transition_state():613] send defer: 8
+2024-06-12 18:53:47,100 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: poll_exit
+2024-06-12 18:53:47,101 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: defer
+2024-06-12 18:53:47,101 INFO    HandlerThread:60040 [handler.py:handle_request_defer():184] handle defer: 8
+2024-06-12 18:53:47,102 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: defer
+2024-06-12 18:53:47,103 INFO    SenderThread:60040 [sender.py:send_request_defer():609] handle sender defer: 8
+2024-06-12 18:53:47,103 INFO    SenderThread:60040 [job_builder.py:build():432] Attempting to build job artifact
+2024-06-12 18:53:47,103 INFO    SenderThread:60040 [job_builder.py:_get_source_type():565] is repo sourced job
+2024-06-12 18:53:47,103 INFO    SenderThread:60040 [job_builder.py:_get_program_relpath():583] run is notebook based run
+2024-06-12 18:53:47,104 WARNING SenderThread:60040 [job_builder.py:_log_if_verbose():267] Source type is set to 'repo' but some required information is missing from the environment. A job will not be created from this run. See https://docs.wandb.ai/guides/launch/create-job
+2024-06-12 18:53:47,104 INFO    SenderThread:60040 [sender.py:transition_state():613] send defer: 9
+2024-06-12 18:53:47,104 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: defer
+2024-06-12 18:53:47,104 INFO    HandlerThread:60040 [handler.py:handle_request_defer():184] handle defer: 9
+2024-06-12 18:53:47,105 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: defer
+2024-06-12 18:53:47,105 INFO    SenderThread:60040 [sender.py:send_request_defer():609] handle sender defer: 9
+2024-06-12 18:53:47,105 INFO    SenderThread:60040 [dir_watcher.py:finish():358] shutting down directory watcher
+2024-06-12 18:53:47,268 INFO    SenderThread:60040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/output.log
+2024-06-12 18:53:47,268 INFO    SenderThread:60040 [dir_watcher.py:finish():388] scan: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files
+2024-06-12 18:53:47,269 INFO    SenderThread:60040 [dir_watcher.py:finish():402] scan save: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/wandb-metadata.json wandb-metadata.json
+2024-06-12 18:53:47,269 INFO    SenderThread:60040 [dir_watcher.py:finish():402] scan save: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/conda-environment.yaml conda-environment.yaml
+2024-06-12 18:53:47,269 INFO    SenderThread:60040 [dir_watcher.py:finish():402] scan save: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/requirements.txt requirements.txt
+2024-06-12 18:53:47,269 INFO    SenderThread:60040 [dir_watcher.py:finish():402] scan save: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/output.log output.log
+2024-06-12 18:53:47,269 INFO    SenderThread:60040 [dir_watcher.py:finish():402] scan save: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/config.yaml config.yaml
+2024-06-12 18:53:47,270 INFO    SenderThread:60040 [dir_watcher.py:finish():402] scan save: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/wandb-summary.json wandb-summary.json
+2024-06-12 18:53:47,270 INFO    SenderThread:60040 [sender.py:transition_state():613] send defer: 10
+2024-06-12 18:53:47,270 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: defer
+2024-06-12 18:53:47,271 INFO    HandlerThread:60040 [handler.py:handle_request_defer():184] handle defer: 10
+2024-06-12 18:53:47,276 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: defer
+2024-06-12 18:53:47,276 INFO    SenderThread:60040 [sender.py:send_request_defer():609] handle sender defer: 10
+2024-06-12 18:53:47,277 INFO    SenderThread:60040 [file_pusher.py:finish():169] shutting down file pusher
+2024-06-12 18:53:47,675 INFO    wandb-upload_0:60040 [upload_job.py:push():130] Uploaded file /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/conda-environment.yaml
+2024-06-12 18:53:47,774 INFO    wandb-upload_2:60040 [upload_job.py:push():130] Uploaded file /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/output.log
+2024-06-12 18:53:47,797 INFO    wandb-upload_1:60040 [upload_job.py:push():130] Uploaded file /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/requirements.txt
+2024-06-12 18:53:47,801 INFO    wandb-upload_4:60040 [upload_job.py:push():130] Uploaded file /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/wandb-summary.json
+2024-06-12 18:53:47,812 INFO    wandb-upload_3:60040 [upload_job.py:push():130] Uploaded file /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/files/config.yaml
+2024-06-12 18:53:48,013 INFO    Thread-11 (_thread_body):60040 [sender.py:transition_state():613] send defer: 11
+2024-06-12 18:53:48,014 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: defer
+2024-06-12 18:53:48,014 INFO    HandlerThread:60040 [handler.py:handle_request_defer():184] handle defer: 11
+2024-06-12 18:53:48,015 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: defer
+2024-06-12 18:53:48,015 INFO    SenderThread:60040 [sender.py:send_request_defer():609] handle sender defer: 11
+2024-06-12 18:53:48,015 INFO    SenderThread:60040 [file_pusher.py:join():175] waiting for file pusher
+2024-06-12 18:53:48,016 INFO    SenderThread:60040 [sender.py:transition_state():613] send defer: 12
+2024-06-12 18:53:48,016 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: defer
+2024-06-12 18:53:48,016 INFO    HandlerThread:60040 [handler.py:handle_request_defer():184] handle defer: 12
+2024-06-12 18:53:48,017 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: defer
+2024-06-12 18:53:48,017 INFO    SenderThread:60040 [sender.py:send_request_defer():609] handle sender defer: 12
+2024-06-12 18:53:48,017 INFO    SenderThread:60040 [file_stream.py:finish():601] file stream finish called
+2024-06-12 18:53:48,050 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: poll_exit
+2024-06-12 18:53:48,169 INFO    SenderThread:60040 [file_stream.py:finish():605] file stream finish is done
+2024-06-12 18:53:48,169 INFO    SenderThread:60040 [sender.py:transition_state():613] send defer: 13
+2024-06-12 18:53:48,170 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: poll_exit
+2024-06-12 18:53:48,170 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: defer
+2024-06-12 18:53:48,171 INFO    HandlerThread:60040 [handler.py:handle_request_defer():184] handle defer: 13
+2024-06-12 18:53:48,171 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: defer
+2024-06-12 18:53:48,171 INFO    SenderThread:60040 [sender.py:send_request_defer():609] handle sender defer: 13
+2024-06-12 18:53:48,172 INFO    SenderThread:60040 [sender.py:transition_state():613] send defer: 14
+2024-06-12 18:53:48,172 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: defer
+2024-06-12 18:53:48,172 INFO    HandlerThread:60040 [handler.py:handle_request_defer():184] handle defer: 14
+2024-06-12 18:53:48,173 DEBUG   SenderThread:60040 [sender.py:send():378] send: final
+2024-06-12 18:53:48,173 DEBUG   SenderThread:60040 [sender.py:send():378] send: footer
+2024-06-12 18:53:48,173 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: defer
+2024-06-12 18:53:48,173 INFO    SenderThread:60040 [sender.py:send_request_defer():609] handle sender defer: 14
+2024-06-12 18:53:48,175 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: poll_exit
+2024-06-12 18:53:48,176 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: poll_exit
+2024-06-12 18:53:48,179 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: internal_messages
+2024-06-12 18:53:48,180 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: server_info
+2024-06-12 18:53:48,181 DEBUG   SenderThread:60040 [sender.py:send_request():405] send_request: server_info
+2024-06-12 18:53:48,223 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: get_summary
+2024-06-12 18:53:48,224 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: sampled_history
+2024-06-12 18:53:48,344 DEBUG   HandlerThread:60040 [handler.py:handle_request():158] handle_request: shutdown
+2024-06-12 18:53:48,345 INFO    HandlerThread:60040 [handler.py:finish():882] shutting down handler
+2024-06-12 18:53:49,181 INFO    WriterThread:60040 [datastore.py:close():296] close: /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/run-2vmzqu3d.wandb
+2024-06-12 18:53:49,343 INFO    SenderThread:60040 [sender.py:finish():1545] shutting down sender
+2024-06-12 18:53:49,344 INFO    SenderThread:60040 [file_pusher.py:finish():169] shutting down file pusher
+2024-06-12 18:53:49,344 INFO    SenderThread:60040 [file_pusher.py:join():175] waiting for file pusher

CellPilot/notebooks/wandb/run-20240612_184958-2vmzqu3d/logs/debug.log

@@ -0,0 +1,88 @@
+2024-06-12 18:49:58,661 INFO    MainThread:58594 [wandb_setup.py:_flush():76] Current SDK version is 0.17.0
+2024-06-12 18:49:58,662 INFO    MainThread:58594 [wandb_setup.py:_flush():76] Configure stats pid to 58594
+2024-06-12 18:49:58,662 INFO    MainThread:58594 [wandb_setup.py:_flush():76] Loading settings from /home/ubuntu/.config/wandb/settings
+2024-06-12 18:49:58,662 INFO    MainThread:58594 [wandb_setup.py:_flush():76] Loading settings from /home/ubuntu/thesis/SAMHI/notebooks/wandb/settings
+2024-06-12 18:49:58,663 INFO    MainThread:58594 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-06-12 18:49:58,663 INFO    MainThread:58594 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-06-12 18:49:58,663 INFO    MainThread:58594 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program': '<python with no main file>'}
+2024-06-12 18:49:58,663 INFO    MainThread:58594 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-06-12 18:49:58,664 INFO    MainThread:58594 [wandb_init.py:_log_setup():520] Logging user logs to /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/logs/debug.log
+2024-06-12 18:49:58,664 INFO    MainThread:58594 [wandb_init.py:_log_setup():521] Logging internal logs to /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_184958-2vmzqu3d/logs/debug-internal.log
+2024-06-12 18:49:58,664 INFO    MainThread:58594 [wandb_init.py:_jupyter_setup():466] configuring jupyter hooks <wandb.sdk.wandb_init._WandbInit object at 0x7fcce84ff070>
+2024-06-12 18:49:58,665 INFO    MainThread:58594 [wandb_init.py:init():560] calling init triggers
+2024-06-12 18:49:58,665 INFO    MainThread:58594 [wandb_init.py:init():567] wandb.init called with sweep_config: {}
+config: {}
+2024-06-12 18:49:58,666 INFO    MainThread:58594 [wandb_init.py:init():610] starting backend
+2024-06-12 18:49:58,666 INFO    MainThread:58594 [wandb_init.py:init():614] setting up manager
+2024-06-12 18:49:58,668 INFO    MainThread:58594 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-06-12 18:49:58,670 INFO    MainThread:58594 [wandb_init.py:init():622] backend started and connected
+2024-06-12 18:49:58,685 INFO    MainThread:58594 [wandb_run.py:_label_probe_notebook():1328] probe notebook
+2024-06-12 18:49:58,686 INFO    MainThread:58594 [wandb_run.py:_label_probe_notebook():1338] Unable to probe notebook: 'NoneType' object has no attribute 'get'
+2024-06-12 18:49:58,686 INFO    MainThread:58594 [wandb_init.py:init():711] updated telemetry
+2024-06-12 18:49:58,697 INFO    MainThread:58594 [wandb_init.py:init():744] communicating run to backend with 90.0 second timeout
+2024-06-12 18:49:59,163 INFO    MainThread:58594 [wandb_run.py:_on_init():2396] communicating current version
+2024-06-12 18:49:59,251 INFO    MainThread:58594 [wandb_run.py:_on_init():2405] got version response upgrade_message: "wandb version 0.17.1 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
+
+2024-06-12 18:49:59,252 INFO    MainThread:58594 [wandb_init.py:init():795] starting run threads in backend
+2024-06-12 18:50:02,295 INFO    MainThread:58594 [wandb_run.py:_console_start():2374] atexit reg
+2024-06-12 18:50:02,296 INFO    MainThread:58594 [wandb_run.py:_redirect():2229] redirect: wrap_raw
+2024-06-12 18:50:02,296 INFO    MainThread:58594 [wandb_run.py:_redirect():2294] Wrapping output streams.
+2024-06-12 18:50:02,296 INFO    MainThread:58594 [wandb_run.py:_redirect():2319] Redirects installed.
+2024-06-12 18:50:02,297 INFO    MainThread:58594 [wandb_init.py:init():838] run started, returning control to user process
+2024-06-12 18:50:05,538 INFO    MainThread:58594 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-12 18:50:05,538 INFO    MainThread:58594 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-12 18:50:05,602 INFO    MainThread:58594 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-12 18:50:05,603 INFO    MainThread:58594 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-12 18:50:05,603 INFO    MainThread:58594 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-12 18:50:27,343 INFO    MainThread:58594 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-12 18:50:33,879 INFO    MainThread:58594 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-12 18:50:33,880 INFO    MainThread:58594 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-12 18:53:35,079 INFO    MainThread:58594 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-12 18:53:35,088 INFO    MainThread:58594 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-12 18:53:35,088 INFO    MainThread:58594 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-12 18:53:36,133 INFO    MainThread:58594 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-12 18:53:36,144 INFO    MainThread:58594 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-12 18:53:36,145 INFO    MainThread:58594 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-12 18:53:38,897 INFO    MainThread:58594 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-12 18:53:38,904 INFO    MainThread:58594 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-12 18:53:38,905 INFO    MainThread:58594 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-12 18:53:39,570 INFO    MainThread:58594 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-12 18:53:39,577 INFO    MainThread:58594 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-12 18:53:39,577 INFO    MainThread:58594 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-12 18:53:39,742 INFO    MainThread:58594 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-12 18:53:39,749 INFO    MainThread:58594 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-12 18:53:39,750 INFO    MainThread:58594 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-12 18:53:40,107 INFO    MainThread:58594 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-12 18:53:40,116 INFO    MainThread:58594 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-12 18:53:40,117 INFO    MainThread:58594 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-12 18:53:41,411 INFO    MainThread:58594 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-12 18:53:41,421 INFO    MainThread:58594 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-12 18:53:41,421 INFO    MainThread:58594 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-12 18:53:41,596 INFO    MainThread:58594 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-12 18:53:41,604 INFO    MainThread:58594 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-12 18:53:41,605 INFO    MainThread:58594 [wandb_init.py:_pause_backend():431] pausing backend
+2024-06-12 18:53:43,998 INFO    MainThread:58594 [wandb_init.py:_resume_backend():436] resuming backend
+2024-06-12 18:53:44,033 INFO    MainThread:58594 [wandb_setup.py:_flush():76] Current SDK version is 0.17.0
+2024-06-12 18:53:44,033 INFO    MainThread:58594 [wandb_setup.py:_flush():76] Configure stats pid to 58594
+2024-06-12 18:53:44,034 INFO    MainThread:58594 [wandb_setup.py:_flush():76] Loading settings from /home/ubuntu/.config/wandb/settings
+2024-06-12 18:53:44,035 INFO    MainThread:58594 [wandb_setup.py:_flush():76] Loading settings from /home/ubuntu/thesis/SAMHI/notebooks/wandb/settings
+2024-06-12 18:53:44,035 INFO    MainThread:58594 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2024-06-12 18:53:44,036 INFO    MainThread:58594 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
+2024-06-12 18:53:44,036 INFO    MainThread:58594 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program': '<python with no main file>'}
+2024-06-12 18:53:44,037 INFO    MainThread:58594 [wandb_setup.py:_flush():76] Applying login settings: {}
+2024-06-12 18:53:44,038 INFO    MainThread:58594 [wandb_init.py:_log_setup():520] Logging user logs to /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_185344-3wpkmoo8/logs/debug.log
+2024-06-12 18:53:44,038 INFO    MainThread:58594 [wandb_init.py:_log_setup():521] Logging internal logs to /home/ubuntu/thesis/SAMHI/notebooks/wandb/run-20240612_185344-3wpkmoo8/logs/debug-internal.log
+2024-06-12 18:53:44,039 INFO    MainThread:58594 [wandb_init.py:init():560] calling init triggers
+2024-06-12 18:53:44,040 INFO    MainThread:58594 [wandb_init.py:init():567] wandb.init called with sweep_config: {}
+config: {}
+2024-06-12 18:53:44,040 INFO    MainThread:58594 [wandb_init.py:init():585] re-initializing run, found existing run on stack: 2vmzqu3d
+2024-06-12 18:53:44,044 INFO    MainThread:58594 [wandb_run.py:_finish():2103] finishing run philippresearch/SAMHI/2vmzqu3d
+2024-06-12 18:53:44,045 INFO    MainThread:58594 [jupyter.py:save_history():445] not saving jupyter history
+2024-06-12 18:53:44,045 INFO    MainThread:58594 [jupyter.py:save_ipynb():373] not saving jupyter notebook
+2024-06-12 18:53:44,046 INFO    MainThread:58594 [wandb_init.py:_jupyter_teardown():448] cleaning up jupyter logic
+2024-06-12 18:53:44,046 INFO    MainThread:58594 [wandb_run.py:_atexit_cleanup():2343] got exitcode: 0
+2024-06-12 18:53:44,046 INFO    MainThread:58594 [wandb_run.py:_restore():2326] restore
+2024-06-12 18:53:44,047 INFO    MainThread:58594 [wandb_run.py:_restore():2332] restore done
+2024-06-12 18:53:49,347 INFO    MainThread:58594 [wandb_run.py:_footer_history_summary_info():3994] rendering history
+2024-06-12 18:53:49,347 INFO    MainThread:58594 [wandb_run.py:_footer_history_summary_info():4026] rendering summary
+2024-06-12 18:53:49,358 INFO    MainThread:58594 [wandb_run.py:_footer_sync_info():3953] logging synced files