diff --git a/.gitattributes b/.gitattributes index 0bb2c110bec125f5e60bc569bb8f99cbaee2a340..4f0c26be8ff509d6aa08a723a290686fb1908058 100644 --- a/.gitattributes +++ b/.gitattributes @@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text zipdeepness/deepness/images/icon.png filter=lfs diff=lfs merge=lfs -text +zipdeepness/images/icon.png filter=lfs diff=lfs merge=lfs -text diff --git a/zipdeepness/README.md b/zipdeepness/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e8d22bce930e79342f58cd0cdef5305e8be62454 --- /dev/null +++ b/zipdeepness/README.md @@ -0,0 +1,7 @@ +# Deepness: Deep Neural Remote Sensing + +Plugin for QGIS to perform map/image segmentation, regression and object detection with (ONNX) neural network models. + +Please visit the documentation webpage for details: https://qgis-plugin-deepness.readthedocs.io/ + +Or the repository: https://github.com/PUTvision/qgis-plugin-deepness diff --git a/zipdeepness/__init__.py b/zipdeepness/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f39b0a2ef08e2c60b2c0ef47be3d85a38bbbc57c --- /dev/null +++ b/zipdeepness/__init__.py @@ -0,0 +1,19 @@ +"""Main plugin module - entry point for the plugin.""" +import os + +# increase limit of pixels (2^30), before importing cv2. +# We are doing it here to make sure it will be done before importing cv2 for the first time +os.environ["OPENCV_IO_MAX_IMAGE_PIXELS"] = pow(2, 40).__str__() + + +# noinspection PyPep8Naming +def classFactory(iface): # pylint: disable=invalid-name + """Load Deepness class from file Deepness. + :param iface: A QGIS interface instance. + :type iface: QgsInterface + """ + from deepness.dialogs.packages_installer import packages_installer_dialog + packages_installer_dialog.check_required_packages_and_install_if_necessary(iface=iface) + + from deepness.deepness import Deepness + return Deepness(iface) \ No newline at end of file diff --git a/zipdeepness/common/__init__.py b/zipdeepness/common/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c4b1577d62082b1c69aa61c20449064def038db9 --- /dev/null +++ b/zipdeepness/common/__init__.py @@ -0,0 +1,2 @@ +""" Submodule that contains the common functions for the deepness plugin. +""" diff --git a/zipdeepness/common/channels_mapping.py b/zipdeepness/common/channels_mapping.py new file mode 100644 index 0000000000000000000000000000000000000000..5cf5e7374a564c7cbf6208b7fe40c57e39f3086d --- /dev/null +++ b/zipdeepness/common/channels_mapping.py @@ -0,0 +1,262 @@ +""" +Raster layer (ortophoto) which is being processed consist of channels (usually Red, Green, Blue). +The neural model expects input channels with some model-defined meaning. +Channel mappings in this file define how the ortophoto channels translate to model inputs (e.g first model input is Red, second Green). +""" + +import copy +from typing import Dict, List + + +class ImageChannel: + """ + Defines an image channel - how is it being stored in the data source. + See note at top of this file for details. + """ + + def __init__(self, name): + self.name = name + + def get_band_number(self): + raise NotImplementedError('Base class not implemented!') + + def get_byte_number(self): + raise NotImplementedError('Base class not implemented!') + + +class ImageChannelStandaloneBand(ImageChannel): + """ + Defines an image channel, where each image channel is a separate band in the data source. + See note at top of this file for details. + """ + + def __init__(self, band_number: int, name: str): + super().__init__(name) + self.band_number = band_number # index within bands (counted from one) + + def __str__(self): + txt = f'ImageChannelStandaloneBand(name={self.name}, ' \ + f'band_number={self.band_number})' + return txt + + def get_band_number(self): + return self.band_number + + def get_byte_number(self): + raise NotImplementedError('Something went wrong if we are here!') + + +class ImageChannelCompositeByte(ImageChannel): + """ + Defines an image channel, where each image channel is a smaller part of a bigger value (e.g. one byte within uint32 for each pixel). + See note at top of this file for details. + """ + + def __init__(self, byte_number: int, name: str): + super().__init__(name) + self.byte_number = byte_number # position in composite byte (byte number in ARGB32, counted from zero) + + def __str__(self): + txt = f'ImageChannelCompositeByte(name={self.name}, ' \ + f'byte_number={self.byte_number})' + return txt + + def get_band_number(self): + raise NotImplementedError('Something went wrong if we are here!') + + def get_byte_number(self): + return self.byte_number + + +class ChannelsMapping: + """ + Defines mapping of model input channels to input image channels (bands). + See note at top of this file for details. + """ + + INVALID_INPUT_CHANNEL = -1 + + def __init__(self): + self._number_of_model_inputs = 0 + self._number_of_model_output_channels = 0 + self._image_channels = [] # type: List[ImageChannel] # what channels are available from input image + + # maps model channels to input image channels + # model_channel_number: image_channel_index (index in self._image_channels) + self._mapping = {} # type: Dict[int, int] + + def __str__(self): + txt = f'ChannelsMapping(' \ + f'number_of_model_inputs={self._number_of_model_inputs}, ' \ + f'image_channels = {self._image_channels}, ' \ + f'mapping {self._mapping})' + return txt + + def __eq__(self, other): + if self._number_of_model_inputs != other._number_of_model_inputs: + return False + return True + + def get_as_default_mapping(self): + """ + Get the same channels mapping as we have right now, but without the mapping itself + (so just a definition of inputs and outputs) + + Returns + ------- + ChannelsMapping + """ + default_channels_mapping = copy.deepcopy(self) + default_channels_mapping._mapping = {} + return default_channels_mapping + + def are_all_inputs_standalone_bands(self): + """ + Checks whether all image_channels are standalone bands (ImageChannelStandaloneBand) + """ + for image_channel in self._image_channels: + if not isinstance(image_channel, ImageChannelStandaloneBand): + return False + return True + + def are_all_inputs_composite_byte(self): + """ + Checks whether all image_channels are composite byte (ImageChannelCompositeByte) + """ + for image_channel in self._image_channels: + if not isinstance(image_channel, ImageChannelCompositeByte): + return False + return True + + def set_number_of_model_inputs(self, number_of_model_inputs: int): + """ Set how many input channels does the model has + Parameters + ---------- + number_of_model_inputs : int + """ + self._number_of_model_inputs = number_of_model_inputs + + def set_number_of_model_output_channels(self, number_of_output_channels: int): + """ Set how many output channels does the model has + + Parameters + ---------- + number_of_output_channels : int + """ + self._number_of_model_output_channels = number_of_output_channels + + def set_number_of_model_inputs_same_as_image_channels(self): + """ Set the number of model input channels to be the same as number of image channels + """ + self._number_of_model_inputs = len(self._image_channels) + + def get_number_of_model_inputs(self) -> int: + """ Get number of model input channels + + Returns + ------- + int + """ + return self._number_of_model_inputs + + def get_number_of_model_output_channels(self) -> int: + """ Get number of model output channels + + Returns + ------- + int + """ + return self._number_of_model_output_channels + + def get_number_of_image_channels(self) -> int: + """ Get number of image input channels + + Returns + ------- + int + """ + return len(self._image_channels) + + def set_image_channels(self, image_channels: List[ImageChannel]): + """ Set what are the image channels + + Parameters + ---------- + image_channels : List[ImageChannel] + Image channels to set + """ + self._image_channels = image_channels + if not self.are_all_inputs_standalone_bands() and not self.are_all_inputs_composite_byte(): + raise Exception("Unsupported image channels composition!") + + def get_image_channels(self) -> List[ImageChannel]: + """ Get the current image channels definition + + Returns + ------- + List[ImageChannel] + """ + return self._image_channels + + def get_image_channel_index_for_model_input(self, model_input_number) -> int: + """ + Similar to 'get_image_channel_for_model_input', but return an index in array of inputs, + instead of ImageChannel + """ + if len(self._image_channels) == 0: + raise Exception("No image channels!") + + image_channel_index = self._mapping.get(model_input_number, model_input_number) + image_channel_index = min(image_channel_index, len(self._image_channels) - 1) + return image_channel_index + + def get_image_channel_for_model_input(self, model_input_number: int) -> ImageChannel: + """ + Get ImageChannel which should be used for the specified model input + + Parameters + ---------- + model_input_number : int + Model input number, counted from 0 + + Returns + ------- + ImageChannel + """ + image_channel_index = self.get_image_channel_index_for_model_input(model_input_number) + return self._image_channels[image_channel_index] + + def set_image_channel_for_model_input(self, model_input_number: int, image_channel_index: int) -> ImageChannel: + """ + Set image_channel_index which should be used for this model input + """ + if image_channel_index >= len(self._image_channels): + raise Exception("Invalid image channel index!") + # image_channel = self._image_channels[image_channel_index] + self._mapping[model_input_number] = image_channel_index + + def get_mapping_as_list(self) -> List[int]: + """ Get the mapping of model input channels to image channels, but as a list (e.g. to store it in QGis configuration) + + Returns + ------- + List[int] + """ + mapping_list = [] + for i in range(self._number_of_model_inputs): + if i in self._mapping: + mapping_list.append(self._mapping[i]) + else: + mapping_list.append(-1) + return mapping_list + + def load_mapping_from_list(self, mapping_list: List[int]): + """ + Load self._mapping from a plain list of channels (which is saved in config) + """ + for i in range(min(self._number_of_model_inputs), len(mapping_list)): + proposed_channel = mapping_list[i] + if proposed_channel == -1 or proposed_channel >= self._number_of_model_inputs: + continue + + self._mapping[i] = proposed_channel diff --git a/zipdeepness/common/config_entry_key.py b/zipdeepness/common/config_entry_key.py new file mode 100644 index 0000000000000000000000000000000000000000..546aa3112a19be3cfa4ec9cb945a11b79699f411 --- /dev/null +++ b/zipdeepness/common/config_entry_key.py @@ -0,0 +1,95 @@ +""" +This file contains utilities to write and read configuration parameters to the QGis Project configuration +""" + +import enum + +from qgis.core import QgsProject + +from deepness.common.defines import PLUGIN_NAME + + +class ConfigEntryKey(enum.Enum): + """ + Entries to be stored in Project Configuration. + Second element of enum value (in tuple) is the default value for this field + """ + + MODEL_FILE_PATH = enum.auto(), '' # Path to the model file + INPUT_LAYER_ID = enum.auto(), '' + PROCESSED_AREA_TYPE = enum.auto(), '' # string of ProcessedAreaType, e.g. "ProcessedAreaType.VISIBLE_PART.value" + MODEL_TYPE = enum.auto(), '' # string of ModelType enum, e.g. "ModelType.SEGMENTATION.value" + PREPROCESSING_RESOLUTION = enum.auto(), 3.0 + MODEL_BATCH_SIZE = enum.auto(), 1 + PROCESS_LOCAL_CACHE = enum.auto(), False + PREPROCESSING_TILES_OVERLAP = enum.auto(), 15 + + SEGMENTATION_PROBABILITY_THRESHOLD_ENABLED = enum.auto(), True + SEGMENTATION_PROBABILITY_THRESHOLD_VALUE = enum.auto(), 0.5 + SEGMENTATION_REMOVE_SMALL_SEGMENT_ENABLED = enum.auto(), True + SEGMENTATION_REMOVE_SMALL_SEGMENT_SIZE = enum.auto(), 9 + + REGRESSION_OUTPUT_SCALING = enum.auto(), 1.0 + + DETECTION_CONFIDENCE = enum.auto(), 0.5 + DETECTION_IOU = enum.auto(), 0.5 + DETECTOR_TYPE = enum.auto(), 'YOLO_v5_v7_DEFAULT' + + DATA_EXPORT_DIR = enum.auto(), '' + DATA_EXPORT_TILES_ENABLED = enum.auto(), True + DATA_EXPORT_SEGMENTATION_MASK_ENABLED = enum.auto(), False + DATA_EXPORT_SEGMENTATION_MASK_ID = enum.auto(), '' + + INPUT_CHANNELS_MAPPING__ADVANCED_MODE = enum.auto, False + INPUT_CHANNELS_MAPPING__MAPPING_LIST_STR = enum.auto, [] + + def get(self): + """ + Get the value store in config (or a default one) for the specified field + """ + read_function = None + + # check the default value to determine the entry type + default_value = self.value[1] # second element in the 'value' tuple + if isinstance(default_value, int): + read_function = QgsProject.instance().readNumEntry + elif isinstance(default_value, float): + read_function = QgsProject.instance().readDoubleEntry + elif isinstance(default_value, bool): + read_function = QgsProject.instance().readBoolEntry + elif isinstance(default_value, str): + read_function = QgsProject.instance().readEntry + elif isinstance(default_value, str): + read_function = QgsProject.instance().readListEntry + else: + raise Exception("Unsupported entry type!") + + value, _ = read_function(PLUGIN_NAME, self.name, default_value) + return value + + def set(self, value): + """ Set the value store in config, for the specified field + + Parameters + ---------- + value : + Value to set in the configuration + """ + write_function = None + + # check the default value to determine the entry type + default_value = self.value[1] # second element in the 'value' tuple + if isinstance(default_value, int): + write_function = QgsProject.instance().writeEntry + elif isinstance(default_value, float): + write_function = QgsProject.instance().writeEntryDouble + elif isinstance(default_value, bool): + write_function = QgsProject.instance().writeEntryBool + elif isinstance(default_value, str): + write_function = QgsProject.instance().writeEntry + elif isinstance(default_value, list): + write_function = QgsProject.instance().writeEntry + else: + raise Exception("Unsupported entry type!") + + write_function(PLUGIN_NAME, self.name, value) diff --git a/zipdeepness/common/defines.py b/zipdeepness/common/defines.py new file mode 100644 index 0000000000000000000000000000000000000000..c78854ddcbafaf232be4c1e19060785801f5cbfd --- /dev/null +++ b/zipdeepness/common/defines.py @@ -0,0 +1,12 @@ +""" +This file contain common definitions used in the project +""" + +import os + +PLUGIN_NAME = 'Deepness' +LOG_TAB_NAME = PLUGIN_NAME + + +# enable some debugging options (e.g. printing exceptions) - set in terminal before running qgis +IS_DEBUG = os.getenv("IS_DEBUG", 'False').lower() in ('true', '1', 't') diff --git a/zipdeepness/common/errors.py b/zipdeepness/common/errors.py new file mode 100644 index 0000000000000000000000000000000000000000..e56b11747e0e43266fd72a15ad8c1149105a63a3 --- /dev/null +++ b/zipdeepness/common/errors.py @@ -0,0 +1,10 @@ +""" +This file contains common exceptions used in the project +""" + + +class OperationFailedException(Exception): + """ + Base class for a failed operation, in order to have a good error message to show for the user + """ + pass diff --git a/zipdeepness/common/lazy_package_loader.py b/zipdeepness/common/lazy_package_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..51365ae32c044dcf0bdb8e471d19eabc6f7520d4 --- /dev/null +++ b/zipdeepness/common/lazy_package_loader.py @@ -0,0 +1,24 @@ +""" +This file contains utility to lazy import packages +""" + +import importlib + + +class LazyPackageLoader: + """ Allows to wrap python package into a lazy version, so that the package will be loaded once it is actually used + + Usage: + cv2 = LazyPackageLoader('cv2') # This will not import cv2 yet + ... + cv2.waitKey(3) # here will be the actual import + """ + + def __init__(self, package_name): + self._package_name = package_name + self._package = None + + def __getattr__(self, name): + if self._package is None: + self._package = importlib.import_module(self._package_name) + return getattr(self._package, name) diff --git a/zipdeepness/common/misc.py b/zipdeepness/common/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..e6078449c4f0c41ebbeaa4068fefc28a6f891850 --- /dev/null +++ b/zipdeepness/common/misc.py @@ -0,0 +1,9 @@ +""" +This file contains miscellaneous stuff used in the project +""" + +import os +import tempfile + +_TMP_DIR = tempfile.TemporaryDirectory() +TMP_DIR_PATH = os.path.join(_TMP_DIR.name, 'qgis') diff --git a/zipdeepness/common/processing_overlap.py b/zipdeepness/common/processing_overlap.py new file mode 100644 index 0000000000000000000000000000000000000000..a7c50f6403801c48e5f4589c8811a200677ceb28 --- /dev/null +++ b/zipdeepness/common/processing_overlap.py @@ -0,0 +1,37 @@ +import enum +from typing import Dict, List + + +class ProcessingOverlapOptions(enum.Enum): + OVERLAP_IN_PIXELS = 'Overlap in pixels' + OVERLAP_IN_PERCENT = 'Overlap in percent' + + +class ProcessingOverlap: + """ Represents overlap between tiles during processing + """ + def __init__(self, selected_option: ProcessingOverlapOptions, percentage: float = None, overlap_px: int = None): + self.selected_option = selected_option + + if selected_option == ProcessingOverlapOptions.OVERLAP_IN_PERCENT and percentage is None: + raise Exception(f"Percentage must be specified when using {ProcessingOverlapOptions.OVERLAP_IN_PERCENT}") + if selected_option == ProcessingOverlapOptions.OVERLAP_IN_PIXELS and overlap_px is None: + raise Exception(f"Overlap in pixels must be specified when using {ProcessingOverlapOptions.OVERLAP_IN_PIXELS}") + + if selected_option == ProcessingOverlapOptions.OVERLAP_IN_PERCENT: + self._percentage = percentage + elif selected_option == ProcessingOverlapOptions.OVERLAP_IN_PIXELS: + self._overlap_px = overlap_px + else: + raise Exception(f"Unknown option: {selected_option}") + + def get_overlap_px(self, tile_size_px: int) -> int: + """ Returns the overlap in pixels + + :param tile_size_px: Tile size in pixels + :return: Returns the overlap in pixels + """ + if self.selected_option == ProcessingOverlapOptions.OVERLAP_IN_PIXELS: + return self._overlap_px + else: + return int(tile_size_px * self._percentage / 100 * 2) // 2 # TODO: check if this is correct diff --git a/zipdeepness/common/processing_parameters/__init__.py b/zipdeepness/common/processing_parameters/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/zipdeepness/common/processing_parameters/detection_parameters.py b/zipdeepness/common/processing_parameters/detection_parameters.py new file mode 100644 index 0000000000000000000000000000000000000000..c96232c607fe30bf291da238449f01122642e907 --- /dev/null +++ b/zipdeepness/common/processing_parameters/detection_parameters.py @@ -0,0 +1,70 @@ +import enum +from dataclasses import dataclass + +from deepness.common.processing_parameters.map_processing_parameters import MapProcessingParameters +from deepness.processing.models.model_base import ModelBase + + +@dataclass +class DetectorTypeParameters: + """ + Defines some model-specific parameters for each model 'type' (e.g. default YOLOv7 has different model output shape than the one trained with Ultralytics' YOLO) + """ + has_inverted_output_shape: bool = False # whether the output shape of the model is inverted (and we need to apply np.transpose(model_output, (1, 0))) + skipped_objectness_probability: bool = False # whether the model output has has no 'objectness' probability, and only probability for each class + ignore_objectness_probability: bool = False # if the model output has the 'objectness' probability, we can still ignore it (it is needeed sometimes, when the probability was always 1...). The behavior should be the same as with `skipped_objectness_probability` (of course one model output needs be skipped) + + +class DetectorType(enum.Enum): + """ Type of the detector model """ + + YOLO_v5_v7_DEFAULT = 'YOLO_v5_or_v7_default' + YOLO_v6 = 'YOLO_v6' + YOLO_v9 = 'YOLO_v9' + YOLO_ULTRALYTICS = 'YOLO_Ultralytics' + YOLO_ULTRALYTICS_SEGMENTATION = 'YOLO_Ultralytics_segmentation' + YOLO_ULTRALYTICS_OBB = 'YOLO_Ultralytics_obb' + + def get_parameters(self): + if self == DetectorType.YOLO_v5_v7_DEFAULT: + return DetectorTypeParameters() # all default + elif self == DetectorType.YOLO_v6: + return DetectorTypeParameters( + ignore_objectness_probability=True, + ) + elif self == DetectorType.YOLO_v9: + return DetectorTypeParameters( + has_inverted_output_shape=True, + skipped_objectness_probability=True, + ) + elif self == DetectorType.YOLO_ULTRALYTICS or self == DetectorType.YOLO_ULTRALYTICS_SEGMENTATION or self == DetectorType.YOLO_ULTRALYTICS_OBB: + return DetectorTypeParameters( + has_inverted_output_shape=True, + skipped_objectness_probability=True, + ) + else: + raise ValueError(f'Unknown detector type: {self}') + + def get_formatted_description(self): + txt = '' + txt += ' ' * 10 + f'Inverted output shape: {self.get_parameters().has_inverted_output_shape}\n' + txt += ' ' * 10 + f'Skipped objectness : {self.get_parameters().skipped_objectness_probability}\n' + txt += ' ' * 10 + f'Ignore objectness: {self.get_parameters().ignore_objectness_probability}\n' + return txt + + def get_all_display_values(): + return [x.value for x in DetectorType] + + +@dataclass +class DetectionParameters(MapProcessingParameters): + """ + Parameters for Inference of detection model (including pre/post-processing) obtained from UI. + """ + + model: ModelBase # wrapper of the loaded model + + confidence: float + iou_threshold: float + + detector_type: DetectorType = DetectorType.YOLO_v5_v7_DEFAULT # parameters specific for each model type diff --git a/zipdeepness/common/processing_parameters/map_processing_parameters.py b/zipdeepness/common/processing_parameters/map_processing_parameters.py new file mode 100644 index 0000000000000000000000000000000000000000..755cf7459f8fd1c317e0576e1a65768f7ccf7437 --- /dev/null +++ b/zipdeepness/common/processing_parameters/map_processing_parameters.py @@ -0,0 +1,56 @@ +import enum +from dataclasses import dataclass +from typing import Optional + +from deepness.common.channels_mapping import ChannelsMapping +from deepness.common.processing_overlap import ProcessingOverlap + + +class ProcessedAreaType(enum.Enum): + VISIBLE_PART = 'Visible part' + ENTIRE_LAYER = 'Entire layer' + FROM_POLYGONS = 'From polygons' + + @classmethod + def get_all_names(cls): + return [e.value for e in cls] + +@dataclass +class MapProcessingParameters: + """ + Common parameters for map processing obtained from UI. + + TODO: Add default values here, to later set them in UI at startup + """ + + resolution_cm_per_px: float # image resolution to used during processing + processed_area_type: ProcessedAreaType # whether to perform operation on the entire field or part + tile_size_px: int # Tile size for processing (model input size) + batch_size: int # Batch size for processing + local_cache: bool # Whether to use local cache for tiles (on disk, /tmp directory) + + input_layer_id: str # raster layer to process + mask_layer_id: Optional[str] # Processing of masked layer - if processed_area_type is FROM_POLYGONS + + processing_overlap: ProcessingOverlap # aka "stride" - how much to overlap tiles during processing + + input_channels_mapping: ChannelsMapping # describes mapping of image channels to model inputs + + @property + def tile_size_m(self): + return self.tile_size_px * self.resolution_cm_per_px / 100 + + @property + def processing_overlap_px(self) -> int: + """ + Always divisible by 2, because overlap is on both sides of the tile + """ + return self.processing_overlap.get_overlap_px(self.tile_size_px) + + @property + def resolution_m_per_px(self): + return self.resolution_cm_per_px / 100 + + @property + def processing_stride_px(self): + return self.tile_size_px - self.processing_overlap_px diff --git a/zipdeepness/common/processing_parameters/recognition_parameters.py b/zipdeepness/common/processing_parameters/recognition_parameters.py new file mode 100644 index 0000000000000000000000000000000000000000..e350b6855a7d3e4c8b960c9aa609d354d7a144d4 --- /dev/null +++ b/zipdeepness/common/processing_parameters/recognition_parameters.py @@ -0,0 +1,17 @@ +import enum +from dataclasses import dataclass +from typing import Optional + +from deepness.common.processing_parameters.map_processing_parameters import \ + MapProcessingParameters +from deepness.processing.models.model_base import ModelBase + + +@dataclass +class RecognitionParameters(MapProcessingParameters): + """ + Parameters for Inference of Recognition model (including pre/post-processing) obtained from UI. + """ + + query_image_path: str # path to query image + model: ModelBase # wrapper of the loaded model diff --git a/zipdeepness/common/processing_parameters/regression_parameters.py b/zipdeepness/common/processing_parameters/regression_parameters.py new file mode 100644 index 0000000000000000000000000000000000000000..51e9eacc92ee34982869f7cb37e6b5b973498be4 --- /dev/null +++ b/zipdeepness/common/processing_parameters/regression_parameters.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass + +from deepness.common.processing_parameters.map_processing_parameters import MapProcessingParameters +from deepness.processing.models.model_base import ModelBase + + +@dataclass +class RegressionParameters(MapProcessingParameters): + """ + Parameters for Inference of Regression model (including pre/post-processing) obtained from UI. + """ + + output_scaling: float # scaling factor for the model output (keep 1 if maximum model output value is 1) + model: ModelBase # wrapper of the loaded model diff --git a/zipdeepness/common/processing_parameters/segmentation_parameters.py b/zipdeepness/common/processing_parameters/segmentation_parameters.py new file mode 100644 index 0000000000000000000000000000000000000000..bc69510d485011a9f44f6c2fb06a0c1c6a3cc459 --- /dev/null +++ b/zipdeepness/common/processing_parameters/segmentation_parameters.py @@ -0,0 +1,16 @@ +from dataclasses import dataclass + +from deepness.common.processing_parameters.map_processing_parameters import MapProcessingParameters +from deepness.processing.models.model_base import ModelBase + + +@dataclass +class SegmentationParameters(MapProcessingParameters): + """ + Parameters for Inference of Segmentation model (including pre/post-processing) obtained from UI. + """ + + postprocessing_dilate_erode_size: int # dilate/erode operation size, once we have a single class map. 0 if inactive. Implementation may use median filer instead of erode/dilate + model: ModelBase # wrapper of the loaded model + + pixel_classification__probability_threshold: float # Minimum required class probability for pixel. 0 if disabled diff --git a/zipdeepness/common/processing_parameters/standardization_parameters.py b/zipdeepness/common/processing_parameters/standardization_parameters.py new file mode 100644 index 0000000000000000000000000000000000000000..615ca3ed919fb752ae73b5273e31770e8491abb1 --- /dev/null +++ b/zipdeepness/common/processing_parameters/standardization_parameters.py @@ -0,0 +1,11 @@ +import numpy as np + + +class StandardizationParameters: + def __init__(self, channels_number: int): + self.mean = np.array([0.0 for _ in range(channels_number)], dtype=np.float32) + self.std = np.array([1.0 for _ in range(channels_number)], dtype=np.float32) + + def set_mean_std(self, mean: np.array, std: np.array): + self.mean = np.array(mean, dtype=np.float32) + self.std = np.array(std, dtype=np.float32) diff --git a/zipdeepness/common/processing_parameters/superresolution_parameters.py b/zipdeepness/common/processing_parameters/superresolution_parameters.py new file mode 100644 index 0000000000000000000000000000000000000000..036b7c631b8feb0ba108185de17e519427b3d669 --- /dev/null +++ b/zipdeepness/common/processing_parameters/superresolution_parameters.py @@ -0,0 +1,18 @@ +import enum +from dataclasses import dataclass +from typing import Optional + +from deepness.common.channels_mapping import ChannelsMapping +from deepness.common.processing_parameters.map_processing_parameters import MapProcessingParameters +from deepness.processing.models.model_base import ModelBase + + +@dataclass +class SuperresolutionParameters(MapProcessingParameters): + """ + Parameters for Inference of Super Resolution model (including pre/post-processing) obtained from UI. + """ + + output_scaling: float # scaling factor for the model output (keep 1 if maximum model output value is 1) + model: ModelBase # wrapper of the loaded model + scale_factor: int # scale factor for the model output size diff --git a/zipdeepness/common/processing_parameters/training_data_export_parameters.py b/zipdeepness/common/processing_parameters/training_data_export_parameters.py new file mode 100644 index 0000000000000000000000000000000000000000..b80f7ac6f1bce87cd1edd523bcfe68a13e18bf1b --- /dev/null +++ b/zipdeepness/common/processing_parameters/training_data_export_parameters.py @@ -0,0 +1,15 @@ +from dataclasses import dataclass +from typing import Optional + +from deepness.common.processing_parameters.map_processing_parameters import MapProcessingParameters + + +@dataclass +class TrainingDataExportParameters(MapProcessingParameters): + """ + Parameters for Exporting Data obtained from UI. + """ + + export_image_tiles: bool # whether to export input image tiles + segmentation_mask_layer_id: Optional[str] # id for mask, to be exported as separate tiles + output_directory_path: str # path where the output files will be saved diff --git a/zipdeepness/common/temp_files_handler.py b/zipdeepness/common/temp_files_handler.py new file mode 100644 index 0000000000000000000000000000000000000000..3962e23e5a0712c45b10df7d6e163704ca0004fb --- /dev/null +++ b/zipdeepness/common/temp_files_handler.py @@ -0,0 +1,19 @@ +import os.path as path +import shutil +from tempfile import mkdtemp + + +class TempFilesHandler: + def __init__(self) -> None: + self._temp_dir = mkdtemp() + + print(f'Created temp dir: {self._temp_dir} for processing') + + def get_results_img_path(self): + return path.join(self._temp_dir, 'results.dat') + + def get_area_mask_img_path(self): + return path.join(self._temp_dir, 'area_mask.dat') + + def __del__(self): + shutil.rmtree(self._temp_dir) diff --git a/zipdeepness/deepness.py b/zipdeepness/deepness.py new file mode 100644 index 0000000000000000000000000000000000000000..9349af480eeae41c149bc1457ec2c5936ab24820 --- /dev/null +++ b/zipdeepness/deepness.py @@ -0,0 +1,317 @@ +"""Main plugin file - entry point for the plugin. + +Links the UI and the processing. + +Skeleton of this file was generate with the QGis plugin to create plugin skeleton - QGIS PluginBuilder +""" + +import logging +import traceback + +from qgis.core import Qgis, QgsApplication, QgsProject, QgsVectorLayer +from qgis.gui import QgisInterface +from qgis.PyQt.QtCore import QCoreApplication, Qt +from qgis.PyQt.QtGui import QIcon +from qgis.PyQt.QtWidgets import QAction, QMessageBox + +from deepness.common.defines import IS_DEBUG, PLUGIN_NAME +from deepness.common.lazy_package_loader import LazyPackageLoader +from deepness.common.processing_parameters.map_processing_parameters import MapProcessingParameters, ProcessedAreaType +from deepness.common.processing_parameters.training_data_export_parameters import TrainingDataExportParameters +from deepness.deepness_dockwidget import DeepnessDockWidget +from deepness.dialogs.resizable_message_box import ResizableMessageBox +from deepness.images.get_image_path import get_icon_path +from deepness.processing.map_processor.map_processing_result import (MapProcessingResult, MapProcessingResultCanceled, + MapProcessingResultFailed, + MapProcessingResultSuccess) +from deepness.processing.map_processor.map_processor_training_data_export import MapProcessorTrainingDataExport + +cv2 = LazyPackageLoader('cv2') + + +class Deepness: + """ QGIS Plugin Implementation - main class of the plugin. + Creates the UI classes and processing models and links them together. + """ + + def __init__(self, iface: QgisInterface): + """ + :param iface: An interface instance that will be passed to this class + which provides the hook by which you can manipulate the QGIS + application at run time. + :type iface: QgsInterface + """ + self.iface = iface + + # Declare instance attributes + self.actions = [] + self.menu = self.tr(u'&Deepness') + + self.toolbar = self.iface.addToolBar(u'Deepness') + self.toolbar.setObjectName(u'Deepness') + + self.pluginIsActive = False + self.dockwidget = None + self._map_processor = None + + # noinspection PyMethodMayBeStatic + def tr(self, message): + """Get the translation for a string using Qt translation API. + + We implement this ourselves since we do not inherit QObject. + + :param message: String for translation. + :type message: str, QString + + :returns: Translated version of message. + :rtype: QString + """ + # noinspection PyTypeChecker,PyArgumentList,PyCallByClass + return QCoreApplication.translate('Deepness', message) + + def add_action( + self, + icon_path, + text, + callback, + enabled_flag=True, + add_to_menu=True, + add_to_toolbar=True, + status_tip=None, + whats_this=None, + parent=None): + """Add a toolbar icon to the toolbar. + + :param icon_path: Path to the icon for this action. Can be a resource + path (e.g. ':/plugins/foo/bar.png') or a normal file system path. + :type icon_path: str + + :param text: Text that should be shown in menu items for this action. + :type text: str + + :param callback: Function to be called when the action is triggered. + :type callback: function + + :param enabled_flag: A flag indicating if the action should be enabled + by default. Defaults to True. + :type enabled_flag: bool + + :param add_to_menu: Flag indicating whether the action should also + be added to the menu. Defaults to True. + :type add_to_menu: bool + + :param add_to_toolbar: Flag indicating whether the action should also + be added to the toolbar. Defaults to True. + :type add_to_toolbar: bool + + :param status_tip: Optional text to show in a popup when mouse pointer + hovers over the action. + :type status_tip: str + + :param parent: Parent widget for the new action. Defaults None. + :type parent: QWidget + + :param whats_this: Optional text to show in the status bar when the + mouse pointer hovers over the action. + + :returns: The action that was created. Note that the action is also + added to self.actions list. + :rtype: QAction + """ + + icon = QIcon(icon_path) + action = QAction(icon, text, parent) + action.triggered.connect(callback) + action.setEnabled(enabled_flag) + + if status_tip is not None: + action.setStatusTip(status_tip) + + if whats_this is not None: + action.setWhatsThis(whats_this) + + if add_to_toolbar: + self.toolbar.addAction(action) + + if add_to_menu: + self.iface.addPluginToMenu( + self.menu, + action) + + self.actions.append(action) + + return action + + def initGui(self): + """Create the menu entries and toolbar icons inside the QGIS GUI.""" + + icon_path = get_icon_path() + self.add_action( + icon_path, + text=self.tr(u'Deepness'), + callback=self.run, + parent=self.iface.mainWindow()) + + if IS_DEBUG: + self.run() + + def onClosePlugin(self): + """Cleanup necessary items here when plugin dockwidget is closed""" + + # disconnects + self.dockwidget.closingPlugin.disconnect(self.onClosePlugin) + + # remove this statement if dockwidget is to remain + # for reuse if plugin is reopened + # Commented next statement since it causes QGIS crashe + # when closing the docked window: + # self.dockwidget = None + + self.pluginIsActive = False + + def unload(self): + """Removes the plugin menu item and icon from QGIS GUI.""" + + for action in self.actions: + self.iface.removePluginMenu( + self.tr(u'&Deepness'), + action) + self.iface.removeToolBarIcon(action) + # remove the toolbar + del self.toolbar + + def _layers_changed(self, _): + pass + + def run(self): + """Run method that loads and starts the plugin""" + + if not self.pluginIsActive: + self.pluginIsActive = True + + # dockwidget may not exist if: + # first run of plugin + # removed on close (see self.onClosePlugin method) + if self.dockwidget is None: + # Create the dockwidget (after translation) and keep reference + self.dockwidget = DeepnessDockWidget(self.iface) + self._layers_changed(None) + QgsProject.instance().layersAdded.connect(self._layers_changed) + QgsProject.instance().layersRemoved.connect(self._layers_changed) + + # connect to provide cleanup on closing of dockwidget + self.dockwidget.closingPlugin.connect(self.onClosePlugin) + self.dockwidget.run_model_inference_signal.connect(self._run_model_inference) + self.dockwidget.run_training_data_export_signal.connect(self._run_training_data_export) + + self.iface.addDockWidget(Qt.RightDockWidgetArea, self.dockwidget) + self.dockwidget.show() + + def _are_map_processing_parameters_are_correct(self, params: MapProcessingParameters): + if self._map_processor and self._map_processor.is_busy(): + msg = "Error! Processing already in progress! Please wait or cancel previous task." + self.iface.messageBar().pushMessage(PLUGIN_NAME, msg, level=Qgis.Critical, duration=7) + return False + + rlayer = QgsProject.instance().mapLayers()[params.input_layer_id] + if rlayer is None: + msg = "Error! Please select the layer to process first!" + self.iface.messageBar().pushMessage(PLUGIN_NAME, msg, level=Qgis.Critical, duration=7) + return False + + if isinstance(rlayer, QgsVectorLayer): + msg = "Error! Please select a raster layer (vector layer selected)" + self.iface.messageBar().pushMessage(PLUGIN_NAME, msg, level=Qgis.Critical, duration=7) + return False + + return True + + def _display_processing_started_info(self): + msg = "Processing in progress... Cool! It's tea time!" + self.iface.messageBar().pushMessage(PLUGIN_NAME, msg, level=Qgis.Info, duration=2) + + def _run_training_data_export(self, training_data_export_parameters: TrainingDataExportParameters): + if not self._are_map_processing_parameters_are_correct(training_data_export_parameters): + return + + vlayer = None + + rlayer = QgsProject.instance().mapLayers()[training_data_export_parameters.input_layer_id] + if training_data_export_parameters.processed_area_type == ProcessedAreaType.FROM_POLYGONS: + vlayer = QgsProject.instance().mapLayers()[training_data_export_parameters.mask_layer_id] + + self._map_processor = MapProcessorTrainingDataExport( + rlayer=rlayer, + vlayer_mask=vlayer, # layer with masks + map_canvas=self.iface.mapCanvas(), + params=training_data_export_parameters) + self._map_processor.finished_signal.connect(self._map_processor_finished) + self._map_processor.show_img_signal.connect(self._show_img) + QgsApplication.taskManager().addTask(self._map_processor) + self._display_processing_started_info() + + def _run_model_inference(self, params: MapProcessingParameters): + from deepness.processing.models.model_types import ModelDefinition # import here to avoid pulling external dependencies to early + + if not self._are_map_processing_parameters_are_correct(params): + return + + vlayer = None + + rlayer = QgsProject.instance().mapLayers()[params.input_layer_id] + if params.processed_area_type == ProcessedAreaType.FROM_POLYGONS: + vlayer = QgsProject.instance().mapLayers()[params.mask_layer_id] + + model_definition = ModelDefinition.get_definition_for_params(params) + map_processor_class = model_definition.map_processor_class + + self._map_processor = map_processor_class( + rlayer=rlayer, + vlayer_mask=vlayer, + map_canvas=self.iface.mapCanvas(), + params=params) + self._map_processor.finished_signal.connect(self._map_processor_finished) + self._map_processor.show_img_signal.connect(self._show_img) + QgsApplication.taskManager().addTask(self._map_processor) + self._display_processing_started_info() + + @staticmethod + def _show_img(img_rgb, window_name: str): + """ Helper function to show an image while developing and debugging the plugin """ + # We are importing it here, because it is debug tool, + # and we don't want to have it in the main scope from the project startup + img_bgr = img_rgb[..., ::-1] + cv2.namedWindow(window_name, cv2.WINDOW_NORMAL) + cv2.resizeWindow(window_name, 800, 800) + cv2.imshow(window_name, img_bgr) + cv2.waitKey(1) + + def _map_processor_finished(self, result: MapProcessingResult): + """ Slot for finished processing of the ortophoto """ + if isinstance(result, MapProcessingResultFailed): + msg = f'Error! Processing error: "{result.message}"!' + self.iface.messageBar().pushMessage(PLUGIN_NAME, msg, level=Qgis.Critical, duration=14) + if result.exception is not None: + logging.error(msg) + trace = '\n'.join(traceback.format_tb(result.exception.__traceback__)[-1:]) + msg = f'{msg}\n\n\n' \ + f'Details: ' \ + f'{str(result.exception.__class__.__name__)} - {result.exception}\n' \ + f'Last Traceback: \n' \ + f'{trace}' + QMessageBox.critical(self.dockwidget, "Unhandled exception", msg) + elif isinstance(result, MapProcessingResultCanceled): + msg = f'Info! Processing canceled by user!' + self.iface.messageBar().pushMessage(PLUGIN_NAME, msg, level=Qgis.Info, duration=7) + elif isinstance(result, MapProcessingResultSuccess): + msg = 'Processing finished!' + self.iface.messageBar().pushMessage(PLUGIN_NAME, msg, level=Qgis.Success, duration=3) + message_to_show = result.message + + msgBox = ResizableMessageBox(self.dockwidget) + msgBox.setWindowTitle("Processing Result") + msgBox.setText(message_to_show) + msgBox.setStyleSheet("QLabel{min-width:800 px; font-size: 24px;} QPushButton{ width:250px; font-size: 18px; }") + msgBox.exec() + + self._map_processor = None diff --git a/zipdeepness/deepness_dockwidget.py b/zipdeepness/deepness_dockwidget.py new file mode 100644 index 0000000000000000000000000000000000000000..24d1409e670d0bebb3b661e6d0b9572cbc6e3d25 --- /dev/null +++ b/zipdeepness/deepness_dockwidget.py @@ -0,0 +1,603 @@ +""" +This file contain the main widget of the plugin +""" + +import logging +import os +from typing import Optional + +from qgis.core import Qgis, QgsMapLayerProxyModel, QgsProject +from qgis.PyQt import QtWidgets, uic +from qgis.PyQt.QtCore import pyqtSignal +from qgis.PyQt.QtWidgets import QComboBox, QFileDialog, QMessageBox + +from deepness.common.config_entry_key import ConfigEntryKey +from deepness.common.defines import IS_DEBUG, PLUGIN_NAME +from deepness.common.errors import OperationFailedException +from deepness.common.lazy_package_loader import LazyPackageLoader +from deepness.common.processing_overlap import ProcessingOverlap, ProcessingOverlapOptions +from deepness.common.processing_parameters.detection_parameters import DetectionParameters, DetectorType +from deepness.common.processing_parameters.map_processing_parameters import MapProcessingParameters, ProcessedAreaType +from deepness.common.processing_parameters.recognition_parameters import RecognitionParameters +from deepness.common.processing_parameters.regression_parameters import RegressionParameters +from deepness.common.processing_parameters.segmentation_parameters import SegmentationParameters +from deepness.common.processing_parameters.superresolution_parameters import SuperresolutionParameters +from deepness.common.processing_parameters.training_data_export_parameters import TrainingDataExportParameters +from deepness.processing.models.model_base import ModelBase +from deepness.widgets.input_channels_mapping.input_channels_mapping_widget import InputChannelsMappingWidget +from deepness.widgets.training_data_export_widget.training_data_export_widget import TrainingDataExportWidget + +FORM_CLASS, _ = uic.loadUiType(os.path.join( + os.path.dirname(__file__), 'deepness_dockwidget.ui')) + + +class DeepnessDockWidget(QtWidgets.QDockWidget, FORM_CLASS): + """ + Main widget of the plugin. + 'Dock' means it is a 'dcoked' widget, embedded in the QGis application window. + + The UI design is defined in the `deepness_dockwidget.ui` fiel - recommended to be open in QtDesigner. + Note: Default values for ui edits are based on 'ConfigEntryKey' default value, not taken from the UI form. + """ + + closingPlugin = pyqtSignal() + run_model_inference_signal = pyqtSignal(MapProcessingParameters) # run Segmentation or Detection + run_training_data_export_signal = pyqtSignal(TrainingDataExportParameters) + + def __init__(self, iface, parent=None): + super(DeepnessDockWidget, self).__init__(parent) + self.iface = iface + self._model = None # type: Optional[ModelBase] + self.setupUi(self) + + self._input_channels_mapping_widget = InputChannelsMappingWidget(self) # mapping of model and input ortophoto channels + self._training_data_export_widget = TrainingDataExportWidget(self) # widget with UI for data export tool + + self._create_connections() + self._setup_misc_ui() + self._load_ui_from_config() + + def _show_debug_warning(self): + """ Show label with warning if we are running debug mode """ + self.label_debugModeWarning.setVisible(IS_DEBUG) + + def _load_ui_from_config(self): + """ Load the UI values from the project configuration + """ + layers = QgsProject.instance().mapLayers() + + try: + input_layer_id = ConfigEntryKey.INPUT_LAYER_ID.get() + if input_layer_id and input_layer_id in layers: + self.mMapLayerComboBox_inputLayer.setLayer(layers[input_layer_id]) + + processed_area_type_txt = ConfigEntryKey.PROCESSED_AREA_TYPE.get() + self.comboBox_processedAreaSelection.setCurrentText(processed_area_type_txt) + + model_type_txt = ConfigEntryKey.MODEL_TYPE.get() + self.comboBox_modelType.setCurrentText(model_type_txt) + + self._input_channels_mapping_widget.load_ui_from_config() + self._training_data_export_widget.load_ui_from_config() + + # NOTE: load the model after setting the model_type above + model_file_path = ConfigEntryKey.MODEL_FILE_PATH.get() + if model_file_path: + self.lineEdit_modelPath.setText(model_file_path) + self._load_model_and_display_info(abort_if_no_file_path=True) # to prepare other ui components + + # needs to be loaded after the model is set up + self.doubleSpinBox_resolution_cm_px.setValue(ConfigEntryKey.PREPROCESSING_RESOLUTION.get()) + self.spinBox_batchSize.setValue(ConfigEntryKey.MODEL_BATCH_SIZE.get()) + self.checkBox_local_cache.setChecked(ConfigEntryKey.PROCESS_LOCAL_CACHE.get()) + self.spinBox_processingTileOverlapPercentage.setValue(ConfigEntryKey.PREPROCESSING_TILES_OVERLAP.get()) + + self.doubleSpinBox_probabilityThreshold.setValue( + ConfigEntryKey.SEGMENTATION_PROBABILITY_THRESHOLD_VALUE.get()) + self.checkBox_pixelClassEnableThreshold.setChecked( + ConfigEntryKey.SEGMENTATION_PROBABILITY_THRESHOLD_ENABLED.get()) + self._set_probability_threshold_enabled() + self.spinBox_dilateErodeSize.setValue( + ConfigEntryKey.SEGMENTATION_REMOVE_SMALL_SEGMENT_SIZE.get()) + self.checkBox_removeSmallAreas.setChecked( + ConfigEntryKey.SEGMENTATION_REMOVE_SMALL_SEGMENT_ENABLED.get()) + self._set_remove_small_segment_enabled() + + self.doubleSpinBox_regressionScaling.setValue(ConfigEntryKey.REGRESSION_OUTPUT_SCALING.get()) + + self.doubleSpinBox_confidence.setValue(ConfigEntryKey.DETECTION_CONFIDENCE.get()) + self.doubleSpinBox_iouScore.setValue(ConfigEntryKey.DETECTION_IOU.get()) + self.comboBox_detectorType.setCurrentText(ConfigEntryKey.DETECTOR_TYPE.get()) + except Exception: + logging.exception("Failed to load the ui state from config!") + + def _save_ui_to_config(self): + """ Save value from the UI forms to the project config + """ + ConfigEntryKey.MODEL_FILE_PATH.set(self.lineEdit_modelPath.text()) + ConfigEntryKey.INPUT_LAYER_ID.set(self._get_input_layer_id()) + ConfigEntryKey.MODEL_TYPE.set(self.comboBox_modelType.currentText()) + ConfigEntryKey.PROCESSED_AREA_TYPE.set(self.comboBox_processedAreaSelection.currentText()) + + ConfigEntryKey.PREPROCESSING_RESOLUTION.set(self.doubleSpinBox_resolution_cm_px.value()) + ConfigEntryKey.MODEL_BATCH_SIZE.set(self.spinBox_batchSize.value()) + ConfigEntryKey.PROCESS_LOCAL_CACHE.set(self.checkBox_local_cache.isChecked()) + ConfigEntryKey.PREPROCESSING_TILES_OVERLAP.set(self.spinBox_processingTileOverlapPercentage.value()) + + ConfigEntryKey.SEGMENTATION_PROBABILITY_THRESHOLD_ENABLED.set( + self.checkBox_pixelClassEnableThreshold.isChecked()) + ConfigEntryKey.SEGMENTATION_PROBABILITY_THRESHOLD_VALUE.set(self.doubleSpinBox_probabilityThreshold.value()) + ConfigEntryKey.SEGMENTATION_REMOVE_SMALL_SEGMENT_ENABLED.set( + self.checkBox_removeSmallAreas.isChecked()) + ConfigEntryKey.SEGMENTATION_REMOVE_SMALL_SEGMENT_SIZE.set(self.spinBox_dilateErodeSize.value()) + + ConfigEntryKey.REGRESSION_OUTPUT_SCALING.set(self.doubleSpinBox_regressionScaling.value()) + + ConfigEntryKey.DETECTION_CONFIDENCE.set(self.doubleSpinBox_confidence.value()) + ConfigEntryKey.DETECTION_IOU.set(self.doubleSpinBox_iouScore.value()) + ConfigEntryKey.DETECTOR_TYPE.set(self.comboBox_detectorType.currentText()) + + self._input_channels_mapping_widget.save_ui_to_config() + self._training_data_export_widget.save_ui_to_config() + + def _rlayer_updated(self): + self._input_channels_mapping_widget.set_rlayer(self._get_input_layer()) + + def _setup_misc_ui(self): + """ Setup some misceleounous ui forms + """ + from deepness.processing.models.model_types import \ + ModelDefinition # import here to avoid pulling external dependencies to early + + self._show_debug_warning() + combobox = self.comboBox_processedAreaSelection + for name in ProcessedAreaType.get_all_names(): + combobox.addItem(name) + + self.verticalLayout_inputChannelsMapping.addWidget(self._input_channels_mapping_widget) + self.verticalLayout_trainingDataExport.addWidget(self._training_data_export_widget) + + self.mMapLayerComboBox_inputLayer.setFilters(QgsMapLayerProxyModel.RasterLayer) + self.mMapLayerComboBox_areaMaskLayer.setFilters(QgsMapLayerProxyModel.VectorLayer) + + self.mGroupBox_8.setCollapsed(True) # collapse the group by default + self._set_processed_area_mask_options() + self._set_processing_overlap_enabled() + + for model_definition in ModelDefinition.get_model_definitions(): + self.comboBox_modelType.addItem(model_definition.model_type.value) + + for detector_type in DetectorType.get_all_display_values(): + self.comboBox_detectorType.addItem(detector_type) + self._detector_type_changed() + + self._rlayer_updated() # to force refresh the dependant ui elements + + def _set_processed_area_mask_options(self): + show_mask_combobox = (self.get_selected_processed_area_type() == ProcessedAreaType.FROM_POLYGONS) + self.mMapLayerComboBox_areaMaskLayer.setVisible(show_mask_combobox) + self.label_areaMaskLayer.setVisible(show_mask_combobox) + + def get_selected_processed_area_type(self) -> ProcessedAreaType: + combobox = self.comboBox_processedAreaSelection # type: QComboBox + txt = combobox.currentText() + return ProcessedAreaType(txt) + + def _create_connections(self): + self.pushButton_runInference.clicked.connect(self._run_inference) + self.pushButton_runTrainingDataExport.clicked.connect(self._run_training_data_export) + self.pushButton_browseQueryImagePath.clicked.connect(self._browse_query_image_path) + self.pushButton_browseModelPath.clicked.connect(self._browse_model_path) + self.comboBox_processedAreaSelection.currentIndexChanged.connect(self._set_processed_area_mask_options) + self.comboBox_modelType.currentIndexChanged.connect(self._model_type_changed) + self.comboBox_detectorType.currentIndexChanged.connect(self._detector_type_changed) + self.pushButton_reloadModel.clicked.connect(self._load_model_and_display_info) + self.pushButton_loadDefaultModelParameters.clicked.connect(self._load_default_model_parameters) + self.mMapLayerComboBox_inputLayer.layerChanged.connect(self._rlayer_updated) + self.checkBox_pixelClassEnableThreshold.stateChanged.connect(self._set_probability_threshold_enabled) + self.checkBox_removeSmallAreas.stateChanged.connect(self._set_remove_small_segment_enabled) + self.radioButton_processingTileOverlapPercentage.toggled.connect(self._set_processing_overlap_enabled) + self.radioButton_processingTileOverlapPixels.toggled.connect(self._set_processing_overlap_enabled) + + def _model_type_changed(self): + from deepness.processing.models.model_types import \ + ModelType # import here to avoid pulling external dependencies to early + + model_type = ModelType(self.comboBox_modelType.currentText()) + + segmentation_enabled = False + detection_enabled = False + regression_enabled = False + superresolution_enabled = False + recognition_enabled = False + + if model_type == ModelType.SEGMENTATION: + segmentation_enabled = True + elif model_type == ModelType.DETECTION: + detection_enabled = True + elif model_type == ModelType.REGRESSION: + regression_enabled = True + elif model_type == ModelType.SUPERRESOLUTION: + superresolution_enabled = True + elif model_type == ModelType.RECOGNITION: + recognition_enabled = True + else: + raise Exception(f"Unsupported model type ({model_type})!") + + self.mGroupBox_segmentationParameters.setVisible(segmentation_enabled) + self.mGroupBox_detectionParameters.setVisible(detection_enabled) + self.mGroupBox_regressionParameters.setVisible(regression_enabled) + self.mGroupBox_superresolutionParameters.setVisible(superresolution_enabled) + self.mGroupBox_recognitionParameters.setVisible(recognition_enabled) + + def _detector_type_changed(self): + detector_type = DetectorType(self.comboBox_detectorType.currentText()) + self.label_detectorTypeDescription.setText(detector_type.get_formatted_description()) + + def _set_processing_overlap_enabled(self): + overlap_percentage_enabled = self.radioButton_processingTileOverlapPercentage.isChecked() + self.spinBox_processingTileOverlapPercentage.setEnabled(overlap_percentage_enabled) + + overlap_pixels_enabled = self.radioButton_processingTileOverlapPixels.isChecked() + self.spinBox_processingTileOverlapPixels.setEnabled(overlap_pixels_enabled) + + def _set_probability_threshold_enabled(self): + self.doubleSpinBox_probabilityThreshold.setEnabled(self.checkBox_pixelClassEnableThreshold.isChecked()) + + def _set_remove_small_segment_enabled(self): + self.spinBox_dilateErodeSize.setEnabled(self.checkBox_removeSmallAreas.isChecked()) + + def _browse_model_path(self): + file_path, _ = QFileDialog.getOpenFileName( + self, + 'Select Model ONNX file...', + os.path.expanduser('~'), + 'All files (*.*);; ONNX files (*.onnx)') + if file_path: + self.lineEdit_modelPath.setText(file_path) + self._load_model_and_display_info() + + def _browse_query_image_path(self): + file_path, _ = QFileDialog.getOpenFileName( + self, + "Select image file...", + os.path.expanduser("~"), + "All files (*.*)", + ) + if file_path: + self.lineEdit_recognitionPath.setText(file_path) + + def _load_default_model_parameters(self): + """ + Load the default parameters from model metadata + """ + value = self._model.get_metadata_resolution() + if value is not None: + self.doubleSpinBox_resolution_cm_px.setValue(value) + + value = self._model.get_model_batch_size() + if value is not None: + self.spinBox_batchSize.setValue(value) + self.spinBox_batchSize.setEnabled(False) + else: + self.spinBox_batchSize.setEnabled(True) + + value = self._model.get_metadata_tile_size() + if value is not None: + self.spinBox_tileSize_px.setValue(value) + + value = self._model.get_metadata_tiles_overlap() + if value is not None: + self.spinBox_processingTileOverlapPercentage.setValue(value) + + value = self._model.get_metadata_model_type() + if value is not None: + print(f'{value =}') + self.comboBox_modelType.setCurrentText(value) + + value = self._model.get_detector_type() + if value is not None: + self.comboBox_detectorType.setCurrentText(value) + + value = self._model.get_metadata_segmentation_threshold() + if value is not None: + self.checkBox_pixelClassEnableThreshold.setChecked(bool(value != 0)) + self.doubleSpinBox_probabilityThreshold.setValue(value) + + value = self._model.get_metadata_segmentation_small_segment() + if value is not None: + self.checkBox_removeSmallAreas.setChecked(bool(value != 0)) + self.spinBox_dilateErodeSize.setValue(value) + + value = self._model.get_metadata_regression_output_scaling() + if value is not None: + self.doubleSpinBox_regressionScaling.setValue(value) + + value = self._model.get_metadata_detection_confidence() + if value is not None: + self.doubleSpinBox_confidence.setValue(value) + + value = self._model.get_metadata_detection_iou_threshold() + if value is not None: + self.doubleSpinBox_iouScore.setValue(value) + + def _load_model_with_type_from_metadata(self, model_class_from_ui, file_path): + """ + If model has model_type in metadata - use this type to create proper model class. + Otherwise model_class_from_ui will be used + """ + from deepness.processing.models.model_types import ( # import here to avoid pulling external dependencies to early + ModelDefinition, ModelType) + + model_class = model_class_from_ui + + model_type_str_from_metadata = ModelBase.get_model_type_from_metadata(file_path) + if model_type_str_from_metadata is not None: + model_type = ModelType(model_type_str_from_metadata) + model_class = ModelDefinition.get_definition_for_type(model_type).model_class + self.comboBox_modelType.setCurrentText(model_type.value) + + print(f'{model_type_str_from_metadata = }, {model_class = }') + + model = model_class(file_path) + return model + + def _load_model_and_display_info(self, abort_if_no_file_path: bool = False): + """ + Tries to load the model and display its message. + """ + import deepness.processing.models.detector as detector_module # import here to avoid pulling external dependencies to early + from deepness.processing.models.model_types import \ + ModelType # import here to avoid pulling external dependencies to early + + file_path = self.lineEdit_modelPath.text() + + if not file_path and abort_if_no_file_path: + return + + txt = '' + + try: + model_definition = self.get_selected_model_class_definition() + model_class = model_definition.model_class + self._model = self._load_model_with_type_from_metadata( + model_class_from_ui=model_class, + file_path=file_path) + self._model.check_loaded_model_outputs() + input_0_shape = self._model.get_input_shape() + txt += 'Legend: [BATCH_SIZE, CHANNELS, HEIGHT, WIDTH]\n' + txt += 'Inputs:\n' + txt += f'\t- Input: {input_0_shape}\n' + input_size_px = input_0_shape[-1] + batch_size = self._model.get_model_batch_size() + + txt += 'Outputs:\n' + + for i, output_shape in enumerate(self._model.get_output_shapes()): + txt += f'\t- Output {i}: {output_shape}\n' + + # TODO idk how variable input will be handled + self.spinBox_tileSize_px.setValue(input_size_px) + self.spinBox_tileSize_px.setEnabled(False) + + if batch_size is not None: + self.spinBox_batchSize.setValue(batch_size) + self.spinBox_batchSize.setEnabled(False) + else: + self.spinBox_batchSize.setEnabled(True) + + self._input_channels_mapping_widget.set_model(self._model) + + # super resolution + if model_class == ModelType.SUPERRESOLUTION: + output_0_shape = self._model.get_output_shape() + scale_factor = output_0_shape[-1] / input_size_px + self.doubleSpinBox_superresolutionScaleFactor.setValue(int(scale_factor)) + # Disable output format options for super-resolution models + except Exception as e: + if IS_DEBUG: + raise e + txt = "Error! Failed to load the model!\n" \ + "Model may be not usable." + logging.exception(txt) + self.spinBox_tileSize_px.setEnabled(True) + self.spinBox_batchSize.setEnabled(True) + length_limit = 300 + exception_msg = (str(e)[:length_limit] + '..') if len(str(e)) > length_limit else str(e) + msg = txt + f'\n\nException: {exception_msg}' + QMessageBox.critical(self, "Error!", msg) + + self.label_modelInfo.setText(txt) + + if isinstance(self._model, detector_module.Detector): + detector_type = DetectorType(self.comboBox_detectorType.currentText()) + self._model.set_model_type_param(detector_type) + + def get_mask_layer_id(self): + if not self.get_selected_processed_area_type() == ProcessedAreaType.FROM_POLYGONS: + return None + + mask_layer_id = self.mMapLayerComboBox_areaMaskLayer.currentLayer().id() + return mask_layer_id + + def _get_input_layer(self): + return self.mMapLayerComboBox_inputLayer.currentLayer() + + def _get_input_layer_id(self): + layer = self._get_input_layer() + if layer: + return layer.id() + else: + return '' + + def _get_overlap_parameter(self): + if self.radioButton_processingTileOverlapPercentage.isChecked(): + return ProcessingOverlap( + selected_option=ProcessingOverlapOptions.OVERLAP_IN_PERCENT, + percentage=self.spinBox_processingTileOverlapPercentage.value(), + ) + elif self.radioButton_processingTileOverlapPixels.isChecked(): + return ProcessingOverlap( + selected_option=ProcessingOverlapOptions.OVERLAP_IN_PIXELS, + overlap_px=self.spinBox_processingTileOverlapPixels.value(), + ) + else: + raise Exception('Something goes wrong. No overlap parameter selected!') + + def _get_pixel_classification_threshold(self): + if not self.checkBox_pixelClassEnableThreshold.isChecked(): + return 0 + return self.doubleSpinBox_probabilityThreshold.value() + + def get_selected_model_class_definition(self): # -> ModelDefinition: # we cannot import it here yet + """ + Get the currently selected model class (in UI) + """ + from deepness.processing.models.model_types import ( # import here to avoid pulling external dependencies to early + ModelDefinition, ModelType) + + model_type_txt = self.comboBox_modelType.currentText() + model_type = ModelType(model_type_txt) + model_definition = ModelDefinition.get_definition_for_type(model_type) + return model_definition + + def get_inference_parameters(self) -> MapProcessingParameters: + """ Get the parameters for the model interface. + The returned type is derived from `MapProcessingParameters` class, depending on the selected model type. + """ + from deepness.processing.models.model_types import \ + ModelType # import here to avoid pulling external dependencies to early + + map_processing_parameters = self._get_map_processing_parameters() + + if self._model is None: + raise OperationFailedException("Please select and load a model first!") + + model_type = self.get_selected_model_class_definition().model_type + if model_type == ModelType.SEGMENTATION: + params = self.get_segmentation_parameters(map_processing_parameters) + elif model_type == ModelType.REGRESSION: + params = self.get_regression_parameters(map_processing_parameters) + elif model_type == ModelType.SUPERRESOLUTION: + params = self.get_superresolution_parameters(map_processing_parameters) + elif model_type == ModelType.RECOGNITION: + params = self.get_recognition_parameters(map_processing_parameters) + elif model_type == ModelType.DETECTION: + params = self.get_detection_parameters(map_processing_parameters) + + else: + raise Exception(f"Unknown model type '{model_type}'!") + + return params + + def get_segmentation_parameters(self, map_processing_parameters: MapProcessingParameters) -> SegmentationParameters: + postprocessing_dilate_erode_size = self.spinBox_dilateErodeSize.value() \ + if self.checkBox_removeSmallAreas.isChecked() else 0 + + params = SegmentationParameters( + **map_processing_parameters.__dict__, + postprocessing_dilate_erode_size=postprocessing_dilate_erode_size, + pixel_classification__probability_threshold=self._get_pixel_classification_threshold(), + model=self._model, + ) + return params + + def get_regression_parameters(self, map_processing_parameters: MapProcessingParameters) -> RegressionParameters: + params = RegressionParameters( + **map_processing_parameters.__dict__, + output_scaling=self.doubleSpinBox_regressionScaling.value(), + model=self._model, + ) + return params + + def get_superresolution_parameters(self, map_processing_parameters: MapProcessingParameters) -> SuperresolutionParameters: + params = SuperresolutionParameters( + **map_processing_parameters.__dict__, + model=self._model, + scale_factor=self.doubleSpinBox_superresolutionScaleFactor.value(), + output_scaling=self.doubleSpinBox_superresolutionScaling.value(), + ) + return params + + def get_recognition_parameters(self, map_processing_parameters: MapProcessingParameters) -> RecognitionParameters: + params = RecognitionParameters( + **map_processing_parameters.__dict__, + model=self._model, + query_image_path=self.lineEdit_recognitionPath.text(), + ) + return params + + def get_detection_parameters(self, map_processing_parameters: MapProcessingParameters) -> DetectionParameters: + + params = DetectionParameters( + **map_processing_parameters.__dict__, + confidence=self.doubleSpinBox_confidence.value(), + iou_threshold=self.doubleSpinBox_iouScore.value(), + model=self._model, + detector_type=DetectorType(self.comboBox_detectorType.currentText()), + ) + + return params + + def _get_map_processing_parameters(self) -> MapProcessingParameters: + """ + Get common parameters for inference and exporting + """ + processed_area_type = self.get_selected_processed_area_type() + params = MapProcessingParameters( + resolution_cm_per_px=self.doubleSpinBox_resolution_cm_px.value(), + tile_size_px=self.spinBox_tileSize_px.value(), + batch_size=self.spinBox_batchSize.value(), + local_cache=self.checkBox_local_cache.isChecked(), + processed_area_type=processed_area_type, + mask_layer_id=self.get_mask_layer_id(), + input_layer_id=self._get_input_layer_id(), + processing_overlap=self._get_overlap_parameter(), + input_channels_mapping=self._input_channels_mapping_widget.get_channels_mapping(), + ) + return params + + def _run_inference(self): + # check_required_packages_and_install_if_necessary() + try: + params = self.get_inference_parameters() + + if not params.input_layer_id: + raise OperationFailedException("Please select an input layer first!") + except OperationFailedException as e: + msg = str(e) + self.iface.messageBar().pushMessage(PLUGIN_NAME, msg, level=Qgis.Warning, duration=7) + logging.exception(msg) + QMessageBox.critical(self, "Error!", msg) + return + + self._save_ui_to_config() + self.run_model_inference_signal.emit(params) + + def _run_training_data_export(self): + # check_required_packages_and_install_if_necessary() + try: + map_processing_parameters = self._get_map_processing_parameters() + training_data_export_parameters = self._training_data_export_widget.get_training_data_export_parameters( + map_processing_parameters) + + if not map_processing_parameters.input_layer_id: + raise OperationFailedException("Please select an input layer first!") + + # Overwrite common parameter - we don't want channels mapping as for the model, + # but just to take all channels + training_data_export_parameters.input_channels_mapping = \ + self._input_channels_mapping_widget.get_channels_mapping_for_training_data_export() + except OperationFailedException as e: + msg = str(e) + self.iface.messageBar().pushMessage(PLUGIN_NAME, msg, level=Qgis.Warning) + logging.exception(msg) + QMessageBox.critical(self, "Error!", msg) + return + + self._save_ui_to_config() + self.run_training_data_export_signal.emit(training_data_export_parameters) + + def closeEvent(self, event): + self.closingPlugin.emit() + event.accept() diff --git a/zipdeepness/deepness_dockwidget.ui b/zipdeepness/deepness_dockwidget.ui new file mode 100644 index 0000000000000000000000000000000000000000..78c6447444eff0437ada7243227bb0df556909a4 --- /dev/null +++ b/zipdeepness/deepness_dockwidget.ui @@ -0,0 +1,893 @@ + + + DeepnessDockWidgetBase + + + + 0 + 0 + 486 + 1368 + + + + Deepness + + + + + + + true + + + + + 0 + -268 + 452 + 1564 + + + + + 0 + + + 0 + + + 0 + + + + + + 9 + true + + + + color:rgb(198, 70, 0) + + + WARNING: Running plugin in DEBUG mode +(because env variable IS_DEBUG=true) + + + + + + + + 0 + 0 + + + + Input data: + + + + + + <html><head/><body><p><span style=" font-weight:600;">Layer which will be processed.</span></p><p>Most probably this is your ortophoto or map source (like satellite image from google earth).<br/>Needs to be a raster layer.</p></body></html> + + + + + + + <html><head/><body><p>Defines what part of the &quot;Input layer&quot; should be processed.</p><p><br/> - &quot;<span style=" font-style:italic;">Visible Part</span>&quot; allows to process the part currently visible on the map canvas.<br/> - &quot;<span style=" font-style:italic;">Entire Layer</span>&quot; allows to process the entire ortophoto file<br/> - &quot;<span style=" font-style:italic;">From Polygons</span>&quot; allows to select a polygon describing the area to be processed (e.g. if the processed field is a polygon, and we don't want to process outside of it)</p></body></html> + + + + + + + Input layer: + + + + + + + Processed area mask: + + + + + + + Area mask layer: + + + + + + + <html><head/><body><p>Defines the layer which is being used as a mask for the processing of &quot;Input layer&quot;. <br/>Only pixels within this mask layer will be processed.</p><p>Needs to be a vector layer.</p></body></html> + + + + + + + + + + + 0 + 0 + + + + ONNX Model + + + + + + true + + + Path to the model file + + + + + + + Model file path: + + + + + + + true + + + Browse... + + + + + + + Model type: + + + + + + + <html><head/><body><p>Type of the model (model class) which you want to use.<br/>You should obtain this information along with the model file.</p><p>Please refer to the plugin documentation for more details.</p></body></html> + + + + + + + Model info: + + + + + + + + 7 + + + + color: rgb(135, 135, 133); + + + Model not loaded! Please select its path and click "Load Model" button above first! + + + true + + + + + + + + + Reload the model given in the line edit above + + + Reload Model + + + + + + + <html><head/><body><p><span style=" font-weight:600;">Load default model parameters.</span></p><p>ONNX Models can have metadata, which can be parsed and used to set default value for fields in UI, w.g. for tile_size or confidence threshold</p></body></html> + + + Load default parameters + + + + + + + + + + + + + 0 + 0 + + + + Input channels mapping + + + + + + + + + + true + + + + NOTE: This configuration is depending on the input layer and model type. Please make sure to select the "Input layer" and load the model first! + + + true + + + + + + + + + + + 0 + 0 + + + + Processing parameters + + + + + + Tiles overlap: + + + + + + [px] + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + 0 + 0 + + + + + 80 + 0 + + + + <html><head/><body><p>Defines how much tiles should overlap on their neighbours during processing.</p><p>Especially required for model which introduce distortions on the edges of images, so that it can be removed in postprocessing.</p></body></html> + + + + + + 15 + + + + + + + + 0 + 0 + + + + + 80 + 0 + + + + 9999999 + + + + + + + [%] + + + true + + + + + + + + + + true + + + <html><head/><body><p>Size of the images passed to the model.</p><p>Usually needs to be the same as the one used during training.</p></body></html> + + + 99999 + + + 512 + + + + + + + <html><head/><body><p>Defines the processing resolution of the &quot;Input layer&quot;.</p><p><br/></p><p>Determines the resolution of images fed into the model, allowing to scale the input images.</p><p>Should be similar as the resolution used to train the model.</p></body></html> + + + 2 + + + 0.000000000000000 + + + 999999.000000000000000 + + + 3.000000000000000 + + + + + + + Tile size [px]: + + + + + + + Resolution [cm/px]: + + + + + + + + false + true + + + + NOTE: These options may be a fixed value for some models + + + true + + + + + + + Batch size: + + + + + + + <html><head/><body><p>The size of the data batch in the model.</p><p>The size depends on the computing resources, in particular the available RAM / GPU memory.</p></body></html> + + + 1 + + + 9999999 + + + + + + + <html><head/><body><p>If True, local memory caching is performed - this is helpful when large area maps are processed, but is probably slower than processing in RAM.</p><p><br/></p></body></html> + + + Process using local cache + + + + + + + + + + + 0 + 0 + + + + Segmentation parameters + + + + + + <html><head/><body><p>Minimum required probability for the class to be considered as belonging to this class.</p></body></html> + + + 2 + + + 1.000000000000000 + + + 0.050000000000000 + + + 0.500000000000000 + + + + + + + false + + + + + + Argmax (most probable class only) + + + true + + + + + + + Apply class probability threshold: + + + true + + + + + + + <html><head/><body><p>Postprocessing option, to remove small areas (small clusters of pixels) belonging to each class, smoothing the predictions.</p><p>The actual size (in meters) of the smoothing can be calculated as &quot;Resolution&quot; * &quot;value of this parameter&quot;.<br/>Works as application of dilate and erode operation (twice, in reverse order).<br/>Similar effect to median filter.</p></body></html> + + + 9 + + + + + + + + true + + + + NOTE: Applicable only if a segmentation model is used + + + + + + + Remove small segment + areas (dilate/erode size) [px]: + + + true + + + + + + + + + + + 0 + 0 + + + + Superresolution Parameters + + + + + + Upscaling Factor + + + + + + + 2 + + + + + + + 100000000000.000000000000000 + + + 255.000000000000000 + + + + + + + Output scaling + + + + + + + + + + + 0 + 0 + + + + Regression parameters + + + + + + <html><head/><body><p>Scaling factor for model output values.</p><p>Each pixel value will be multiplied by this factor.</p></body></html> + + + 3 + + + 9999.000000000000000 + + + 1.000000000000000 + + + + + + + Output scaling +(keep 1.00 if max output value is 1): + + + + + + + + + + + 0 + 0 + + + + Recognition parameters + + + + + + + true + + + + NOTE: Applicable only if a recognition model is used + + + + + + + Image to localize path: + + + + + + + + + + Browse + + + + + + + + + + Detection parameters + + + + + + Confidence: + + + + + + + Detector type: + + + + + + + + + + IoU threshold: + + + + + + + <html><head/><body><p>Minimal confidence of the potential detection, to consider it as a detection.</p></body></html> + + + 2 + + + 1.000000000000000 + + + 0.050000000000000 + + + + + + + + true + + + + NOTE: Applicable only if a detection model is used + + + + + + + + 9 + + + + Here goes a longer description of detector type... + + + + + + + <html><head/><body><p>Parameter used in Non Maximum Suppression in post processing.</p><p>Defines the threshold of overlap between to neighbouring detections, to consider them as the same object.</p></body></html> + + + 2 + + + 1.000000000000000 + + + 0.050000000000000 + + + + + + + + + + + 0 + 0 + + + + Training data export + + + false + + + + + + Note: This group allows to export the data for the training process, with similar data as during inference. + + + true + + + + + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + <html><head/><body><p>Run the export of the data</p></body></html> + + + Export training data + + + + + + + + + + 0 + + + 0 + + + 0 + + + 0 + + + + + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + + + + + + + <html><head/><body><p>Run the inference, for the selected above paraeters.</p></body></html> + + + Run + + + + + + + + + + + QgsCollapsibleGroupBox + QGroupBox +
qgscollapsiblegroupbox.h
+ 1 + + + QgsDoubleSpinBox + QDoubleSpinBox +
qgsdoublespinbox.h
+ + + QgsMapLayerComboBox + QComboBox +
qgsmaplayercombobox.h
+ + + QgsSpinBox + QSpinBox +
qgsspinbox.h
+ + + + + diff --git a/zipdeepness/dialogs/packages_installer/packages_installer_dialog.py b/zipdeepness/dialogs/packages_installer/packages_installer_dialog.py new file mode 100644 index 0000000000000000000000000000000000000000..1afdbc8023a73e772ed038e2668e559898d3c17c --- /dev/null +++ b/zipdeepness/dialogs/packages_installer/packages_installer_dialog.py @@ -0,0 +1,359 @@ +""" +This QGIS plugin requires some Python packages to be installed and available. +This tool allows to install them in a local directory, if they are not installed yet. +""" + +import importlib +import logging +import os +import subprocess +import sys +import traceback +import urllib +from dataclasses import dataclass +from pathlib import Path +from threading import Thread +from typing import List + +from qgis.PyQt import QtCore, uic +from qgis.PyQt.QtCore import pyqtSignal +from qgis.PyQt.QtGui import QCloseEvent +from qgis.PyQt.QtWidgets import QDialog, QMessageBox, QTextBrowser + +from deepness.common.defines import PLUGIN_NAME + +PYTHON_VERSION = sys.version_info +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +PLUGIN_ROOT_DIR = os.path.realpath(os.path.abspath(os.path.join(SCRIPT_DIR, '..', '..'))) +PACKAGES_INSTALL_DIR = os.path.join(PLUGIN_ROOT_DIR, f'python{PYTHON_VERSION.major}.{PYTHON_VERSION.minor}') + + +FORM_CLASS, _ = uic.loadUiType(os.path.join( + os.path.dirname(__file__), 'packages_installer_dialog.ui')) + +_ERROR_COLOR = '#ff0000' + + +@dataclass +class PackageToInstall: + name: str + version: str + import_name: str # name while importing package + + def __str__(self): + return f'{self.name}{self.version}' + + +REQUIREMENTS_PATH = os.path.join(PLUGIN_ROOT_DIR, 'python_requirements/requirements.txt') + +with open(REQUIREMENTS_PATH, 'r') as f: + raw_txt = f.read() + +libraries_versions = {} + +for line in raw_txt.split('\n'): + if line.startswith('#') or not line.strip(): + continue + + line = line.split(';')[0] + + if '==' in line: + lib, version = line.split('==') + libraries_versions[lib] = '==' + version + elif '>=' in line: + lib, version = line.split('>=') + libraries_versions[lib] = '>=' + version + elif '<=' in line: + lib, version = line.split('<=') + libraries_versions[lib] = '<=' + version + else: + libraries_versions[line] = '' + + +packages_to_install = [ + PackageToInstall(name='opencv-python-headless', version=libraries_versions['opencv-python-headless'], import_name='cv2'), +] + +if sys.platform == "linux" or sys.platform == "linux2": + packages_to_install += [ + PackageToInstall(name='onnxruntime-gpu', version=libraries_versions['onnxruntime-gpu'], import_name='onnxruntime'), + ] + PYTHON_EXECUTABLE_PATH = sys.executable +elif sys.platform == "darwin": # MacOS + packages_to_install += [ + PackageToInstall(name='onnxruntime', version=libraries_versions['onnxruntime-gpu'], import_name='onnxruntime'), + ] + PYTHON_EXECUTABLE_PATH = str(Path(sys.prefix) / 'bin' / 'python3') # sys.executable yields QGIS in macOS +elif sys.platform == "win32": + packages_to_install += [ + PackageToInstall(name='onnxruntime', version=libraries_versions['onnxruntime-gpu'], import_name='onnxruntime'), + ] + PYTHON_EXECUTABLE_PATH = 'python' # sys.executable yields QGis.exe in Windows +else: + raise Exception("Unsupported operating system!") + + +class PackagesInstallerDialog(QDialog, FORM_CLASS): + """ + Dialog witch controls the installation process of packages. + UI design defined in the `packages_installer_dialog.ui` file. + """ + + signal_log_line = pyqtSignal(str) # we need to use signal because we cannot edit GUI from another thread + + INSTALLATION_IN_PROGRESS = False # to make sure we will not start the installation twice + + def __init__(self, iface, parent=None): + super(PackagesInstallerDialog, self).__init__(parent) + self.setupUi(self) + self.iface = iface + self.tb = self.textBrowser_log # type: QTextBrowser + self._create_connections() + self._setup_message() + self.aborted = False + self.thread = None + + def move_to_top(self): + """ Move the window to the top. + Although if installed from plugin manager, the plugin manager will move itself to the top anyway. + """ + self.setWindowState((self.windowState() & ~QtCore.Qt.WindowMinimized) | QtCore.Qt.WindowActive) + + if sys.platform == "linux" or sys.platform == "linux2": + pass + elif sys.platform == "darwin": # MacOS + self.raise_() # FIXME: this does not really work, the window is still behind the plugin manager + elif sys.platform == "win32": + self.activateWindow() + else: + raise Exception("Unsupported operating system!") + + def _create_connections(self): + self.pushButton_close.clicked.connect(self.close) + self.pushButton_install_packages.clicked.connect(self._run_packages_installation) + self.signal_log_line.connect(self._log_line) + + def _log_line(self, txt): + txt = txt \ + .replace(' ', '  ') \ + .replace('\n', '
') + self.tb.append(txt) + + def log(self, txt): + self.signal_log_line.emit(txt) + + def _setup_message(self) -> None: + + self.log(f'

' + f'Plugin {PLUGIN_NAME} - Packages installer

\n' + f'\n' + f'This plugin requires the following Python packages to be installed:') + + for package in packages_to_install: + self.log(f'\t- {package.name}{package.version}') + + self.log('\n\n' + f'If this packages are not installed in the global environment ' + f'(or environment in which QGIS is started) ' + f'you can install these packages in the local directory (which is included to the Python path).\n\n' + f'This Dialog does it for you! (Though you can still install these packages manually instead).\n' + f'Please click "Install packages" button below to install them automatically, ' + f'or "Test and Close" if you installed them manually...\n') + + def _run_packages_installation(self): + if self.INSTALLATION_IN_PROGRESS: + self.log(f'Error! Installation already in progress, cannot start again!') + return + self.aborted = False + self.INSTALLATION_IN_PROGRESS = True + self.thread = Thread(target=self._install_packages) + self.thread.start() + + def _install_packages(self) -> None: + self.log('\n\n') + self.log('=' * 60) + self.log(f'

Attempting to install required packages...

') + os.makedirs(PACKAGES_INSTALL_DIR, exist_ok=True) + + self._install_pip_if_necessary() + + self.log(f'

Attempting to install required packages...

\n') + try: + self._pip_install_packages(packages_to_install) + except Exception as e: + msg = (f'\n ' + f'Packages installation failed with exception: {e}!\n' + f'Please try to install the packages again. ' + f'\nCheck if there is no error related to system packages, ' + f'which may be required to be installed by your system package manager, e.g. "apt". ' + f'Copy errors from the stack above and google for possible solutions. ' + f'Please report these as an issue on the plugin repository tracker!') + self.log(msg) + + # finally, validate the installation, if there was no error so far... + self.log('\n\n Installation of required packages finished. Validating installation...') + self._check_packages_installation_and_log() + self.INSTALLATION_IN_PROGRESS = False + + def reject(self) -> None: + self.close() + + def closeEvent(self, event: QCloseEvent): + self.aborted = True + if self._check_packages_installation_and_log(): + event.accept() + return + + res = QMessageBox.question(self.iface.mainWindow(), + f'{PLUGIN_NAME} - skip installation?', + 'Are you sure you want to abort the installation of the required python packages? ' + 'The plugin may not function correctly without them!', + QMessageBox.No, QMessageBox.Yes) + log_msg = 'User requested to close the dialog, but the packages are not installed correctly!\n' + if res == QMessageBox.Yes: + log_msg += 'And the user confirmed to close the dialog, knowing the risk!' + event.accept() + else: + log_msg += 'The user reconsidered their decision, and will try to install the packages again!' + event.ignore() + log_msg += '\n' + self.log(log_msg) + + def _install_pip_if_necessary(self): + """ + Install pip if not present. + It happens e.g. in flatpak applications. + + TODO - investigate whether we can also install pip in local directory + """ + + self.log(f'

Making sure pip is installed...

') + if check_pip_installed(): + self.log(f'Pip is installed, skipping installation...\n') + return + + install_pip_command = [PYTHON_EXECUTABLE_PATH, '-m', 'ensurepip'] + self.log(f'Running command to install pip: \n $ {" ".join(install_pip_command)} ') + with subprocess.Popen(install_pip_command, + stdout=subprocess.PIPE, + universal_newlines=True, + stderr=subprocess.STDOUT, + env={'SETUPTOOLS_USE_DISTUTILS': 'stdlib'}) as process: + try: + self._do_process_output_logging(process) + except InterruptedError as e: + self.log(str(e)) + return False + + if process.returncode != 0: + msg = (f'' + f'pip installation failed! Consider installing it manually.' + f'') + self.log(msg) + self.log('\n') + + def _pip_install_packages(self, packages: List[PackageToInstall]) -> None: + cmd = [PYTHON_EXECUTABLE_PATH, '-m', 'pip', 'install', '-U', f'--target={PACKAGES_INSTALL_DIR}'] + cmd_string = ' '.join(cmd) + + for pck in packages: + cmd.append(f"{pck}") + cmd_string += f"{pck}" + + self.log(f'Running command: \n $ {cmd_string} ') + with subprocess.Popen(cmd, + stdout=subprocess.PIPE, + universal_newlines=True, + stderr=subprocess.STDOUT) as process: + self._do_process_output_logging(process) + + if process.returncode != 0: + raise RuntimeError('Installation with pip failed') + + msg = (f'\n' + f'Packages installed correctly!' + f'\n\n') + self.log(msg) + + def _do_process_output_logging(self, process: subprocess.Popen) -> None: + """ + :param process: instance of 'subprocess.Popen' + """ + for stdout_line in iter(process.stdout.readline, ""): + if stdout_line.isspace(): + continue + txt = f'{stdout_line.rstrip(os.linesep)}' + self.log(txt) + if self.aborted: + raise InterruptedError('Installation aborted by user') + + def _check_packages_installation_and_log(self) -> bool: + packages_ok = are_packages_importable() + self.pushButton_install_packages.setEnabled(not packages_ok) + + if packages_ok: + msg1 = f'All required packages are importable! You can close this window now!' + self.log(msg1) + return True + + try: + import_packages() + raise Exception("Unexpected successful import of packages?!? It failed a moment ago, we shouldn't be here!") + except Exception: + msg_base = 'Python packages required by the plugin could not be loaded due to the following error:' + logging.exception(msg_base) + tb = traceback.format_exc() + msg1 = (f'' + f'{msg_base} \n ' + f'{tb}\n\n' + f'Please try installing the packages again.' + f'') + self.log(msg1) + + return False + + +dialog = None + + +def import_package(package: PackageToInstall): + importlib.import_module(package.import_name) + + +def import_packages(): + for package in packages_to_install: + import_package(package) + + +def are_packages_importable() -> bool: + try: + import_packages() + except Exception: + logging.exception(f'Python packages required by the plugin could not be loaded due to the following error:') + return False + + return True + + +def check_pip_installed() -> bool: + try: + subprocess.check_output([PYTHON_EXECUTABLE_PATH, '-m', 'pip', '--version']) + return True + except subprocess.CalledProcessError: + return False + + +def check_required_packages_and_install_if_necessary(iface): + os.makedirs(PACKAGES_INSTALL_DIR, exist_ok=True) + if PACKAGES_INSTALL_DIR not in sys.path: + sys.path.append(PACKAGES_INSTALL_DIR) # TODO: check for a less intrusive way to do this + + if are_packages_importable(): + # if packages are importable we are fine, nothing more to do then + return + + global dialog + dialog = PackagesInstallerDialog(iface) + dialog.setWindowModality(QtCore.Qt.WindowModal) + dialog.show() + dialog.move_to_top() diff --git a/zipdeepness/dialogs/packages_installer/packages_installer_dialog.ui b/zipdeepness/dialogs/packages_installer/packages_installer_dialog.ui new file mode 100644 index 0000000000000000000000000000000000000000..38e6b301ad4ac2b6151bb5783a2d2a37ce1e4450 --- /dev/null +++ b/zipdeepness/dialogs/packages_installer/packages_installer_dialog.ui @@ -0,0 +1,65 @@ + + + PackagesInstallerDialog + + + + 0 + 0 + 693 + 494 + + + + Deepness - Packages Installer Dialog + + + + + + + + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + 75 + true + + + + Install packages + + + + + + + Test and Close + + + + + + + + + + diff --git a/zipdeepness/dialogs/resizable_message_box.py b/zipdeepness/dialogs/resizable_message_box.py new file mode 100644 index 0000000000000000000000000000000000000000..b2a67948bfbb3df5cba712f6f87d737cfb9a5807 --- /dev/null +++ b/zipdeepness/dialogs/resizable_message_box.py @@ -0,0 +1,20 @@ +from qgis.PyQt.QtWidgets import QMessageBox, QTextEdit + + +class ResizableMessageBox(QMessageBox): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.setSizeGripEnabled(True) + + def event(self, event): + if event.type() in (event.LayoutRequest, event.Resize): + if event.type() == event.Resize: + res = super().event(event) + else: + res = False + details = self.findChild(QTextEdit) + if details: + details.setMaximumSize(16777215, 16777215) + self.setMaximumSize(16777215, 16777215) + return res + return super().event(event) diff --git a/zipdeepness/images/get_image_path.py b/zipdeepness/images/get_image_path.py new file mode 100644 index 0000000000000000000000000000000000000000..03b5ebfd4ab7d4d50f46e4d7e30c3dc0c34cdff2 --- /dev/null +++ b/zipdeepness/images/get_image_path.py @@ -0,0 +1,29 @@ +""" +This file contains image related functionalities +""" + +import os + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def get_icon_path() -> str: + """ Get path to the file with the main plugin icon + + Returns + ------- + str + Path to the icon + """ + return get_image_path('icon.png') + + +def get_image_path(image_name) -> str: + """ Get path to an image resource, accessing it just by the name of the file (provided it is in the common directory) + + Returns + ------- + str + file path + """ + return os.path.join(SCRIPT_DIR, image_name) diff --git a/zipdeepness/images/icon.png b/zipdeepness/images/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..49a6d9ac8befcc34fa4de6cc05b3e1c3be439537 --- /dev/null +++ b/zipdeepness/images/icon.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dffd56a93df4d230e3b5b6521e3ddce178423d5c5d5692240f2f1d27bd9d070 +size 167299 diff --git a/zipdeepness/landcover_model.onnx b/zipdeepness/landcover_model.onnx new file mode 100644 index 0000000000000000000000000000000000000000..4d36437a93d66989e0d4bf774b213a72ab637cba --- /dev/null +++ b/zipdeepness/landcover_model.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12ae15a9bcc5e28f675e9c829cacbf2ab81776382f92e28645b2f91de3491d93 +size 12336500 diff --git a/zipdeepness/metadata.txt b/zipdeepness/metadata.txt new file mode 100644 index 0000000000000000000000000000000000000000..32b8bd13b2b70470cdb1b55eaf39f9b25b76ded0 --- /dev/null +++ b/zipdeepness/metadata.txt @@ -0,0 +1,56 @@ +# This file contains metadata for your plugin. + +# This file should be included when you package your plugin.# Mandatory items: + +[general] +name=Deepness: Deep Neural Remote Sensing +qgisMinimumVersion=3.22 +description=Inference of deep neural network models (ONNX) for segmentation, detection and regression +version=0.7.0 +author=PUT Vision +email=przemyslaw.aszkowski@gmail.com + +about= + Deepness plugin allows to easily perform segmentation, detection and regression on raster ortophotos with custom ONNX Neural Network models, bringing the power of deep learning to casual users. + Features highlights: + - processing any raster layer (custom ortophoto from file or layers from online providers, e.g Google Satellite) + - limiting processing range to predefined area (visible part or area defined by vector layer polygons) + - common types of models are supported: segmentation, regression, detection + - integration with layers (both for input data and model output layers). Once an output layer is created, it can be saved as a file manually + - model ZOO under development (planes detection on Bing Aerial, Corn field damage, Oil Storage tanks detection, cars detection, ...) + - training data Export Tool - exporting raster and mask as small tiles + - parametrization of the processing for advanced users (spatial resolution, overlap, postprocessing) + Plugin requires external python packages to be installed. After the first plugin startup, a Dialog will show, to assist in this process. Please visit plugin the documentation for details. + +tracker=https://github.com/PUTvision/qgis-plugin-deepness/issues +repository=https://github.com/PUTvision/qgis-plugin-deepness +# End of mandatory metadata + +# Recommended items: + +hasProcessingProvider=no +# Uncomment the following line and add your changelog: +# changelog= + +# Tags are comma separated with spaces allowed +tags=segmentation,detection,classification,machine learning,onnx,neural network,deep learning,regression,deepness,analysis,remote sensing,supervised classification + +homepage=https://qgis-plugin-deepness.readthedocs.io/ +category=Plugins +icon=images/icon.png +# experimental flag +experimental=False + +# deprecated flag (applies to the whole plugin, not just a single version) +deprecated=False + +# Since QGIS 3.8, a comma separated list of plugins to be installed +# (or upgraded) can be specified. +# Check the documentation for more information. +# plugin_dependencies= + +Category of the plugin: Raster, Vector, Database or Web +# category= + +# If the plugin can run on QGIS Server. +server=False diff --git a/zipdeepness/processing/__init__.py b/zipdeepness/processing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..74fb9ed9de376dccabda69d2632c4942cb464310 --- /dev/null +++ b/zipdeepness/processing/__init__.py @@ -0,0 +1,2 @@ +""" Main submodule for image processing and deep learning things. +""" diff --git a/zipdeepness/processing/extent_utils.py b/zipdeepness/processing/extent_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9e3a8548df5c220e5c10adf293dc466c305d7ca8 --- /dev/null +++ b/zipdeepness/processing/extent_utils.py @@ -0,0 +1,219 @@ +""" +This file contains utilities related to Extent processing +""" + +import logging + +from qgis.core import QgsCoordinateTransform, QgsRasterLayer, QgsRectangle, QgsVectorLayer +from qgis.gui import QgsMapCanvas + +from deepness.common.errors import OperationFailedException +from deepness.common.processing_parameters.map_processing_parameters import MapProcessingParameters, ProcessedAreaType +from deepness.processing.processing_utils import BoundingBox, convert_meters_to_rlayer_units + + +def round_extent_to_rlayer_grid(extent: QgsRectangle, rlayer: QgsRasterLayer) -> QgsRectangle: + """ + Round to rlayer "grid" for pixels. + Grid starts at rlayer_extent.xMinimum & yMinimum + with resolution of rlayer_units_per_pixel + + :param extent: Extent to round, needs to be in rlayer CRS units + :param rlayer: layer detemining the grid + """ + # For some ortophotos grid spacing is close to (1.0, 1.0), while it shouldn't be. + # Seems like it is some bug or special "feature" that I do not understand. + # In that case, just return the extent as it is + grid_spacing = rlayer.rasterUnitsPerPixelX(), rlayer.rasterUnitsPerPixelY() + if abs(grid_spacing[0] - 1.0) < 0.0001 and abs(grid_spacing[1] - 1.0) < 0.0001: + logging.warning('Grid spacing is close to 1.0, which is suspicious, returning extent as it is. It shouldn not be a problem for most cases.') + return extent + + grid_start = rlayer.extent().xMinimum(), rlayer.extent().yMinimum() + + x_min = grid_start[0] + int((extent.xMinimum() - grid_start[0]) / grid_spacing[0]) * grid_spacing[0] + x_max = grid_start[0] + int((extent.xMaximum() - grid_start[0]) / grid_spacing[0]) * grid_spacing[0] + y_min = grid_start[1] + int((extent.yMinimum() - grid_start[1]) / grid_spacing[1]) * grid_spacing[1] + y_max = grid_start[1] + int((extent.yMaximum() - grid_start[1]) / grid_spacing[1]) * grid_spacing[1] + + new_extent = QgsRectangle(x_min, y_min, x_max, y_max) + return new_extent + + +def calculate_extended_processing_extent(base_extent: QgsRectangle, + params: MapProcessingParameters, + rlayer: QgsVectorLayer, + rlayer_units_per_pixel: float) -> QgsRectangle: + """Calculate the "extended" processing extent, which is the full processing area, rounded to the tile size and overlap + + Parameters + ---------- + base_extent : QgsRectangle + Base extent of the processed ortophoto, which is not rounded to the tile size + params : MapProcessingParameters + Processing parameters + rlayer : QgsVectorLayer + mask layer + rlayer_units_per_pixel : float + how many rlayer CRS units are in 1 pixel + + Returns + ------- + QgsRectangle + The "extended" processing extent + """ + + # first try to add pixels at every border - same as half-overlap for other tiles + additional_pixels = params.processing_overlap_px // 2 + additional_pixels_in_units = additional_pixels * rlayer_units_per_pixel + + tmp_extent = QgsRectangle( + base_extent.xMinimum() - additional_pixels_in_units, + base_extent.yMinimum() - additional_pixels_in_units, + base_extent.xMaximum() + additional_pixels_in_units, + base_extent.yMaximum() + additional_pixels_in_units, + ) + + rlayer_extent_infinite = rlayer.extent().isEmpty() # empty extent for infinite layers + if not rlayer_extent_infinite: + tmp_extent = tmp_extent.intersect(rlayer.extent()) + + # then add borders to have the extent be equal to N * stride + tile_size, where N is a natural number + tile_size_px = params.tile_size_px + stride_px = params.processing_stride_px # stride in pixels + + current_x_pixels = round(tmp_extent.width() / rlayer_units_per_pixel) + if current_x_pixels <= tile_size_px: + missing_pixels_x = tile_size_px - current_x_pixels # just one tile + else: + pixels_in_last_stride_x = (current_x_pixels - tile_size_px) % stride_px + missing_pixels_x = (stride_px - pixels_in_last_stride_x) % stride_px + + current_y_pixels = round(tmp_extent.height() / rlayer_units_per_pixel) + if current_y_pixels <= tile_size_px: + missing_pixels_y = tile_size_px - current_y_pixels # just one tile + else: + pixels_in_last_stride_y = (current_y_pixels - tile_size_px) % stride_px + missing_pixels_y = (stride_px - pixels_in_last_stride_y) % stride_px + + missing_pixels_x_in_units = missing_pixels_x * rlayer_units_per_pixel + missing_pixels_y_in_units = missing_pixels_y * rlayer_units_per_pixel + tmp_extent.setXMaximum(tmp_extent.xMaximum() + missing_pixels_x_in_units) + tmp_extent.setYMaximum(tmp_extent.yMaximum() + missing_pixels_y_in_units) + + extended_extent = tmp_extent + return extended_extent + + +def is_extent_infinite_or_too_big(rlayer: QgsRasterLayer) -> bool: + """Check whether layer covers whole earth (infinite extent) or or is too big for processing""" + rlayer_extent = rlayer.extent() + + # empty extent happens for infinite layers + if rlayer_extent.isEmpty(): + return True + + rlayer_area_m2 = rlayer_extent.area() * (1 / convert_meters_to_rlayer_units(rlayer, 1)) ** 2 + + if rlayer_area_m2 > (1606006962349394 // 10): # so 1/3 of the earth (this magic value is from bing aerial map area) + return True + + return False + + +def calculate_base_processing_extent_in_rlayer_crs(map_canvas: QgsMapCanvas, + rlayer: QgsRasterLayer, + vlayer_mask: QgsVectorLayer, + params: MapProcessingParameters) -> QgsRectangle: + """ Determine the Base Extent of processing (Extent (rectangle) in which the actual required area is contained) + + Parameters + ---------- + map_canvas : QgsMapCanvas + currently visible map in the UI + rlayer : QgsRasterLayer + ortophotomap which is being processed + vlayer_mask : QgsVectorLayer + mask layer containing the processed area + params : MapProcessingParameters + Processing parameters + + Returns + ------- + QgsRectangle + Base Extent of processing + """ + + rlayer_extent = rlayer.extent() + processed_area_type = params.processed_area_type + rlayer_extent_infinite = is_extent_infinite_or_too_big(rlayer) + + if processed_area_type == ProcessedAreaType.ENTIRE_LAYER: + expected_extent = rlayer_extent + if rlayer_extent_infinite: + msg = "Cannot process entire layer - layer extent is not defined or too big. " \ + "Make sure you are not processing 'Entire layer' which covers entire earth surface!!" + raise OperationFailedException(msg) + elif processed_area_type == ProcessedAreaType.FROM_POLYGONS: + expected_extent_in_vlayer_crs = vlayer_mask.extent() + if vlayer_mask.crs() == rlayer.crs(): + expected_extent = expected_extent_in_vlayer_crs + else: + t = QgsCoordinateTransform() + t.setSourceCrs(vlayer_mask.crs()) + t.setDestinationCrs(rlayer.crs()) + expected_extent = t.transform(expected_extent_in_vlayer_crs) + elif processed_area_type == ProcessedAreaType.VISIBLE_PART: + # transform visible extent from mapCanvas CRS to layer CRS + active_extent_in_canvas_crs = map_canvas.extent() + canvas_crs = map_canvas.mapSettings().destinationCrs() + t = QgsCoordinateTransform() + t.setSourceCrs(canvas_crs) + t.setDestinationCrs(rlayer.crs()) + expected_extent = t.transform(active_extent_in_canvas_crs) + else: + raise Exception("Invalid processed are type!") + + expected_extent = round_extent_to_rlayer_grid(extent=expected_extent, rlayer=rlayer) + + if rlayer_extent_infinite: + base_extent = expected_extent + else: + base_extent = expected_extent.intersect(rlayer_extent) + + return base_extent + + +def calculate_base_extent_bbox_in_full_image(image_size_y: int, + base_extent: QgsRectangle, + extended_extent: QgsRectangle, + rlayer_units_per_pixel) -> BoundingBox: + """Calculate how the base extent fits in extended_extent in terms of pixel position + + Parameters + ---------- + image_size_y : int + Size of the image in y axis in pixels + base_extent : QgsRectangle + Base Extent of processing + extended_extent : QgsRectangle + Extended extent of processing + rlayer_units_per_pixel : _type_ + Number of layer units per a single image pixel + + Returns + ------- + BoundingBox + Bounding box describing position of base extent in the extended extent + """ + base_extent = base_extent + extended_extent = extended_extent + + # should round without a rest anyway, as extends are aligned to rlayer grid + base_extent_bbox_in_full_image = BoundingBox( + x_min=round((base_extent.xMinimum() - extended_extent.xMinimum()) / rlayer_units_per_pixel), + y_min=image_size_y - 1 - round((base_extent.yMaximum() - extended_extent.yMinimum()) / rlayer_units_per_pixel - 1), + x_max=round((base_extent.xMaximum() - extended_extent.xMinimum()) / rlayer_units_per_pixel) - 1, + y_max=image_size_y - 1 - round((base_extent.yMinimum() - extended_extent.yMinimum()) / rlayer_units_per_pixel), + ) + return base_extent_bbox_in_full_image diff --git a/zipdeepness/processing/map_processor/__init__.py b/zipdeepness/processing/map_processor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/zipdeepness/processing/map_processor/map_processing_result.py b/zipdeepness/processing/map_processor/map_processing_result.py new file mode 100644 index 0000000000000000000000000000000000000000..f5c2e85c68f70a5164ea0ddc70f1a1a3ac158a33 --- /dev/null +++ b/zipdeepness/processing/map_processor/map_processing_result.py @@ -0,0 +1,47 @@ +""" This file defines possible outcomes of map processing +""" + + +from typing import Callable, Optional + + +class MapProcessingResult: + """ + Base class for signaling finished processing result + """ + + def __init__(self, message: str, gui_delegate: Optional[Callable] = None): + """ + :param message: message to be shown to the user + :param gui_delegate: function to be called in GUI thread, as it is not safe to call GUI functions from other threads + """ + self.message = message + self.gui_delegate = gui_delegate + + +class MapProcessingResultSuccess(MapProcessingResult): + """ + Processing result on success + """ + + def __init__(self, message: str = '', gui_delegate: Optional[Callable] = None): + super().__init__(message=message, gui_delegate=gui_delegate) + + +class MapProcessingResultFailed(MapProcessingResult): + """ + Processing result on error + """ + + def __init__(self, error_message: str, exception=None): + super().__init__(error_message) + self.exception = exception + + +class MapProcessingResultCanceled(MapProcessingResult): + """ + Processing when processing was aborted + """ + + def __init__(self): + super().__init__(message='') diff --git a/zipdeepness/processing/map_processor/map_processor.py b/zipdeepness/processing/map_processor/map_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..903b307c42219e8846d2ed10c7daf5ede4537868 --- /dev/null +++ b/zipdeepness/processing/map_processor/map_processor.py @@ -0,0 +1,237 @@ +""" This file implements core map processing logic """ + +import logging +from typing import List, Optional, Tuple + +import numpy as np +from qgis.core import QgsRasterLayer, QgsTask, QgsVectorLayer +from qgis.gui import QgsMapCanvas +from qgis.PyQt.QtCore import pyqtSignal + +from deepness.common.defines import IS_DEBUG +from deepness.common.lazy_package_loader import LazyPackageLoader +from deepness.common.processing_parameters.map_processing_parameters import MapProcessingParameters, ProcessedAreaType +from deepness.common.temp_files_handler import TempFilesHandler +from deepness.processing import extent_utils, processing_utils +from deepness.processing.map_processor.map_processing_result import MapProcessingResult, MapProcessingResultFailed +from deepness.processing.tile_params import TileParams + +cv2 = LazyPackageLoader('cv2') + + +class MapProcessor(QgsTask): + """ + Base class for processing the ortophoto with parameters received from the UI. + + Actual processing is done in specialized child classes. Here we have the "core" functionality, + like iterating over single tiles. + + Objects of this class are created and managed by the 'Deepness'. + Work is done within QgsTask, for seamless integration with QGis GUI and logic. + """ + + # error message if finished with error, empty string otherwise + finished_signal = pyqtSignal(MapProcessingResult) + # request to show an image. Params: (image, window_name) + show_img_signal = pyqtSignal(object, str) + + def __init__(self, + rlayer: QgsRasterLayer, + vlayer_mask: Optional[QgsVectorLayer], + map_canvas: QgsMapCanvas, + params: MapProcessingParameters): + """ init + Parameters + ---------- + rlayer : QgsRasterLayer + Raster layer which is being processed + vlayer_mask : Optional[QgsVectorLayer] + Vector layer with outline of area which should be processed (within rlayer) + map_canvas : QgsMapCanvas + active map canvas (in the GUI), required if processing visible map area + params : MapProcessingParameters + see MapProcessingParameters + """ + QgsTask.__init__(self, self.__class__.__name__) + self._processing_finished = False + self.rlayer = rlayer + self.vlayer_mask = vlayer_mask + self.params = params + self._assert_qgis_doesnt_need_reload() + self._processing_result = MapProcessingResultFailed('Failed to get processing result!') + + self.stride_px = self.params.processing_stride_px # stride in pixels + self.rlayer_units_per_pixel = processing_utils.convert_meters_to_rlayer_units( + self.rlayer, self.params.resolution_m_per_px) # number of rlayer units for one tile pixel + + self.file_handler = TempFilesHandler() if self.params.local_cache else None + + # extent in which the actual required area is contained, without additional extensions, rounded to rlayer grid + self.base_extent = extent_utils.calculate_base_processing_extent_in_rlayer_crs( + map_canvas=map_canvas, + rlayer=self.rlayer, + vlayer_mask=self.vlayer_mask, + params=self.params) + + # extent which should be used during model inference, as it includes extra margins to have full tiles, + # rounded to rlayer grid + self.extended_extent = extent_utils.calculate_extended_processing_extent( + base_extent=self.base_extent, + rlayer=self.rlayer, + params=self.params, + rlayer_units_per_pixel=self.rlayer_units_per_pixel) + + # processed rlayer dimensions (for extended_extent) + self.img_size_x_pixels = round(self.extended_extent.width() / self.rlayer_units_per_pixel) # how many columns (x) + self.img_size_y_pixels = round(self.extended_extent.height() / self.rlayer_units_per_pixel) # how many rows (y) + + # Coordinate of base image within extended image (images for base_extent and extended_extent) + self.base_extent_bbox_in_full_image = extent_utils.calculate_base_extent_bbox_in_full_image( + image_size_y=self.img_size_y_pixels, + base_extent=self.base_extent, + extended_extent=self.extended_extent, + rlayer_units_per_pixel=self.rlayer_units_per_pixel) + + # Number of tiles in x and y dimensions which will be used during processing + # As we are using "extended_extent" this should divide without any rest + self.x_bins_number = round((self.img_size_x_pixels - self.params.tile_size_px) / self.stride_px) + 1 + self.y_bins_number = round((self.img_size_y_pixels - self.params.tile_size_px) / self.stride_px) + 1 + + # Mask determining area to process (within extended_extent coordinates) + self.area_mask_img = processing_utils.create_area_mask_image( + vlayer_mask=self.vlayer_mask, + rlayer=self.rlayer, + extended_extent=self.extended_extent, + rlayer_units_per_pixel=self.rlayer_units_per_pixel, + image_shape_yx=(self.img_size_y_pixels, self.img_size_x_pixels), + files_handler=self.file_handler) # type: Optional[np.ndarray] + + self._result_img = None + + def set_results_img(self, img): + if self._result_img is not None: + raise Exception("Result image already created!") + + self._result_img = img + + def get_result_img(self): + if self._result_img is None: + raise Exception("Result image not yet created!") + + return self._result_img + + def _assert_qgis_doesnt_need_reload(self): + """ If the plugin is somehow invalid, it cannot compare the enums correctly + I suppose it could be fixed somehow, but no need to investigate it now, + it affects only the development + """ + + if self.params.processed_area_type.__class__ != ProcessedAreaType: + raise Exception("Disable plugin, restart QGis and enable plugin again!") + + def run(self): + try: + self._processing_result = self._run() + except Exception as e: + logging.exception("Error occurred in MapProcessor:") + msg = "Unhandled exception occurred. See Python Console for details" + self._processing_result = MapProcessingResultFailed(msg, exception=e) + if IS_DEBUG: + raise e + + self._processing_finished = True + return True + + def _run(self) -> MapProcessingResult: + raise NotImplementedError('Base class not implemented!') + + def finished(self, result: bool): + if result: + gui_delegate = self._processing_result.gui_delegate + if gui_delegate is not None: + gui_delegate() + else: + self._processing_result = MapProcessingResultFailed("Unhandled processing error!") + self.finished_signal.emit(self._processing_result) + + @staticmethod + def is_busy(): + return True + + def _show_image(self, img, window_name='img'): + self.show_img_signal.emit(img, window_name) + + def limit_extended_extent_image_to_base_extent_with_mask(self, full_img): + """ + Limit an image which is for extended_extent to the base_extent image. + If a limiting polygon was used for processing, it will be also applied. + :param full_img: + :return: + """ + # TODO look for some inplace operation to save memory + # cv2.copyTo(src=full_img, mask=area_mask_img, dst=full_img) # this doesn't work due to implementation details + + for i in range(full_img.shape[0]): + full_img[i] = cv2.copyTo(src=full_img[i], mask=self.area_mask_img) + + b = self.base_extent_bbox_in_full_image + result_img = full_img[:, b.y_min:b.y_max+1, b.x_min:b.x_max+1] + return result_img + + def _get_array_or_mmapped_array(self, final_shape_px): + if self.file_handler is not None: + full_result_img = np.memmap( + self.file_handler.get_results_img_path(), + dtype=np.uint8, + mode='w+', + shape=final_shape_px) + else: + full_result_img = np.zeros(final_shape_px, np.uint8) + + return full_result_img + + def tiles_generator(self) -> Tuple[np.ndarray, TileParams]: + """ + Iterate over all tiles, as a Python generator function + """ + total_tiles = self.x_bins_number * self.y_bins_number + + for y_bin_number in range(self.y_bins_number): + for x_bin_number in range(self.x_bins_number): + tile_no = y_bin_number * self.x_bins_number + x_bin_number + progress = tile_no / total_tiles * 100 + self.setProgress(progress) + print(f" Processing tile {tile_no} / {total_tiles} [{progress:.2f}%]") + tile_params = TileParams( + x_bin_number=x_bin_number, y_bin_number=y_bin_number, + x_bins_number=self.x_bins_number, y_bins_number=self.y_bins_number, + params=self.params, + processing_extent=self.extended_extent, + rlayer_units_per_pixel=self.rlayer_units_per_pixel) + + if not tile_params.is_tile_within_mask(self.area_mask_img): + continue # tile outside of mask - to be skipped + + tile_img = processing_utils.get_tile_image( + rlayer=self.rlayer, extent=tile_params.extent, params=self.params) + + yield tile_img, tile_params + + def tiles_generator_batched(self) -> Tuple[np.ndarray, List[TileParams]]: + """ + Iterate over all tiles, as a Python generator function, but return them in batches + """ + + tile_img_batch, tile_params_batch = [], [] + + for tile_img, tile_params in self.tiles_generator(): + tile_img_batch.append(tile_img) + tile_params_batch.append(tile_params) + + if len(tile_img_batch) >= self.params.batch_size: + yield np.array(tile_img_batch), tile_params_batch + tile_img_batch, tile_params_batch = [], [] + + if len(tile_img_batch) > 0: + yield np.array(tile_img_batch), tile_params_batch + tile_img_batch, tile_params_batch = [], [] diff --git a/zipdeepness/processing/map_processor/map_processor_detection.py b/zipdeepness/processing/map_processor/map_processor_detection.py new file mode 100644 index 0000000000000000000000000000000000000000..8b5658b4249eb21970cf3302dec0f7d7a1bdd7a7 --- /dev/null +++ b/zipdeepness/processing/map_processor/map_processor_detection.py @@ -0,0 +1,288 @@ +""" This file implements map processing for detection model """ +from typing import List + +import cv2 +import numpy as np +from qgis.core import QgsFeature, QgsGeometry, QgsProject, QgsVectorLayer +from qgis.PyQt.QtCore import QVariant +from qgis.core import QgsFields, QgsField + +from deepness.common.processing_parameters.detection_parameters import DetectionParameters +from deepness.processing import processing_utils +from deepness.processing.map_processor.map_processing_result import (MapProcessingResult, MapProcessingResultCanceled, + MapProcessingResultSuccess) +from deepness.processing.map_processor.map_processor_with_model import MapProcessorWithModel +from deepness.processing.map_processor.utils.ckdtree import cKDTree +from deepness.processing.models.detector import Detection, Detector +from deepness.processing.tile_params import TileParams +from deepness.processing.models.detector import DetectorType + + +class MapProcessorDetection(MapProcessorWithModel): + """ + MapProcessor specialized for detecting objects (where there is a finite list of detected objects + of different classes, which area (bounding boxes) may overlap) + """ + + def __init__(self, + params: DetectionParameters, + **kwargs): + super().__init__( + params=params, + model=params.model, + **kwargs) + self.detection_parameters = params + self.model = params.model # type: Detector + self.model.set_inference_params( + confidence=params.confidence, + iou_threshold=params.iou_threshold + ) + self.model.set_model_type_param(model_type=params.detector_type) + self._all_detections = None + + def get_all_detections(self) -> List[Detection]: + return self._all_detections + + def _run(self) -> MapProcessingResult: + all_bounding_boxes = [] # type: List[Detection] + for tile_img_batched, tile_params_batched in self.tiles_generator_batched(): + if self.isCanceled(): + return MapProcessingResultCanceled() + + bounding_boxes_in_tile_batched = self._process_tile(tile_img_batched, tile_params_batched) + all_bounding_boxes += [d for det in bounding_boxes_in_tile_batched for d in det] + + with_rot = self.detection_parameters.detector_type == DetectorType.YOLO_ULTRALYTICS_OBB + + if len(all_bounding_boxes) > 0: + all_bounding_boxes_nms = self.remove_overlaping_detections(all_bounding_boxes, iou_threshold=self.detection_parameters.iou_threshold, with_rot=with_rot) + all_bounding_boxes_restricted = self.limit_bounding_boxes_to_processed_area(all_bounding_boxes_nms) + else: + all_bounding_boxes_restricted = [] + + gui_delegate = self._create_vlayer_for_output_bounding_boxes(all_bounding_boxes_restricted) + + result_message = self._create_result_message(all_bounding_boxes_restricted) + self._all_detections = all_bounding_boxes_restricted + return MapProcessingResultSuccess( + message=result_message, + gui_delegate=gui_delegate, + ) + + def limit_bounding_boxes_to_processed_area(self, bounding_boxes: List[Detection]) -> List[Detection]: + """ + Limit all bounding boxes to the constrained area that we process. + E.g. if we are detecting peoples in a circle, we don't want to count peoples in the entire rectangle + + :return: + """ + bounding_boxes_restricted = [] + for det in bounding_boxes: + # if bounding box is not in the area_mask_img (at least in some percentage) - remove it + + if self.area_mask_img is not None: + det_slice = det.bbox.get_slice() + area_subimg = self.area_mask_img[det_slice] + pixels_in_area = np.count_nonzero(area_subimg) + else: + det_bounding_box = det.bbox + pixels_in_area = self.base_extent_bbox_in_full_image.calculate_overlap_in_pixels(det_bounding_box) + total_pixels = det.bbox.get_area() + coverage = pixels_in_area / total_pixels + if coverage > 0.5: # some arbitrary value, 50% seems reasonable + bounding_boxes_restricted.append(det) + + return bounding_boxes_restricted + + def _create_result_message(self, bounding_boxes: List[Detection]) -> str: + # hack, allways one output + model_outputs = self._get_indexes_of_model_output_channels_to_create() + channels = range(model_outputs[0]) + + counts_mapping = {} + total_counts = 0 + for channel_id in channels: + filtered_bounding_boxes = [det for det in bounding_boxes if det.clss == channel_id] + counts = len(filtered_bounding_boxes) + counts_mapping[channel_id] = counts + total_counts += counts + + txt = f'Detection done for {len(channels)} model output classes, with the following statistics:\n' + for channel_id in channels: + counts = counts_mapping[channel_id] + + if total_counts: + counts_percentage = counts / total_counts * 100 + else: + counts_percentage = 0 + + txt += f' - {self.model.get_channel_name(0, channel_id)}: counts = {counts} ({counts_percentage:.2f} %)\n' + + return txt + + def _create_vlayer_for_output_bounding_boxes(self, bounding_boxes: List[Detection]): + vlayers = [] + + # hack, allways one output + model_outputs = self._get_indexes_of_model_output_channels_to_create() + channels = range(model_outputs[0]) + + for channel_id in channels: + filtered_bounding_boxes = [det for det in bounding_boxes if det.clss == channel_id] + print(f'Detections for class {channel_id}: {len(filtered_bounding_boxes)}') + + vlayer = QgsVectorLayer("multipolygon", self.model.get_channel_name(0, channel_id), "memory") + vlayer.setCrs(self.rlayer.crs()) + prov = vlayer.dataProvider() + prov.addAttributes([QgsField("confidence", QVariant.Double)]) + vlayer.updateFields() + + features = [] + for det in filtered_bounding_boxes: + feature = QgsFeature() + if det.mask is None: + bbox_corners_pixels = det.bbox.get_4_corners() + bbox_corners_crs = processing_utils.transform_points_list_xy_to_target_crs( + points=bbox_corners_pixels, + extent=self.extended_extent, + rlayer_units_per_pixel=self.rlayer_units_per_pixel, + ) + #feature = QgsFeature() #move outside of the if block + polygon_xy_vec_vec = [ + bbox_corners_crs + ] + geometry = QgsGeometry.fromPolygonXY(polygon_xy_vec_vec) + #feature.setGeometry(geometry) + #features.append(feature) + else: + contours, _ = cv2.findContours(det.mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + contours = sorted(contours, key=cv2.contourArea, reverse=True) + + x_offset, y_offset = det.mask_offsets + + if len(contours) > 0: + countur = contours[0] + + corners = [] + for point in countur: + corners.append(int(point[0][0]) + x_offset) + corners.append(int(point[0][1]) + y_offset) + + mask_corners_pixels = cv2.convexHull(np.array(corners).reshape((-1, 2))).squeeze() + + mask_corners_crs = processing_utils.transform_points_list_xy_to_target_crs( + points=mask_corners_pixels, + extent=self.extended_extent, + rlayer_units_per_pixel=self.rlayer_units_per_pixel, + ) + + #feature = QgsFeature() + polygon_xy_vec_vec = [ + mask_corners_crs + ] + geometry = QgsGeometry.fromPolygonXY(polygon_xy_vec_vec) + #feature.setGeometry(geometry) + #features.append(feature) + feature.setGeometry(geometry) + feature.setAttributes([float(det.conf)]) + features.append(feature) + + #vlayer = QgsVectorLayer("multipolygon", self.model.get_channel_name(0, channel_id), "memory") + #vlayer.setCrs(self.rlayer.crs()) + #prov = vlayer.dataProvider() + + color = vlayer.renderer().symbol().color() + OUTPUT_VLAYER_COLOR_TRANSPARENCY = 80 + color.setAlpha(OUTPUT_VLAYER_COLOR_TRANSPARENCY) + vlayer.renderer().symbol().setColor(color) + # TODO - add also outline for the layer (thicker black border) + + prov.addFeatures(features) + vlayer.updateExtents() + + vlayers.append(vlayer) + + # accessing GUI from non-GUI thread is not safe, so we need to delegate it to the GUI thread + def add_to_gui(): + group = QgsProject.instance().layerTreeRoot().insertGroup(0, 'model_output') + for vlayer in vlayers: + QgsProject.instance().addMapLayer(vlayer, False) + group.addLayer(vlayer) + + return add_to_gui + + @staticmethod + def remove_overlaping_detections(bounding_boxes: List[Detection], iou_threshold: float, with_rot: bool = False) -> List[Detection]: + bboxes = [] + probs = [] + for det in bounding_boxes: + if with_rot: + bboxes.append(det.get_bbox_xyxy_rot()) + else: + bboxes.append(det.get_bbox_xyxy()) + probs.append(det.conf) + + bboxes = np.array(bboxes) + probs = np.array(probs) + + pick_ids = Detector.non_max_suppression_fast(boxes=bboxes, probs=probs, iou_threshold=iou_threshold, with_rot=with_rot) + + filtered_bounding_boxes = [x for i, x in enumerate(bounding_boxes) if i in pick_ids] + filtered_bounding_boxes = sorted(filtered_bounding_boxes, reverse=True) + + pick_ids_kde = MapProcessorDetection.non_max_kdtree(filtered_bounding_boxes, iou_threshold) + + filtered_bounding_boxes = [x for i, x in enumerate(filtered_bounding_boxes) if i in pick_ids_kde] + + return filtered_bounding_boxes + + @staticmethod + def non_max_kdtree(bounding_boxes: List[Detection], iou_threshold: float) -> List[int]: + """ Remove overlapping bounding boxes using kdtree + + :param bounding_boxes: List of bounding boxes in (xyxy format) + :param iou_threshold: Threshold for intersection over union + :return: Pick ids to keep + """ + + centers = np.array([det.get_bbox_center() for det in bounding_boxes]) + + kdtree = cKDTree(centers) + pick_ids = set() + removed_ids = set() + + for i, bbox in enumerate(bounding_boxes): + if i in removed_ids: + continue + + indices = kdtree.query(bbox.get_bbox_center(), k=min(10, len(bounding_boxes))) + + for j in indices: + if j in removed_ids: + continue + + if i == j: + continue + + iou = bbox.bbox.calculate_intersection_over_smaler_area(bounding_boxes[j].bbox) + + if iou > iou_threshold: + removed_ids.add(j) + + pick_ids.add(i) + + return pick_ids + + @staticmethod + def convert_bounding_boxes_to_absolute_positions(bounding_boxes_relative: List[Detection], + tile_params: TileParams): + for det in bounding_boxes_relative: + det.convert_to_global(offset_x=tile_params.start_pixel_x, offset_y=tile_params.start_pixel_y) + + def _process_tile(self, tile_img: np.ndarray, tile_params_batched: List[TileParams]) -> np.ndarray: + bounding_boxes_batched: List[Detection] = self.model.process(tile_img) + + for bounding_boxes, tile_params in zip(bounding_boxes_batched, tile_params_batched): + self.convert_bounding_boxes_to_absolute_positions(bounding_boxes, tile_params) + + return bounding_boxes_batched diff --git a/zipdeepness/processing/map_processor/map_processor_recognition.py b/zipdeepness/processing/map_processor/map_processor_recognition.py new file mode 100644 index 0000000000000000000000000000000000000000..340a568edda5c1680a4d719a75b40cecc2fda15f --- /dev/null +++ b/zipdeepness/processing/map_processor/map_processor_recognition.py @@ -0,0 +1,221 @@ +""" This file implements map processing for Recognition model """ + +import os +import uuid +from typing import List + +import numpy as np +from numpy.linalg import norm +from osgeo import gdal, osr +from qgis.core import QgsProject, QgsRasterLayer + +from deepness.common.defines import IS_DEBUG +from deepness.common.lazy_package_loader import LazyPackageLoader +from deepness.common.misc import TMP_DIR_PATH +from deepness.common.processing_parameters.recognition_parameters import RecognitionParameters +from deepness.processing.map_processor.map_processing_result import (MapProcessingResult, MapProcessingResultCanceled, + MapProcessingResultFailed, + MapProcessingResultSuccess) +from deepness.processing.map_processor.map_processor_with_model import MapProcessorWithModel + +cv2 = LazyPackageLoader('cv2') + + +class MapProcessorRecognition(MapProcessorWithModel): + """ + MapProcessor specialized for Recognition model + """ + + def __init__(self, params: RecognitionParameters, **kwargs): + super().__init__(params=params, model=params.model, **kwargs) + self.recognition_parameters = params + self.model = params.model + + def _run(self) -> MapProcessingResult: + try: + query_img = cv2.imread(self.recognition_parameters.query_image_path) + assert query_img is not None, f"Error occurred while reading query image: {self.recognition_parameters.query_image_path}" + except Exception as e: + return MapProcessingResultFailed(f"Error occurred while reading query image: {e}") + + # some hardcoded code for recognition model + query_img = cv2.cvtColor(query_img, cv2.COLOR_BGR2RGB) + query_img_resized = cv2.resize(query_img, self.model.get_input_shape()[2:4][::-1]) + query_img_batched = np.array([query_img_resized]) + + query_img_emb = self.model.process(query_img_batched)[0][0] + + final_shape_px = ( + self.img_size_y_pixels, + self.img_size_x_pixels, + ) + + stride = self.stride_px + full_result_img = np.zeros(final_shape_px, np.float32) + mask = np.zeros_like(full_result_img, dtype=np.int16) + highest = 0 + for tile_img_batched, tile_params_batched in self.tiles_generator_batched(): + if self.isCanceled(): + return MapProcessingResultCanceled() + + tile_result_batched = self._process_tile(tile_img_batched)[0] + + for tile_result, tile_params in zip(tile_result_batched, tile_params_batched): + cossim = np.dot(query_img_emb, tile_result)/(norm(query_img_emb)*norm(tile_result)) + + x_bin = tile_params.x_bin_number + y_bin = tile_params.y_bin_number + size = self.params.tile_size_px + + if cossim > highest: + highest = cossim + x_high = x_bin + y_high = y_bin + + full_result_img[y_bin*stride:y_bin*stride+size, x_bin*stride:x_bin*stride + size] += cossim + mask[y_bin*stride:y_bin*stride+size, x_bin*stride:x_bin*stride + size] += 1 + + full_result_img = full_result_img/mask + self.set_results_img(full_result_img) + + gui_delegate = self._create_rlayers_from_images_for_base_extent(self.get_result_img(), x_high, y_high, size, stride) + result_message = self._create_result_message(self.get_result_img(), x_high*self.params.tile_size_px, y_high*self.params.tile_size_px) + return MapProcessingResultSuccess( + message=result_message, + gui_delegate=gui_delegate, + ) + + def _create_result_message(self, result_img: List[np.ndarray], x_high, y_high) -> str: + txt = f"Recognition ended, best result found at {x_high}, {y_high}, {result_img.shape}" + return txt + + def limit_extended_extent_image_to_base_extent_with_mask(self, full_img): + """ + Limit an image which is for extended_extent to the base_extent image. + If a limiting polygon was used for processing, it will be also applied. + :param full_img: + :return: + """ + # TODO look for some inplace operation to save memory + # cv2.copyTo(src=full_img, mask=area_mask_img, dst=full_img) # this doesn't work due to implementation details + # full_img = cv2.copyTo(src=full_img, mask=self.area_mask_img) + + b = self.base_extent_bbox_in_full_image + result_img = full_img[ + int(b.y_min * self.recognition_parameters.scale_factor): int( + b.y_max * self.recognition_parameters.scale_factor + ), + int(b.x_min * self.recognition_parameters.scale_factor): int( + b.x_max * self.recognition_parameters.scale_factor + ), + :, + ] + return result_img + + def load_rlayer_from_file(self, file_path): + """ + Create raster layer from tif file + """ + file_name = os.path.basename(file_path) + base_file_name = file_name.split("___")[ + 0 + ] # we remove the random_id string we created a moment ago + rlayer = QgsRasterLayer(file_path, base_file_name) + if rlayer.width() == 0: + raise Exception( + "0 width - rlayer not loaded properly. Probably invalid file path?" + ) + rlayer.setCrs(self.rlayer.crs()) + return rlayer + + def _create_rlayers_from_images_for_base_extent( + self, result_img: np.ndarray, + x_high, + y_high, + size, + stride + ): + y = y_high * stride + x = x_high * stride + + result_img[y, x:x+size-1] = 1 + result_img[y+size-1, x:x+size-1] = 1 + result_img[y:y+size-1, x] = 1 + result_img[y:y+size-1, x+size-1] = 1 + + # TODO: We are creating a new file for each layer. + # Maybe can we pass ownership of this file to QGis? + # Or maybe even create vlayer directly from array, without a file? + + random_id = str(uuid.uuid4()).replace("-", "") + file_path = os.path.join(TMP_DIR_PATH, f"{random_id}.tif") + self.save_result_img_as_tif(file_path=file_path, img=np.expand_dims(result_img, axis=2)) + + rlayer = self.load_rlayer_from_file(file_path) + OUTPUT_RLAYER_OPACITY = 0.5 + rlayer.renderer().setOpacity(OUTPUT_RLAYER_OPACITY) + + # accessing GUI from non-GUI thread is not safe, so we need to delegate it to the GUI thread + def add_to_gui(): + group = ( + QgsProject.instance() + .layerTreeRoot() + .insertGroup(0, "Cosine similarity score") + ) + QgsProject.instance().addMapLayer(rlayer, False) + group.addLayer(rlayer) + + return add_to_gui + + def save_result_img_as_tif(self, file_path: str, img: np.ndarray): + """ + As we cannot pass easily an numpy array to be displayed as raster layer, we create temporary geotif files, + which will be loaded as layer later on + + Partially based on example from: + https://gis.stackexchange.com/questions/82031/gdal-python-set-projection-of-a-raster-not-working + """ + os.makedirs(os.path.dirname(file_path), exist_ok=True) + + extent = self.base_extent + crs = self.rlayer.crs() + + geo_transform = [ + extent.xMinimum(), + self.rlayer_units_per_pixel, + 0, + extent.yMaximum(), + 0, + -self.rlayer_units_per_pixel, + ] + + driver = gdal.GetDriverByName("GTiff") + n_lines = img.shape[0] + n_cols = img.shape[1] + n_chanels = img.shape[2] + # data_type = gdal.GDT_Byte + data_type = gdal.GDT_Float32 + grid_data = driver.Create( + "grid_data", n_cols, n_lines, n_chanels, data_type + ) # , options) + # loop over chanels + for i in range(1, img.shape[2] + 1): + grid_data.GetRasterBand(i).WriteArray(img[:, :, i - 1]) + + # crs().srsid() - maybe we can use the ID directly - but how? + # srs.ImportFromEPSG() + srs = osr.SpatialReference() + srs.SetFromUserInput(crs.authid()) + + grid_data.SetProjection(srs.ExportToWkt()) + grid_data.SetGeoTransform(geo_transform) + driver.CreateCopy(file_path, grid_data, 0) + + def _process_tile(self, tile_img: np.ndarray) -> np.ndarray: + result = self.model.process(tile_img) + + # NOTE - currently we are saving result as float32, so we are losing some accuraccy. + # result = np.clip(result, 0, 255) # old version with uint8_t - not used anymore + # result = result.astype(np.float32) + + return result diff --git a/zipdeepness/processing/map_processor/map_processor_regression.py b/zipdeepness/processing/map_processor/map_processor_regression.py new file mode 100644 index 0000000000000000000000000000000000000000..15c3c8dc9e9f9789f2ab1415cdaa66c57b744c2b --- /dev/null +++ b/zipdeepness/processing/map_processor/map_processor_regression.py @@ -0,0 +1,174 @@ +""" This file implements map processing for regression model """ + +import os +import uuid +from typing import List + +import numpy as np +from osgeo import gdal, osr +from qgis.core import QgsProject, QgsRasterLayer + +from deepness.common.misc import TMP_DIR_PATH +from deepness.common.processing_parameters.regression_parameters import RegressionParameters +from deepness.processing.map_processor.map_processing_result import (MapProcessingResult, MapProcessingResultCanceled, + MapProcessingResultSuccess) +from deepness.processing.map_processor.map_processor_with_model import MapProcessorWithModel + + +class MapProcessorRegression(MapProcessorWithModel): + """ + MapProcessor specialized for Regression model (where each pixel has a value representing some feature intensity) + """ + + def __init__(self, + params: RegressionParameters, + **kwargs): + super().__init__( + params=params, + model=params.model, + **kwargs) + self.regression_parameters = params + self.model = params.model + + def _run(self) -> MapProcessingResult: + number_of_output_channels = len(self._get_indexes_of_model_output_channels_to_create()) + final_shape_px = (number_of_output_channels, self.img_size_y_pixels, self.img_size_x_pixels) + + # NOTE: consider whether we can use float16/uint16 as datatype + full_result_imgs = self._get_array_or_mmapped_array(final_shape_px) + + for tile_img_batched, tile_params_batched in self.tiles_generator_batched(): + if self.isCanceled(): + return MapProcessingResultCanceled() + + tile_results_batched = self._process_tile(tile_img_batched) + + for tile_results, tile_params in zip(tile_results_batched, tile_params_batched): + tile_params.set_mask_on_full_img( + tile_result=tile_results, + full_result_img=full_result_imgs) + + # plt.figure(); plt.imshow(full_result_img); plt.show(block=False); plt.pause(0.001) + full_result_imgs = self.limit_extended_extent_images_to_base_extent_with_mask(full_imgs=full_result_imgs) + self.set_results_img(full_result_imgs) + + gui_delegate = self._create_rlayers_from_images_for_base_extent(self.get_result_img()) + result_message = self._create_result_message(self.get_result_img()) + return MapProcessingResultSuccess( + message=result_message, + gui_delegate=gui_delegate, + ) + + def _create_result_message(self, result_imgs: List[np.ndarray]) -> str: + txt = f'Regression done, with the following statistics:\n' + for output_id, _ in enumerate(self._get_indexes_of_model_output_channels_to_create()): + result_img = result_imgs[output_id] + + average_value = np.mean(result_img) + std = np.std(result_img) + + txt += f' - {self.model.get_channel_name(output_id, 0)}: average_value = {average_value:.2f} (std = {std:.2f}, ' \ + f'min={np.min(result_img)}, max={np.max(result_img)})\n' + + return txt + + def limit_extended_extent_images_to_base_extent_with_mask(self, full_imgs: List[np.ndarray]): + """ + Same as 'limit_extended_extent_image_to_base_extent_with_mask' but for a list of images. + See `limit_extended_extent_image_to_base_extent_with_mask` for details. + :param full_imgs: + :return: + """ + return self.limit_extended_extent_image_to_base_extent_with_mask(full_img=full_imgs) + + def load_rlayer_from_file(self, file_path): + """ + Create raster layer from tif file + """ + file_name = os.path.basename(file_path) + base_file_name = file_name.split('___')[0] # we remove the random_id string we created a moment ago + rlayer = QgsRasterLayer(file_path, base_file_name) + if rlayer.width() == 0: + raise Exception("0 width - rlayer not loaded properly. Probably invalid file path?") + rlayer.setCrs(self.rlayer.crs()) + return rlayer + + def _create_rlayers_from_images_for_base_extent(self, result_imgs: List[np.ndarray]): + # TODO: We are creating a new file for each layer. + # Maybe can we pass ownership of this file to QGis? + # Or maybe even create vlayer directly from array, without a file? + rlayers = [] + + for output_id, _ in enumerate(self._get_indexes_of_model_output_channels_to_create()): + + random_id = str(uuid.uuid4()).replace('-', '') + file_path = os.path.join(TMP_DIR_PATH, f'{self.model.get_channel_name(output_id, 0)}__{random_id}.tif') + self.save_result_img_as_tif(file_path=file_path, img=result_imgs[output_id]) + + rlayer = self.load_rlayer_from_file(file_path) + OUTPUT_RLAYER_OPACITY = 0.5 + rlayer.renderer().setOpacity(OUTPUT_RLAYER_OPACITY) + rlayers.append(rlayer) + + def add_to_gui(): + group = QgsProject.instance().layerTreeRoot().insertGroup(0, 'model_output') + for rlayer in rlayers: + QgsProject.instance().addMapLayer(rlayer, False) + group.addLayer(rlayer) + + return add_to_gui + + def save_result_img_as_tif(self, file_path: str, img: np.ndarray): + """ + As we cannot pass easily an numpy array to be displayed as raster layer, we create temporary geotif files, + which will be loaded as layer later on + + Partially based on example from: + https://gis.stackexchange.com/questions/82031/gdal-python-set-projection-of-a-raster-not-working + """ + os.makedirs(os.path.dirname(file_path), exist_ok=True) + + extent = self.base_extent + crs = self.rlayer.crs() + + geo_transform = [extent.xMinimum(), self.rlayer_units_per_pixel, 0, + extent.yMaximum(), 0, -self.rlayer_units_per_pixel] + + driver = gdal.GetDriverByName('GTiff') + n_lines = img.shape[0] + n_cols = img.shape[1] + # data_type = gdal.GDT_Byte + data_type = gdal.GDT_Float32 + grid_data = driver.Create('grid_data', n_cols, n_lines, 1, data_type) # , options) + grid_data.GetRasterBand(1).WriteArray(img) + + # crs().srsid() - maybe we can use the ID directly - but how? + # srs.ImportFromEPSG() + srs = osr.SpatialReference() + srs.SetFromUserInput(crs.authid()) + + grid_data.SetProjection(srs.ExportToWkt()) + grid_data.SetGeoTransform(geo_transform) + driver.CreateCopy(file_path, grid_data, 0) + print(f'***** {file_path = }') + + def _process_tile(self, tile_img: np.ndarray) -> np.ndarray: + many_result = self.model.process(tile_img) + many_outputs = [] + + for result in many_result: + result[np.isnan(result)] = 0 + result *= self.regression_parameters.output_scaling + + # NOTE - currently we are saving result as float32, so we are losing some accuraccy. + # result = np.clip(result, 0, 255) # old version with uint8_t - not used anymore + result = result.astype(np.float32) + + if len(result.shape) == 3: + result = np.expand_dims(result, axis=1) + + many_outputs.append(result[:, 0]) + + many_outputs = np.array(many_outputs).transpose((1, 0, 2, 3)) + + return many_outputs diff --git a/zipdeepness/processing/map_processor/map_processor_segmentation.py b/zipdeepness/processing/map_processor/map_processor_segmentation.py new file mode 100644 index 0000000000000000000000000000000000000000..75f98910862afc05292b9da0b609b65d8e293893 --- /dev/null +++ b/zipdeepness/processing/map_processor/map_processor_segmentation.py @@ -0,0 +1,386 @@ +""" This file implements map processing for segmentation model """ + +from typing import Callable + +import numpy as np +from qgis.core import QgsProject, QgsVectorLayer, QgsCoordinateTransform, QgsCoordinateReferenceSystem, QgsField, QgsFeature, QgsGeometry, QgsMessageLog +from deepness.common.lazy_package_loader import LazyPackageLoader +from deepness.common.processing_parameters.segmentation_parameters import SegmentationParameters +from deepness.processing import processing_utils +from deepness.processing.map_processor.map_processing_result import (MapProcessingResult, MapProcessingResultCanceled, + MapProcessingResultSuccess) +from deepness.processing.map_processor.map_processor_with_model import MapProcessorWithModel +from deepness.processing.tile_params import TileParams +import geopandas as gpd +import traceback +import pandas as pd +from rasterio.features import shapes +from affine import Affine +from shapely.geometry import shape + +cv2 = LazyPackageLoader('cv2') + + +class MapProcessorSegmentation(MapProcessorWithModel): + """ + MapProcessor specialized for Segmentation model (where each pixel is assigned to one class). + """ + + def __init__(self, + params: SegmentationParameters, + **kwargs): + super().__init__( + params=params, + model=params.model, + **kwargs) + self.segmentation_parameters = params + self.model = params.model + self.new_class_names = { + "0": "background", + "2": "zone-verte", + "3": "eau", + "4": "route", + "5": "non-residentiel", + "6": "Villa", + "7": "traditionnel", + "8": "appartement", + "9": "autre" + } + + + def tile_mask_to_gdf_latlon(self,tile: TileParams, mask: np.ndarray, raster_layer_crs=None): + + + # remove channel if exists + if mask.ndim == 3: + mask = mask[0] + + H, W = mask.shape + + mask_binary = mask != 0 + + x_min = tile.extent.xMinimum() + y_max = tile.extent.yMaximum() + pixel_size = tile.rlayer_units_per_pixel + transform = Affine(pixel_size, 0, x_min, 0, -pixel_size, y_max) + + + results = ( + {"properties": {"class_id": int(v)}, "geometry": s} + for s, v in shapes(mask.astype(np.int16), mask=mask_binary, transform=transform) + ) + geoms = [] + for r in results: + class_id = r["properties"]["class_id"] + class_name = self.new_class_names.get(str(class_id), "unknown") + geom = shape(r["geometry"]) + geoms.append({"geometry": geom, "class_id": class_id, "class_name": class_name}) + if geoms: + gdf = gpd.GeoDataFrame(geoms, geometry="geometry", crs=raster_layer_crs.authid() if raster_layer_crs else "EPSG:3857") + else: + # empty GeoDataFrame with the right columns + gdf = gpd.GeoDataFrame(columns=["geometry", "class_id", "class_name"], geometry="geometry", crs=raster_layer_crs.authid() if raster_layer_crs else "EPSG:3857") + +# Transform to lat/lon if needed + if not gdf.empty: + gdf = gdf.to_crs("EPSG:4326") + + gdf = gpd.GeoDataFrame(geoms, geometry="geometry", crs=raster_layer_crs.authid() if raster_layer_crs else "EPSG:3857") + + # Transform to lat/lon if needed + gdf = gdf.to_crs("EPSG:4326") + + # # compute CRS coordinates for each pixel + # xs = x_min + np.arange(W) * pixel_size + # ys = y_max - np.arange(H) * pixel_size # Y decreases downward + + # xs_grid, ys_grid = np.meshgrid(xs, ys) + + # # flatten + # xs_flat = xs_grid.flatten() + # ys_flat = ys_grid.flatten() + # classes_flat = mask.flatten() + + # # create points + # points = [Point(x, y) for x, y in zip(xs_flat, ys_flat)] + + # gdf = gpd.GeoDataFrame({'class': classes_flat}, geometry=points) + + # # set CRS + # if raster_layer_crs is None: + # raster_layer_crs = QgsCoordinateReferenceSystem("EPSG:3857") # fallback + # gdf.set_crs(raster_layer_crs.authid(), inplace=True) + + # # transform to lat/lon (EPSG:4326) + # transformer = QgsCoordinateTransform(raster_layer_crs, QgsCoordinateReferenceSystem("EPSG:4326"), QgsProject.instance()) + # gdf['geometry'] = gdf['geometry'].apply(lambda pt: Point(*transformer.transform(pt.x, pt.y))) + # # transformed_coords = [transformer.transform(pt.x, pt.y) for pt in gdf.geometry] + # # gdf['geometry'] = [Point(x, y) for x, y in transformed_coords] + + + return gdf + + + def _run(self) -> MapProcessingResult: + final_shape_px = (len(self._get_indexes_of_model_output_channels_to_create()), self.img_size_y_pixels, self.img_size_x_pixels) + + full_result_img = self._get_array_or_mmapped_array(final_shape_px) + gdf_list = [] + raster_layer = QgsProject.instance().mapLayer(self.params.input_layer_id) + raster_layer_crs = raster_layer.crs() if raster_layer else QgsCoordinateReferenceSystem("EPSG:3857") + + for tile_img_batched, tile_params_batched in self.tiles_generator_batched(): + if self.isCanceled(): + return MapProcessingResultCanceled() + + tile_result_batched = self._process_tile(tile_img_batched) + + for tile_result, tile_params in zip(tile_result_batched, tile_params_batched): + tile_params.set_mask_on_full_img( + tile_result=tile_result, + full_result_img=full_result_img) + try: + gdf_tile = self.tile_mask_to_gdf_latlon( + tile=tile_params, + mask=tile_result, + raster_layer_crs=raster_layer_crs, + ) + gdf_list.append(gdf_tile) + QgsMessageLog.logMessage(f"Tile {tile_params.x_bin_number},{tile_params.y_bin_number}: got {len(gdf_tile)} points", "Segmentation", 0) + except Exception as e: + QgsMessageLog.logMessage(f"Tile {tile_params.x_bin_number},{tile_params.y_bin_number} failed: {e}", "Segmentation", 2) + QgsMessageLog.logMessage(traceback.format_exc(), "Segmentation", 2) + + blur_size = int(self.segmentation_parameters.postprocessing_dilate_erode_size // 2) * 2 + 1 # needs to be odd + + for i in range(full_result_img.shape[0]): + full_result_img[i] = cv2.medianBlur(full_result_img[i], blur_size) + + full_result_img = self.limit_extended_extent_image_to_base_extent_with_mask(full_img=full_result_img) + + self.set_results_img(full_result_img) + if gdf_list: + final_gdf = gpd.GeoDataFrame(pd.concat(gdf_list, ignore_index=True), crs=gdf_list[0].crs) + csv_file = r"C:\Users\carin\Documents\segmen.csv" + final_gdf.to_csv(csv_file, index=False) + else: + final_gdf = None + print("No GeoDataFrame generated.") + + + gui_delegate = self._create_vlayer_from_mask_for_base_extent(self.get_result_img()) + + result_message = self._create_result_message(self.get_result_img()) + return MapProcessingResultSuccess( + message=result_message, + gui_delegate=gui_delegate, + ) + + def _check_output_layer_is_sigmoid_and_has_more_than_one_name(self, output_id: int) -> bool: + if self.model.outputs_names is None or self.model.outputs_are_sigmoid is None: + return False + + return len(self.model.outputs_names[output_id]) > 1 and self.model.outputs_are_sigmoid[output_id] + + def _create_result_message(self, result_img: np.ndarray) -> str: + + txt = f'Segmentation done, with the following statistics:\n' + + for output_id, layer_sizes in enumerate(self._get_indexes_of_model_output_channels_to_create()): + + txt += f'Channels for output {output_id}:\n' + + unique, counts = np.unique(result_img[output_id], return_counts=True) + counts_map = {} + for i in range(len(unique)): + counts_map[unique[i]] = counts[i] + + # # we cannot simply take image dimensions, because we may have irregular processing area from polygon + number_of_pixels_in_processing_area = np.sum([counts_map[k] for k in counts_map.keys()]) + total_area = number_of_pixels_in_processing_area * self.params.resolution_m_per_px**2 + + for channel_id in range(layer_sizes): + pixels_count = counts_map.get(channel_id + 1, 0) # we add 1 to avoid 0 values, find the MADD1 code for explanation + area = pixels_count * self.params.resolution_m_per_px**2 + + if total_area > 0 and not np.isnan(total_area) and not np.isinf(total_area): + area_percentage = area / total_area * 100 + else: + area_percentage = 0.0 + # TODO + + txt += f'\t- {self.model.get_channel_name(output_id, channel_id)}: area = {area:.2f} m^2 ({area_percentage:.2f} %)\n' + + return txt + + def _create_vlayer_from_mask_for_base_extent(self, mask_img, vector_gdf: gpd.GeoDataFrame = None) -> Callable: + """ create vector layer with polygons from the mask image + :return: function to be called in GUI thread + """ + vlayers = [] + + for output_id, layer_sizes in enumerate(self._get_indexes_of_model_output_channels_to_create()): + output_vlayers = [] + for channel_id in range(layer_sizes): + local_mask_img = np.uint8(mask_img[output_id] == (channel_id + 1)) # we add 1 to avoid 0 values, find the MADD1 code for explanation + + contours, hierarchy = cv2.findContours(local_mask_img, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours = processing_utils.transform_contours_yx_pixels_to_target_crs( + contours=contours, + extent=self.base_extent, + rlayer_units_per_pixel=self.rlayer_units_per_pixel) + features = [] + + if len(contours): + processing_utils.convert_cv_contours_to_features( + features=features, + cv_contours=contours, + hierarchy=hierarchy[0], + is_hole=False, + current_holes=[], + current_contour_index=0) + else: + pass # just nothing, we already have an empty list of features + + layer_name = self.model.get_channel_name(output_id, channel_id) + vlayer = QgsVectorLayer("multipolygon", layer_name, "memory") + vlayer.setCrs(self.rlayer.crs()) + prov = vlayer.dataProvider() + + color = vlayer.renderer().symbol().color() + OUTPUT_VLAYER_COLOR_TRANSPARENCY = 80 + color.setAlpha(OUTPUT_VLAYER_COLOR_TRANSPARENCY) + vlayer.renderer().symbol().setColor(color) + # TODO - add also outline for the layer (thicker black border) + + prov.addFeatures(features) + vlayer.updateExtents() + + output_vlayers.append(vlayer) + + vlayers.append(output_vlayers) + + # accessing GUI from non-GUI thread is not safe, so we need to delegate it to the GUI thread + def add_to_gui(): + group = QgsProject.instance().layerTreeRoot().insertGroup(0, 'model_output') + + if len(vlayers) == 1: + for vlayer in vlayers[0]: + QgsProject.instance().addMapLayer(vlayer, False) + group.addLayer(vlayer) + else: + for i, output_vlayers in enumerate(vlayers): + output_group = group.insertGroup(0, f'output_{i}') + for vlayer in output_vlayers: + QgsProject.instance().addMapLayer(vlayer, False) + output_group.addLayer(vlayer) + + return add_to_gui + + def _process_tile(self, tile_img_batched: np.ndarray) -> np.ndarray: + + res = self.model.process(tile_img_batched) + + # Thresholding optional (apply only on non-background) + threshold = self.segmentation_parameters.pixel_classification__probability_threshold + # onnx_classes = np.argmax(build, axis=1)[0].astype(np.int8) + # confidences = np.max(build, axis=1)[0] + + # onnx_classes[confidences < threshold] = 0 + # BUILDING_ID = 1 + + # final_mask = onnx_classes.copy() + + # final_mask[onnx_classes == BUILDING_ID] = type_build[onnx_classes == BUILDING_ID] + + # final_mask = np.expand_dims(final_mask, axis=(0, 1)) + final_mask = self.process_dual_batch(res, threshold) + + return final_mask + + def process_dual_batch(self, result, threshold=0.0): + + N = result[0].shape[0] + threshold = self.segmentation_parameters.pixel_classification__probability_threshold + + # Run the dual model + res = result[0] # (N, num_classes, H, W) + segmentation_maps = result[1] # list of N segmentation maps (H, W) + + # Prepare output batch + final_batch_masks = np.zeros((N, 1, 512, 512), dtype=np.int8) + individual_masks = [] + + new_class = {0: 0, 1: 5, 2: 6, 3: 7, 4: 8} + BUILDING_ID = 1 + + for i in range(N): + seg_map = segmentation_maps[i] # (H, W) + + # Map old classes to building classes + build_mask = np.zeros_like(seg_map, dtype=np.int8) + for k, v in new_class.items(): + build_mask[seg_map == k] = v + + # ONNX class predictions and confidences + onnx_classes = np.argmax(res[i], axis=0).astype(np.int8) # (H, W) + confidences = np.max(res[i], axis=0) # (H, W) + + # Threshold low-confidence predictions + onnx_classes[confidences < threshold] = 0 + + # Merge building predictions with type mask + final_mask = onnx_classes.copy() + final_mask[onnx_classes == BUILDING_ID] = build_mask[onnx_classes == BUILDING_ID] + + # Save results + + final_batch_masks[i, 0] = final_mask + + return final_batch_masks + + + # def _process_tile(self, tile_img_batched: np.ndarray) -> np.ndarray: + # many_result = self.model.process(tile_img_batched) + # many_outputs = [] + + # for result in many_result: + # result[result < self.segmentation_parameters.pixel_classification__probability_threshold] = 0.0 + + # if len(result.shape) == 3: + # result = np.expand_dims(result, axis=1) + + # if (result.shape[1] == 1): + # result = (result != 0).astype(int) + 1 # we add 1 to avoid 0 values, find the MADD1 code for explanation + # else: + # shape = result.shape + # result = np.argmax(result, axis=1).reshape(shape[0], 1, shape[2], shape[3]) + 1 # we add 1 to avoid 0 values, find the MADD1 code for explanation + + # assert len(result.shape) == 4 + # assert result.shape[1] == 1 + + # many_outputs.append(result[:, 0]) + + # many_outputs = np.array(many_outputs).transpose((1, 0, 2, 3)) + + # return many_outputs + + # def _process_tile(self, tile_img_batched: np.ndarray) -> np.ndarray: + + # merged_probs = self.model.process(tile_img_batched) + + # # Thresholding optional (apply only on non-background) + # threshold = self.segmentation_parameters.pixel_classification__probability_threshold + # merged_probs[:, 1:, :, :][merged_probs[:, 1:, :, :] < threshold] = 0.0 + + # # Argmax over channels to get single-channel mask + # single_channel_mask = np.argmax(merged_probs, axis=1).astype(np.uint8) # shape: (1, H, W) + + # # Remove 'other' class pixels + # single_channel_mask[single_channel_mask == 1] = 0 + # single_channel_mask = np.expand_dims(single_channel_mask, axis=1) + + # return single_channel_mask + + + diff --git a/zipdeepness/processing/map_processor/map_processor_superresolution.py b/zipdeepness/processing/map_processor/map_processor_superresolution.py new file mode 100644 index 0000000000000000000000000000000000000000..b1d0d3d5c77260589cc20ffb91d325449dbf050c --- /dev/null +++ b/zipdeepness/processing/map_processor/map_processor_superresolution.py @@ -0,0 +1,175 @@ +""" This file implements map processing for Super Resolution model """ + +import os +import uuid +from typing import List + +import numpy as np +from osgeo import gdal, osr +from qgis.core import QgsProject, QgsRasterLayer + +from deepness.common.misc import TMP_DIR_PATH +from deepness.common.processing_parameters.superresolution_parameters import SuperresolutionParameters +from deepness.processing.map_processor.map_processing_result import (MapProcessingResult, MapProcessingResultCanceled, + MapProcessingResultSuccess) +from deepness.processing.map_processor.map_processor_with_model import MapProcessorWithModel + + +class MapProcessorSuperresolution(MapProcessorWithModel): + """ + MapProcessor specialized for Super Resolution model (whic is is used to upscale the input image to a higher resolution) + """ + + def __init__(self, + params: SuperresolutionParameters, + **kwargs): + super().__init__( + params=params, + model=params.model, + **kwargs) + self.superresolution_parameters = params + self.model = params.model + + def _run(self) -> MapProcessingResult: + number_of_output_channels = self.model.get_number_of_output_channels() + + # always one output + number_of_output_channels = number_of_output_channels[0] + + final_shape_px = (int(self.img_size_y_pixels*self.superresolution_parameters.scale_factor), int(self.img_size_x_pixels*self.superresolution_parameters.scale_factor), number_of_output_channels) + + # NOTE: consider whether we can use float16/uint16 as datatype + full_result_imgs = self._get_array_or_mmapped_array(final_shape_px) + + for tile_img_batched, tile_params_batched in self.tiles_generator_batched(): + if self.isCanceled(): + return MapProcessingResultCanceled() + + tile_results_batched = self._process_tile(tile_img_batched) + + for tile_results, tile_params in zip(tile_results_batched, tile_params_batched): + full_result_imgs[int(tile_params.start_pixel_y*self.superresolution_parameters.scale_factor):int((tile_params.start_pixel_y+tile_params.stride_px)*self.superresolution_parameters.scale_factor), + int(tile_params.start_pixel_x*self.superresolution_parameters.scale_factor):int((tile_params.start_pixel_x+tile_params.stride_px)*self.superresolution_parameters.scale_factor), + :] = tile_results.transpose(1, 2, 0) # transpose to chanels last + + # plt.figure(); plt.imshow(full_result_img); plt.show(block=False); plt.pause(0.001) + full_result_imgs = self.limit_extended_extent_image_to_base_extent_with_mask(full_img=full_result_imgs) + self.set_results_img(full_result_imgs) + + gui_delegate = self._create_rlayers_from_images_for_base_extent(self.get_result_img()) + result_message = self._create_result_message(self.get_result_img()) + return MapProcessingResultSuccess( + message=result_message, + gui_delegate=gui_delegate, + ) + + def _create_result_message(self, result_img: List[np.ndarray]) -> str: + channels = self._get_indexes_of_model_output_channels_to_create() + txt = f'Super-resolution done \n' + + if len(channels) > 0: + total_area = result_img.shape[0] * result_img.shape[1] * (self.params.resolution_m_per_px / self.superresolution_parameters.scale_factor)**2 + txt += f'Total are is {total_area:.2f} m^2' + return txt + + def limit_extended_extent_image_to_base_extent_with_mask(self, full_img): + """ + Limit an image which is for extended_extent to the base_extent image. + If a limiting polygon was used for processing, it will be also applied. + :param full_img: + :return: + """ + # TODO look for some inplace operation to save memory + # cv2.copyTo(src=full_img, mask=area_mask_img, dst=full_img) # this doesn't work due to implementation details + # full_img = cv2.copyTo(src=full_img, mask=self.area_mask_img) + + b = self.base_extent_bbox_in_full_image + result_img = full_img[int(b.y_min*self.superresolution_parameters.scale_factor):int(b.y_max*self.superresolution_parameters.scale_factor), + int(b.x_min*self.superresolution_parameters.scale_factor):int(b.x_max*self.superresolution_parameters.scale_factor), + :] + return result_img + + def load_rlayer_from_file(self, file_path): + """ + Create raster layer from tif file + """ + file_name = os.path.basename(file_path) + base_file_name = file_name.split('___')[0] # we remove the random_id string we created a moment ago + rlayer = QgsRasterLayer(file_path, base_file_name) + if rlayer.width() == 0: + raise Exception("0 width - rlayer not loaded properly. Probably invalid file path?") + rlayer.setCrs(self.rlayer.crs()) + return rlayer + + def _create_rlayers_from_images_for_base_extent(self, result_imgs: List[np.ndarray]): + # TODO: We are creating a new file for each layer. + # Maybe can we pass ownership of this file to QGis? + # Or maybe even create vlayer directly from array, without a file? + rlayers = [] + + for i, channel_id in enumerate(['Super Resolution']): + result_img = result_imgs + random_id = str(uuid.uuid4()).replace('-', '') + file_path = os.path.join(TMP_DIR_PATH, f'{channel_id}___{random_id}.tif') + self.save_result_img_as_tif(file_path=file_path, img=result_img) + + rlayer = self.load_rlayer_from_file(file_path) + OUTPUT_RLAYER_OPACITY = 0.5 + rlayer.renderer().setOpacity(OUTPUT_RLAYER_OPACITY) + rlayers.append(rlayer) + + def add_to_gui(): + group = QgsProject.instance().layerTreeRoot().insertGroup(0, 'Super Resolution Results') + for rlayer in rlayers: + QgsProject.instance().addMapLayer(rlayer, False) + group.addLayer(rlayer) + + return add_to_gui + + def save_result_img_as_tif(self, file_path: str, img: np.ndarray): + """ + As we cannot pass easily an numpy array to be displayed as raster layer, we create temporary geotif files, + which will be loaded as layer later on + + Partially based on example from: + https://gis.stackexchange.com/questions/82031/gdal-python-set-projection-of-a-raster-not-working + """ + os.makedirs(os.path.dirname(file_path), exist_ok=True) + + extent = self.base_extent + crs = self.rlayer.crs() + + geo_transform = [extent.xMinimum(), self.rlayer_units_per_pixel/self.superresolution_parameters.scale_factor, 0, + extent.yMaximum(), 0, -self.rlayer_units_per_pixel/self.superresolution_parameters.scale_factor] + + driver = gdal.GetDriverByName('GTiff') + n_lines = img.shape[0] + n_cols = img.shape[1] + n_chanels = img.shape[2] + # data_type = gdal.GDT_Byte + data_type = gdal.GDT_Float32 + grid_data = driver.Create('grid_data', n_cols, n_lines, n_chanels, data_type) # , options) + # loop over chanels + for i in range(1, img.shape[2]+1): + grid_data.GetRasterBand(i).WriteArray(img[:, :, i-1]) + + # crs().srsid() - maybe we can use the ID directly - but how? + # srs.ImportFromEPSG() + srs = osr.SpatialReference() + srs.SetFromUserInput(crs.authid()) + + grid_data.SetProjection(srs.ExportToWkt()) + grid_data.SetGeoTransform(geo_transform) + driver.CreateCopy(file_path, grid_data, 0) + print(f'***** {file_path = }') + + def _process_tile(self, tile_img: np.ndarray) -> np.ndarray: + result = self.model.process(tile_img) + result[np.isnan(result)] = 0 + result *= self.superresolution_parameters.output_scaling + + # NOTE - currently we are saving result as float32, so we are losing some accuraccy. + # result = np.clip(result, 0, 255) # old version with uint8_t - not used anymore + result = result.astype(np.float32) + + return result diff --git a/zipdeepness/processing/map_processor/map_processor_training_data_export.py b/zipdeepness/processing/map_processor/map_processor_training_data_export.py new file mode 100644 index 0000000000000000000000000000000000000000..a4996ee3834fc29da3c74bd617930f63e9b7bb79 --- /dev/null +++ b/zipdeepness/processing/map_processor/map_processor_training_data_export.py @@ -0,0 +1,95 @@ +""" This file implements map processing for the Training Data Export Tool """ + +import datetime +import os + +import numpy as np +from qgis.core import QgsProject + +from deepness.common.lazy_package_loader import LazyPackageLoader +from deepness.common.processing_parameters.training_data_export_parameters import TrainingDataExportParameters +from deepness.processing import processing_utils +from deepness.processing.map_processor.map_processing_result import (MapProcessingResultCanceled, + MapProcessingResultSuccess) +from deepness.processing.map_processor.map_processor import MapProcessor +from deepness.processing.tile_params import TileParams + +cv2 = LazyPackageLoader('cv2') + + +class MapProcessorTrainingDataExport(MapProcessor): + """ + Map Processor specialized in exporting training data, not doing any prediction with model. + Exports tiles for the ortophoto and a mask layer. + """ + + def __init__(self, + params: TrainingDataExportParameters, + **kwargs): + super().__init__( + params=params, + **kwargs) + self.params = params + self.output_dir_path = self._create_output_dir() + + def _create_output_dir(self) -> str: + datetime_string = datetime.datetime.now().strftime("%d%m%Y_%H%M%S") + full_path = os.path.join(self.params.output_directory_path, datetime_string) + os.makedirs(full_path, exist_ok=True) + return full_path + + def _run(self): + export_segmentation_mask = self.params.segmentation_mask_layer_id is not None + if export_segmentation_mask: + vlayer_segmentation = QgsProject.instance().mapLayers()[self.params.segmentation_mask_layer_id] + vlayer_segmentation.setCrs(self.rlayer.crs()) + segmentation_mask_full = processing_utils.create_area_mask_image( + rlayer=self.rlayer, + vlayer_mask=vlayer_segmentation, + extended_extent=self.extended_extent, + rlayer_units_per_pixel=self.rlayer_units_per_pixel, + image_shape_yx=(self.img_size_y_pixels, self.img_size_x_pixels), + files_handler=self.file_handler) + + segmentation_mask_full = segmentation_mask_full[np.newaxis, ...] + + number_of_written_tiles = 0 + for tile_img, tile_params in self.tiles_generator(): + if self.isCanceled(): + return MapProcessingResultCanceled() + + tile_params = tile_params # type: TileParams + + if self.params.export_image_tiles: + file_name = f'tile_img_{tile_params.x_bin_number}_{tile_params.y_bin_number}.png' + file_path = os.path.join(self.output_dir_path, file_name) + + if tile_img.dtype in [np.uint32, np.int32]: + print(f'Exporting image with data type {tile_img.dtype} is not supported. Trimming to uint16. Consider changing the data type in the source image.') + tile_img = tile_img.astype(np.uint16) + + if tile_img.shape[-1] == 4: + tile_img = cv2.cvtColor(tile_img, cv2.COLOR_RGBA2BGRA) + elif tile_img.shape[-1] == 3: + tile_img = cv2.cvtColor(tile_img, cv2.COLOR_RGB2BGR) + + cv2.imwrite(file_path, tile_img) + number_of_written_tiles += 1 + + if export_segmentation_mask: + segmentation_mask_for_tile = tile_params.get_entire_tile_from_full_img(segmentation_mask_full) + + file_name = f'tile_mask_{tile_params.x_bin_number}_{tile_params.y_bin_number}.png' + file_path = os.path.join(self.output_dir_path, file_name) + + cv2.imwrite(file_path, segmentation_mask_for_tile[0]) + + result_message = self._create_result_message(number_of_written_tiles) + return MapProcessingResultSuccess(result_message) + + def _create_result_message(self, number_of_written_tiles) -> str: + total_area = self.img_size_x_pixels * self.img_size_y_pixels * self.params.resolution_m_per_px**2 + return f'Exporting data finished!\n' \ + f'Exported {number_of_written_tiles} tiles.\n' \ + f'Total processed area: {total_area:.2f} m^2\n' \ + f'Directory: "{self.output_dir_path}"' diff --git a/zipdeepness/processing/map_processor/map_processor_with_model.py b/zipdeepness/processing/map_processor/map_processor_with_model.py new file mode 100644 index 0000000000000000000000000000000000000000..f56a319f5ae98df720c56955508317525fd0c503 --- /dev/null +++ b/zipdeepness/processing/map_processor/map_processor_with_model.py @@ -0,0 +1,27 @@ + +""" This file implements map processing functions common for all map processors using nural model """ + +from typing import List + +from deepness.processing.map_processor.map_processor import MapProcessor +from deepness.processing.models.model_base import ModelBase + + +class MapProcessorWithModel(MapProcessor): + """ + Common base class for MapProcessor with models + """ + + def __init__(self, + model: ModelBase, + **kwargs): + super().__init__( + **kwargs) + self.model = model + + def _get_indexes_of_model_output_channels_to_create(self) -> List[int]: + """ + Decide what model output channels/classes we want to use at presentation level + (e.g. for which channels create a layer with results) + """ + return self.model.get_number_of_output_channels() diff --git a/zipdeepness/processing/map_processor/utils/ckdtree.py b/zipdeepness/processing/map_processor/utils/ckdtree.py new file mode 100644 index 0000000000000000000000000000000000000000..3577a21159b2a7c2b75080625661e4647ef1c210 --- /dev/null +++ b/zipdeepness/processing/map_processor/utils/ckdtree.py @@ -0,0 +1,62 @@ +import heapq + +import numpy as np + + +class cKDTree: + def __init__(self, data): + self.data = np.asarray(data) + self.tree = self._build_kdtree(np.arange(len(data))) + + def _build_kdtree(self, indices, depth=0): + if len(indices) == 0: + return None + axis = depth % self.data.shape[1] # alternate between x and y dimensions + + sorted_indices = indices[np.argsort(self.data[indices, axis])] + mid = len(sorted_indices) // 2 + node = { + 'index': sorted_indices[mid], + 'left': self._build_kdtree(sorted_indices[:mid], depth + 1), + 'right': self._build_kdtree(sorted_indices[mid + 1:], depth + 1) + } + return node + + def query(self, point: np.ndarray, k: int): + if type(point) is not np.ndarray: + point = np.array(point) + + return [index for _, index in self._query(point, k, self.tree)] + + def _query(self, point, k, node, depth=0, best_indices=None, best_distances=None): + if node is None: + return None + + axis = depth % self.data.shape[1] + + if point[axis] < self.data[node['index']][axis]: + next_node = node['left'] + other_node = node['right'] + else: + next_node = node['right'] + other_node = node['left'] + + if best_indices is None: + best_indices = [] + best_distances = [] + + current_distance = np.linalg.norm(self.data[node['index']] - point) + + if len(best_indices) < k: + heapq.heappush(best_indices, (-current_distance, node['index'])) + elif current_distance < -best_indices[0][0]: + heapq.heappop(best_indices) + heapq.heappush(best_indices, (-current_distance, node['index'])) + + if point[axis] < self.data[node['index']][axis] or len(best_indices) < k or abs(point[axis] - self.data[node['index']][axis]) < -best_indices[0][0]: + self._query(point, k, next_node, depth + 1, best_indices, best_distances) + + if point[axis] >= self.data[node['index']][axis] or len(best_indices) < k or abs(point[axis] - self.data[node['index']][axis]) < -best_indices[0][0]: + self._query(point, k, other_node, depth + 1, best_indices, best_distances) + + return best_indices diff --git a/zipdeepness/processing/models/__init__.py b/zipdeepness/processing/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d080009a194567252f67421a85ea8ac478e3b030 --- /dev/null +++ b/zipdeepness/processing/models/__init__.py @@ -0,0 +1,2 @@ +""" Module including classes implemetations for the deep learning inference and related functions +""" \ No newline at end of file diff --git a/zipdeepness/processing/models/buildings_type_MA.onnx b/zipdeepness/processing/models/buildings_type_MA.onnx new file mode 100644 index 0000000000000000000000000000000000000000..d4c0d930a26ece468401dfd85f7350a4a26b92f0 --- /dev/null +++ b/zipdeepness/processing/models/buildings_type_MA.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e7c06401e20f6791e272f5ec694d4ae285c79ed6ceb9213875972114869b90f +size 464134848 diff --git a/zipdeepness/processing/models/detector.py b/zipdeepness/processing/models/detector.py new file mode 100644 index 0000000000000000000000000000000000000000..474802d5ca4a5b5ad070456ade469a35fbfb5eb3 --- /dev/null +++ b/zipdeepness/processing/models/detector.py @@ -0,0 +1,709 @@ +""" Module including the class for the object detection task and related functions +""" +from dataclasses import dataclass +from typing import List, Optional, Tuple +from qgis.core import Qgis, QgsGeometry, QgsRectangle, QgsPointXY + +import cv2 +import numpy as np + +from deepness.common.processing_parameters.detection_parameters import DetectorType +from deepness.processing.models.model_base import ModelBase +from deepness.processing.processing_utils import BoundingBox + + +@dataclass +class Detection: + """Class that represents single detection result in object detection model + + Parameters + ---------- + bbox : BoundingBox + bounding box describing the detection rectangle + conf : float + confidence of the detection + clss : int + class of the detected object + """ + + bbox: BoundingBox + """BoundingBox: bounding box describing the detection rectangle""" + conf: float + """float: confidence of the detection""" + clss: int + """int: class of the detected object""" + mask: Optional[np.ndarray] = None + """np.ndarray: mask of the detected object""" + mask_offsets: Optional[Tuple[int, int]] = None + """Tuple[int, int]: offsets of the mask""" + + def convert_to_global(self, offset_x: int, offset_y: int): + """Apply (x,y) offset to bounding box coordinates + + Parameters + ---------- + offset_x : int + _description_ + offset_y : int + _description_ + """ + self.bbox.apply_offset(offset_x=offset_x, offset_y=offset_y) + + if self.mask is not None: + self.mask_offsets = (offset_x, offset_y) + + def get_bbox_xyxy(self) -> np.ndarray: + """Convert stored bounding box into x1y1x2y2 format + + Returns + ------- + np.ndarray + Array in (x1, y1, x2, y2) format + """ + return self.bbox.get_xyxy() + + def get_bbox_xyxy_rot(self) -> np.ndarray: + """Convert stored bounding box into x1y1x2y2r format + + Returns + ------- + np.ndarray + Array in (x1, y1, x2, y2, r) format + """ + return self.bbox.get_xyxy_rot() + + def get_bbox_center(self) -> Tuple[int, int]: + """Get center of the bounding box + + Returns + ------- + Tuple[int, int] + Center of the bounding box + """ + return self.bbox.get_center() + + def __lt__(self, other): + return self.bbox.get_area() < other.bbox.get_area() + + +class Detector(ModelBase): + """Class implements object detection features + + Detector model is used for detection of objects in images. It is based on YOLOv5/YOLOv7 models style. + """ + + def __init__(self, model_file_path: str): + """Initialize object detection model + + Parameters + ---------- + model_file_path : str + Path to model file""" + super(Detector, self).__init__(model_file_path) + + self.confidence = None + """float: Confidence threshold""" + self.iou_threshold = None + """float: IoU threshold""" + self.model_type: Optional[DetectorType] = None + """DetectorType: Model type""" + + def set_inference_params(self, confidence: float, iou_threshold: float): + """Set inference parameters + + Parameters + ---------- + confidence : float + Confidence threshold + iou_threshold : float + IoU threshold + """ + self.confidence = confidence + self.iou_threshold = iou_threshold + + def set_model_type_param(self, model_type: DetectorType): + """Set model type parameters + + Parameters + ---------- + model_type : str + Model type + """ + self.model_type = model_type + + @classmethod + def get_class_display_name(cls): + """Get class display name + + Returns + ------- + str + Class display name""" + return cls.__name__ + + def get_number_of_output_channels(self): + """Get number of output channels + + Returns + ------- + int + Number of output channels + """ + class_names = self.get_outputs_channel_names()[0] + if class_names is not None: + return [len(class_names)] # If class names are specified, we expect to have exactly this number of channels as specidied + + model_type_params = self.model_type.get_parameters() + + shape_index = -2 if model_type_params.has_inverted_output_shape else -1 + + if len(self.outputs_layers) == 1: + # YOLO_ULTRALYTICS_OBB + if self.model_type == DetectorType.YOLO_ULTRALYTICS_OBB: + return [self.outputs_layers[0].shape[shape_index] - 4 - 1] + + elif model_type_params.skipped_objectness_probability: + return [self.outputs_layers[0].shape[shape_index] - 4] + + return [self.outputs_layers[0].shape[shape_index] - 4 - 1] # shape - 4 bboxes - 1 conf + + # YOLO_ULTRALYTICS_SEGMENTATION + elif len(self.outputs_layers) == 2 and self.model_type == DetectorType.YOLO_ULTRALYTICS_SEGMENTATION: + return [self.outputs_layers[0].shape[shape_index] - 4 - self.outputs_layers[1].shape[1]] + + else: + raise NotImplementedError("Model with multiple output layer is not supported! Use only one output layer.") + + def postprocessing(self, model_output): + """Postprocess model output + + NOTE: Maybe refactor this, as it has many added layers of checks which can be simplified. + + Parameters + ---------- + model_output : list + Model output + + Returns + ------- + list + Batch of lists of detections + """ + if self.confidence is None or self.iou_threshold is None: + return Exception( + "Confidence or IOU threshold is not set for model. Use self.set_inference_params" + ) + + if self.model_type is None: + return Exception( + "Model type is not set for model. Use self.set_model_type_param" + ) + + batch_detection = [] + outputs_range = len(model_output) + + if self.model_type == DetectorType.YOLO_ULTRALYTICS_SEGMENTATION or self.model_type == DetectorType.YOLO_v9: + outputs_range = len(model_output[0]) + + for i in range(outputs_range): + masks = None + rots = None + detections = [] + + if self.model_type == DetectorType.YOLO_v5_v7_DEFAULT: + boxes, conf, classes = self._postprocessing_YOLO_v5_v7_DEFAULT(model_output[0][i]) + elif self.model_type == DetectorType.YOLO_v6: + boxes, conf, classes = self._postprocessing_YOLO_v6(model_output[0][i]) + elif self.model_type == DetectorType.YOLO_v9: + boxes, conf, classes = self._postprocessing_YOLO_v9(model_output[0][i]) + elif self.model_type == DetectorType.YOLO_ULTRALYTICS: + boxes, conf, classes = self._postprocessing_YOLO_ULTRALYTICS(model_output[0][i]) + elif self.model_type == DetectorType.YOLO_ULTRALYTICS_SEGMENTATION: + boxes, conf, classes, masks = self._postprocessing_YOLO_ULTRALYTICS_SEGMENTATION(model_output[0][i], model_output[1][i]) + elif self.model_type == DetectorType.YOLO_ULTRALYTICS_OBB: + boxes, conf, classes, rots = self._postprocessing_YOLO_ULTRALYTICS_OBB(model_output[0][i]) + else: + raise NotImplementedError(f"Model type not implemented! ('{self.model_type}')") + + masks = masks if masks is not None else [None] * len(boxes) + rots = rots if rots is not None else [0.0] * len(boxes) + + for b, c, cl, m, r in zip(boxes, conf, classes, masks, rots): + det = Detection( + bbox=BoundingBox( + x_min=b[0], + x_max=b[2], + y_min=b[1], + y_max=b[3], + rot=r), + conf=c, + clss=cl, + mask=m, + ) + detections.append(det) + + batch_detection.append(detections) + + return batch_detection + + def _postprocessing_YOLO_v5_v7_DEFAULT(self, model_output): + outputs_filtered = np.array( + list(filter(lambda x: x[4] >= self.confidence, model_output)) + ) + + if len(outputs_filtered.shape) < 2: + return [], [], [] + + probabilities = outputs_filtered[:, 4] + + outputs_x1y1x2y2 = self.xywh2xyxy(outputs_filtered) + + pick_indxs = self.non_max_suppression_fast( + outputs_x1y1x2y2, + probs=probabilities, + iou_threshold=self.iou_threshold) + + outputs_nms = outputs_x1y1x2y2[pick_indxs] + + boxes = np.array(outputs_nms[:, :4], dtype=int) + conf = outputs_nms[:, 4] + classes = np.argmax(outputs_nms[:, 5:], axis=1) + + return boxes, conf, classes + + def _postprocessing_YOLO_v6(self, model_output): + outputs_filtered = np.array( + list(filter(lambda x: np.max(x[5:]) >= self.confidence, model_output)) + ) + + if len(outputs_filtered.shape) < 2: + return [], [], [] + + probabilities = outputs_filtered[:, 4] + + outputs_x1y1x2y2 = self.xywh2xyxy(outputs_filtered) + + pick_indxs = self.non_max_suppression_fast( + outputs_x1y1x2y2, + probs=probabilities, + iou_threshold=self.iou_threshold) + + outputs_nms = outputs_x1y1x2y2[pick_indxs] + + boxes = np.array(outputs_nms[:, :4], dtype=int) + conf = np.max(outputs_nms[:, 5:], axis=1) + classes = np.argmax(outputs_nms[:, 5:], axis=1) + + return boxes, conf, classes + + def _postprocessing_YOLO_v9(self, model_output): + model_output = np.transpose(model_output, (1, 0)) + + outputs_filtered = np.array( + list(filter(lambda x: np.max(x[4:]) >= self.confidence, model_output)) + ) + + if len(outputs_filtered.shape) < 2: + return [], [], [] + + probabilities = np.max(outputs_filtered[:, 4:], axis=1) + + outputs_x1y1x2y2 = self.xywh2xyxy(outputs_filtered) + + pick_indxs = self.non_max_suppression_fast( + outputs_x1y1x2y2, + probs=probabilities, + iou_threshold=self.iou_threshold) + + outputs_nms = outputs_x1y1x2y2[pick_indxs] + + boxes = np.array(outputs_nms[:, :4], dtype=int) + conf = np.max(outputs_nms[:, 4:], axis=1) + classes = np.argmax(outputs_nms[:, 4:], axis=1) + + return boxes, conf, classes + + def _postprocessing_YOLO_ULTRALYTICS(self, model_output): + model_output = np.transpose(model_output, (1, 0)) + + outputs_filtered = np.array( + list(filter(lambda x: np.max(x[4:]) >= self.confidence, model_output)) + ) + + if len(outputs_filtered.shape) < 2: + return [], [], [] + + probabilities = np.max(outputs_filtered[:, 4:], axis=1) + + outputs_x1y1x2y2 = self.xywh2xyxy(outputs_filtered) + + pick_indxs = self.non_max_suppression_fast( + outputs_x1y1x2y2, + probs=probabilities, + iou_threshold=self.iou_threshold) + + outputs_nms = outputs_x1y1x2y2[pick_indxs] + + boxes = np.array(outputs_nms[:, :4], dtype=int) + conf = np.max(outputs_nms[:, 4:], axis=1) + classes = np.argmax(outputs_nms[:, 4:], axis=1) + + return boxes, conf, classes + + def _postprocessing_YOLO_ULTRALYTICS_SEGMENTATION(self, detections, protos): + detections = np.transpose(detections, (1, 0)) + + number_of_class = self.get_number_of_output_channels()[0] + mask_start_index = 4 + number_of_class + + outputs_filtered = np.array( + list(filter(lambda x: np.max(x[4:4+number_of_class]) >= self.confidence, detections)) + ) + + if len(outputs_filtered.shape) < 2: + return [], [], [], [] + + probabilities = np.max(outputs_filtered[:, 4:4+number_of_class], axis=1) + + outputs_x1y1x2y2 = self.xywh2xyxy(outputs_filtered) + + pick_indxs = self.non_max_suppression_fast( + outputs_x1y1x2y2, + probs=probabilities, + iou_threshold=self.iou_threshold) + + outputs_nms = outputs_x1y1x2y2[pick_indxs] + + boxes = np.array(outputs_nms[:, :4], dtype=int) + conf = np.max(outputs_nms[:, 4:4+number_of_class], axis=1) + classes = np.argmax(outputs_nms[:, 4:4+number_of_class], axis=1) + masks_in = np.array(outputs_nms[:, mask_start_index:], dtype=float) + + masks = self.process_mask(protos, masks_in, boxes) + + return boxes, conf, classes, masks + + def _postprocessing_YOLO_ULTRALYTICS_OBB(self, model_output): + model_output = np.transpose(model_output, (1, 0)) + + outputs_filtered = np.array( + list(filter(lambda x: np.max(x[4:-1]) >= self.confidence, model_output)) + ) + + if len(outputs_filtered.shape) < 2: + return [], [], [], [] + + probabilities = np.max(outputs_filtered[:, 4:-1], axis=1) + rotations = outputs_filtered[:, -1] + + outputs_x1y1x2y2_rot = self.xywhr2xyxyr(outputs_filtered, rotations) + + pick_indxs = self.non_max_suppression_fast( + outputs_x1y1x2y2_rot, + probs=probabilities, + iou_threshold=self.iou_threshold, + with_rot=True) + + outputs_nms = outputs_x1y1x2y2_rot[pick_indxs] + + boxes = np.array(outputs_nms[:, :4], dtype=int) + conf = np.max(outputs_nms[:, 4:-1], axis=1) + classes = np.argmax(outputs_nms[:, 4:-1], axis=1) + rots = outputs_nms[:, -1] + + return boxes, conf, classes, rots + + # based on https://github.com/ultralytics/ultralytics/blob/main/ultralytics/utils/ops.py#L638C1-L638C67 + def process_mask(self, protos, masks_in, bboxes): + c, mh, mw = protos.shape # CHW + ih, iw = self.input_shape[2:] + + masks = self.sigmoid(np.matmul(masks_in, protos.astype(float).reshape(c, -1))).reshape(-1, mh, mw) + + downsampled_bboxes = bboxes.copy().astype(float) + downsampled_bboxes[:, 0] *= mw / iw + downsampled_bboxes[:, 2] *= mw / iw + downsampled_bboxes[:, 3] *= mh / ih + downsampled_bboxes[:, 1] *= mh / ih + + masks = self.crop_mask(masks, downsampled_bboxes) + scaled_masks = np.zeros((len(masks), ih, iw)) + + for i in range(len(masks)): + scaled_masks[i] = cv2.resize(masks[i], (iw, ih), interpolation=cv2.INTER_LINEAR) + + masks = np.uint8(scaled_masks >= 0.5) + + return masks + + @staticmethod + def sigmoid(x): + return 1 / (1 + np.exp(-x)) + + @staticmethod + # based on https://github.com/ultralytics/ultralytics/blob/main/ultralytics/utils/ops.py#L598C1-L614C65 + def crop_mask(masks, boxes): + n, h, w = masks.shape + x1, y1, x2, y2 = np.split(boxes[:, :, None], 4, axis=1) # x1 shape(n,1,1) + r = np.arange(w, dtype=x1.dtype)[None, None, :] # rows shape(1,1,w) + c = np.arange(h, dtype=x1.dtype)[None, :, None] # cols shape(1,h,1) + + return masks * ((r >= x1) * (r < x2) * (c >= y1) * (c < y2)) + + @staticmethod + def xywh2xyxy(x: np.ndarray) -> np.ndarray: + """Convert bounding box from (x,y,w,h) to (x1,y1,x2,y2) format + + Parameters + ---------- + x : np.ndarray + Bounding box in (x,y,w,h) format with classes' probabilities + + Returns + ------- + np.ndarray + Bounding box in (x1,y1,x2,y2) format + """ + y = np.copy(x) + y[:, 0] = x[:, 0] - x[:, 2] / 2 # top left x + y[:, 1] = x[:, 1] - x[:, 3] / 2 # top left y + y[:, 2] = x[:, 0] + x[:, 2] / 2 # bottom right x + y[:, 3] = x[:, 1] + x[:, 3] / 2 # bottom right y + return y + + @staticmethod + def xywhr2xyxyr(bbox: np.ndarray, rot: np.ndarray) -> np.ndarray: + """Convert bounding box from (x,y,w,h,r) to (x1,y1,x2,y2,r) format, keeping rotated boxes in range [0, pi/2] + + Parameters + ---------- + bbox : np.ndarray + Bounding box in (x,y,w,h) format with classes' probabilities and rotations + + Returns + ------- + np.ndarray + Bounding box in (x1,y1,x2,y2,r) format, keep classes' probabilities + """ + x, y, w, h = bbox[:, 0], bbox[:, 1], bbox[:, 2], bbox[:, 3] + + w_ = np.where(w > h, w, h) + h_ = np.where(w > h, h, w) + r_ = np.where(w > h, rot, rot + np.pi / 2) % np.pi + + new_bbox_xywh = np.stack([x, y, w_, h_], axis=1) + new_bbox_xyxy = Detector.xywh2xyxy(new_bbox_xywh) + + return np.concatenate([new_bbox_xyxy, bbox[:, 4:-1], r_[:, None]], axis=1) + + + @staticmethod + def non_max_suppression_fast(boxes: np.ndarray, probs: np.ndarray, iou_threshold: float, with_rot: bool = False) -> List: + """Apply non-maximum suppression to bounding boxes + + Based on: + https://github.com/amusi/Non-Maximum-Suppression/blob/master/nms.py + + Parameters + ---------- + boxes : np.ndarray + Bounding boxes in (x1,y1,x2,y2) format or (x1,y1,x2,y2,r) format if with_rot is True + probs : np.ndarray + Confidence scores + iou_threshold : float + IoU threshold + with_rot: bool + If True, use rotated IoU + + Returns + ------- + List + List of indexes of bounding boxes to keep + """ + # If no bounding boxes, return empty list + if len(boxes) == 0: + return [] + + # Bounding boxes + boxes = np.array(boxes) + + # coordinates of bounding boxes + start_x = boxes[:, 0] + start_y = boxes[:, 1] + end_x = boxes[:, 2] + end_y = boxes[:, 3] + + if with_rot: + # Rotations of bounding boxes + rotations = boxes[:, 4] + + # Confidence scores of bounding boxes + score = np.array(probs) + + # Picked bounding boxes + picked_boxes = [] + + # Compute areas of bounding boxes + areas = (end_x - start_x + 1) * (end_y - start_y + 1) + + # Sort by confidence score of bounding boxes + order = np.argsort(score) + + # Iterate bounding boxes + while order.size > 0: + # The index of largest confidence score + index = order[-1] + + # Pick the bounding box with largest confidence score + picked_boxes.append(index) + + if not with_rot: + ratio = Detector.compute_iou(index, order, start_x, start_y, end_x, end_y, areas) + else: + ratio = Detector.compute_rotated_iou(index, order, start_x, start_y, end_x, end_y, rotations, areas) + + left = np.where(ratio < iou_threshold) + order = order[left] + + return picked_boxes + + @staticmethod + def compute_iou(index: int, order: np.ndarray, start_x: np.ndarray, start_y: np.ndarray, end_x: np.ndarray, end_y: np.ndarray, areas: np.ndarray) -> np.ndarray: + """Compute IoU for bounding boxes + + Parameters + ---------- + index : int + Index of the bounding box + order : np.ndarray + Order of bounding boxes + start_x : np.ndarray + Start x coordinate of bounding boxes + start_y : np.ndarray + Start y coordinate of bounding boxes + end_x : np.ndarray + End x coordinate of bounding boxes + end_y : np.ndarray + End y coordinate of bounding boxes + areas : np.ndarray + Areas of bounding boxes + + Returns + ------- + np.ndarray + IoU values + """ + + # Compute ordinates of intersection-over-union(IOU) + x1 = np.maximum(start_x[index], start_x[order[:-1]]) + x2 = np.minimum(end_x[index], end_x[order[:-1]]) + y1 = np.maximum(start_y[index], start_y[order[:-1]]) + y2 = np.minimum(end_y[index], end_y[order[:-1]]) + + # Compute areas of intersection-over-union + w = np.maximum(0.0, x2 - x1 + 1) + h = np.maximum(0.0, y2 - y1 + 1) + intersection = w * h + + # Compute the ratio between intersection and union + return intersection / (areas[index] + areas[order[:-1]] - intersection) + + + @staticmethod + def compute_rotated_iou(index: int, order: np.ndarray, start_x: np.ndarray, start_y: np.ndarray, end_x: np.ndarray, end_y: np.ndarray, rotations: np.ndarray, areas: np.ndarray) -> np.ndarray: + """Compute IoU for rotated bounding boxes + + Parameters + ---------- + index : int + Index of the bounding box + order : np.ndarray + Order of bounding boxes + start_x : np.ndarray + Start x coordinate of bounding boxes + start_y : np.ndarray + Start y coordinate of bounding boxes + end_x : np.ndarray + End x coordinate of bounding boxes + end_y : np.ndarray + End y coordinate of bounding boxes + rotations : np.ndarray + Rotations of bounding boxes (in radians, around the center) + areas : np.ndarray + Areas of bounding boxes + + Returns + ------- + np.ndarray + IoU values + """ + + def create_rotated_geom(x1, y1, x2, y2, rotation): + """Helper function to create a rotated QgsGeometry rectangle""" + # Define the corners of the box before rotation + center_x = (x1 + x2) / 2 + center_y = (y1 + y2) / 2 + + # Create a rectangle using QgsRectangle + rect = QgsRectangle(QgsPointXY(x1, y1), QgsPointXY(x2, y2)) + + # Convert to QgsGeometry + geom = QgsGeometry.fromRect(rect) + + # Rotate the geometry around its center + result = geom.rotate(np.degrees(rotation), QgsPointXY(center_x, center_y)) + + if result == Qgis.GeometryOperationResult.Success: + return geom + else: + return QgsGeometry() + + # Create the rotated geometry for the current bounding box + geom1 = create_rotated_geom(start_x[index], start_y[index], end_x[index], end_y[index], rotations[index]) + + iou_values = [] + + # Iterate over the rest of the boxes in order and calculate IoU + for i in range(len(order) - 1): + # Create the rotated geometry for the other boxes in the order + geom2 = create_rotated_geom(start_x[order[i]], start_y[order[i]], end_x[order[i]], end_y[order[i]], rotations[order[i]]) + + # Compute the intersection geometry + intersection_geom = geom1.intersection(geom2) + + # Check if intersection is empty + if intersection_geom.isEmpty(): + intersection_area = 0.0 + else: + # Compute the intersection area + intersection_area = intersection_geom.area() + + # Compute the union area + union_area = areas[index] + areas[order[i]] - intersection_area + + # Compute IoU + iou = intersection_area / union_area if union_area > 0 else 0.0 + iou_values.append(iou) + + return np.array(iou_values) + + + def check_loaded_model_outputs(self): + """Check if model outputs are valid. + Valid model are: + - has 1 or 2 outputs layer + - output layer shape length is 3 + - batch size is 1 + """ + + if len(self.outputs_layers) == 1 or len(self.outputs_layers) == 2: + shape = self.outputs_layers[0].shape + + if len(shape) != 3: + raise Exception( + f"Detection model output should have 3 dimensions: (Batch_size, detections, values). " + f"Actually has: {shape}" + ) + + else: + raise NotImplementedError("Model with multiple output layer is not supported! Use only one output layer.") diff --git a/zipdeepness/processing/models/dual.py b/zipdeepness/processing/models/dual.py new file mode 100644 index 0000000000000000000000000000000000000000..dff77f85a0ea2ba8f62bd9a74fcb1f4f1ac5f4dd --- /dev/null +++ b/zipdeepness/processing/models/dual.py @@ -0,0 +1,168 @@ +from deepness.processing.models.segmentor import Segmentor +import numpy as np +import cv2 +import onnxruntime as ort +import os +from typing import List + +class DualModel(Segmentor): + + def __init__(self, model_file_path: str): + super().__init__(model_file_path) + current_dir = os.path.dirname(os.path.abspath(__file__)) + # Both models are in the same folder + self.second_model_file_path = os.path.join(current_dir, "buildings_type_MA.onnx") + self.second_model = ort.InferenceSession(self.second_model_file_path, providers=["CUDAExecutionProvider", "CPUExecutionProvider"]) + + + + + def preprocess_tiles_ade20k(self,tiles_batched: np.ndarray) -> np.ndarray: + # ADE20K mean/std in 0-1 range + ADE_MEAN = np.array([123.675, 116.280, 103.530]) / 255.0 + ADE_STD = np.array([58.395, 57.120, 57.375]) / 255.0 + + tiles = tiles_batched.astype(np.float32) / 255.0 # 0-1 + + # Standardize per channel + tiles = (tiles - ADE_MEAN) / ADE_STD # broadcasting over (N,H,W,C) + + # Ensure 3D channels (if grayscale) + if tiles.ndim == 3: # (H,W,C) single image + tiles = np.expand_dims(tiles, axis=0) # add batch dim + + if tiles.shape[-1] == 1: # if only 1 channel + tiles = np.repeat(tiles, 3, axis=-1) + + # NHWC -> NCHW + tiles = np.transpose(tiles, (0, 3, 1, 2)) + + return tiles.astype(np.float32) + + + + + def stable_sigmoid(self, x): + out = np.empty_like(x, dtype=np.float32) + positive_mask = x >= 0 + negative_mask = ~positive_mask + + out[positive_mask] = 1 / (1 + np.exp(-x[positive_mask])) + exp_x = np.exp(x[negative_mask]) + out[negative_mask] = exp_x / (1 + exp_x) + return out + + + def stable_softmax(self, x, axis=-1): + max_x = np.max(x, axis=axis, keepdims=True) + exp_x = np.exp(x - max_x) + return exp_x / np.sum(exp_x, axis=axis, keepdims=True) + + + def post_process_semantic_segmentation_numpy(self, class_queries_logits, masks_queries_logits, target_sizes=None): + # Softmax over classes (remove null class) + masks_classes = self.stable_softmax(class_queries_logits, axis=-1)[..., :-1] + + # Sigmoid for masks + masks_probs = self.stable_sigmoid(masks_queries_logits) + + # Combine: torch.einsum("bqc,bqhw->bchw") + segmentation = np.einsum("bqc,bqhw->bchw", masks_classes, masks_probs) + + semantic_segmentation = [] + if target_sizes is not None: + if len(target_sizes) != class_queries_logits.shape[0]: + raise ValueError("target_sizes length must match batch size") + + for idx in range(len(target_sizes)): + out_h, out_w = target_sizes[idx] + logits_resized = np.zeros((segmentation.shape[1], out_h, out_w), dtype=np.float32) + for c in range(segmentation.shape[1]): + logits_resized[c] = cv2.resize( + segmentation[idx, c], + (out_w, out_h), + interpolation=cv2.INTER_LINEAR + ) + semantic_map = np.argmax(logits_resized, axis=0) + semantic_segmentation.append(semantic_map.astype(np.int32)) + else: + for idx in range(segmentation.shape[0]): + semantic_map = np.argmax(segmentation[idx], axis=0) + semantic_segmentation.append(semantic_map.astype(np.int32)) + + return semantic_segmentation + + def return_probs_mask2former(self, + class_queries_logits, + masks_queries_logits, + target_sizes=None): + + # Softmax over classes (remove null class) + masks_classes = self.stable_softmax(class_queries_logits, axis=-1)[..., :-1] + + # Sigmoid for masks + masks_probs = self.stable_sigmoid(masks_queries_logits) + + # Combine: einsum bqc,bqhw -> bchw + segmentation = np.einsum("bqc,bqhw->bchw", masks_classes, masks_probs) + + # Resize if target_sizes is given + if target_sizes is not None: + if len(target_sizes) != segmentation.shape[0]: + raise ValueError("target_sizes length must match batch size") + + segmentation_resized = np.zeros( + (segmentation.shape[0], segmentation.shape[1], target_sizes[0][0], target_sizes[0][1]), + dtype=np.float32 + ) + + for idx in range(segmentation.shape[0]): + out_h, out_w = target_sizes[idx] + for c in range(segmentation.shape[1]): + segmentation_resized[idx, c] = cv2.resize( + segmentation[idx, c], + (out_w, out_h), + interpolation=cv2.INTER_LINEAR + ) + segmentation = segmentation_resized + + + return segmentation # shape: (B, C, H, W) + + + + + + def process(self, tiles_batched: np.ndarray): + input_batch = self.preprocessing(tiles_batched) + input_building_batch = self.preprocess_tiles_ade20k(tiles_batched) + + model_output = self.sess.run( + output_names=None, + input_feed={self.input_name: input_batch}) + res = self.postprocessing(model_output) + logits_np, masks_np = self.second_model.run(["logits", "outputs_mask.35"], {"pixel_values": input_building_batch}) + target_sizes=[(512, 512)]*logits_np.shape[0] + predicted_seg = self.post_process_semantic_segmentation_numpy( + class_queries_logits=logits_np, + masks_queries_logits=masks_np, + target_sizes=target_sizes + ) + # new_class = {0:0, 1:5, 2:6, 3:7, 4:8} + # build_mask = np.zeros((512,512), dtype=np.int8) + # segmentation_map = predicted_seg[0] + # for k,v in new_class.items(): + # build_mask[segmentation_map == k] = v + + return res[0], predicted_seg + + + def get_number_of_output_channels(self) -> List[int]: + return [9] + + def get_output_shapes(self) -> List[tuple]: + return [("N", 9, 512, 512)] + + + + diff --git a/zipdeepness/processing/models/model_base.py b/zipdeepness/processing/models/model_base.py new file mode 100644 index 0000000000000000000000000000000000000000..11dc214f8fd54746da5be72286a8f8e40843b9e9 --- /dev/null +++ b/zipdeepness/processing/models/model_base.py @@ -0,0 +1,444 @@ +""" Module including the base model interfaces and utilities""" +import ast +import json +from typing import List, Optional + +import numpy as np + +from deepness.common.lazy_package_loader import LazyPackageLoader +from deepness.common.processing_parameters.standardization_parameters import StandardizationParameters + +ort = LazyPackageLoader('onnxruntime') + + +class ModelBase: + """ + Wraps the ONNX model used during processing into a common interface + """ + + def __init__(self, model_file_path: str): + """ + + Parameters + ---------- + model_file_path : str + Path to the model file + """ + self.model_file_path = model_file_path + + options = ort.SessionOptions() + options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL + + providers = [ + 'CUDAExecutionProvider', + 'CPUExecutionProvider' + ] + + self.sess = ort.InferenceSession(self.model_file_path, options=options, providers=providers) + inputs = self.sess.get_inputs() + if len(inputs) > 1: + raise Exception("ONNX model: unsupported number of inputs") + input_0 = inputs[0] + + self.input_shape = input_0.shape + self.input_name = input_0.name + + self.outputs_layers = self.sess.get_outputs() + self.standardization_parameters: StandardizationParameters = self.get_metadata_standarization_parameters() + + self.outputs_names = self.get_outputs_channel_names() + + @classmethod + def get_model_type_from_metadata(cls, model_file_path: str) -> Optional[str]: + """ Get model type from metadata + + Parameters + ---------- + model_file_path : str + Path to the model file + + Returns + ------- + Optional[str] + Model type or None if not found + """ + model = cls(model_file_path) + return model.get_metadata_model_type() + + def get_input_shape(self) -> tuple: + """ Get shape of the input for the model + + Returns + ------- + tuple + Shape of the input (batch_size, channels, height, width) + """ + return self.input_shape + + def get_output_shapes(self) -> List[tuple]: + """ Get shapes of the outputs for the model + + Returns + ------- + List[tuple] + Shapes of the outputs (batch_size, channels, height, width) + """ + return [output.shape for output in self.outputs_layers] + + def get_model_batch_size(self) -> Optional[int]: + """ Get batch size of the model + + Returns + ------- + Optional[int] | None + Batch size or None if not found (dynamic batch size) + """ + bs = self.input_shape[0] + + if isinstance(bs, str): + return None + else: + return bs + + def get_input_size_in_pixels(self) -> int: + """ Get number of input pixels in x and y direction (the same value) + + Returns + ------- + int + Number of pixels in x and y direction + """ + return self.input_shape[-2:] + + def get_outputs_channel_names(self) -> Optional[List[List[str]]]: + """ Get class names from metadata + + Returns + ------- + List[List[str]] | None + List of class names for each model output or None if not found + """ + meta = self.sess.get_modelmeta() + + allowed_key_names = ['class_names', 'names'] # support both names for backward compatibility + for name in allowed_key_names: + if name not in meta.custom_metadata_map: + continue + + txt = meta.custom_metadata_map[name] + try: + class_names = json.loads(txt) # default format recommended in the documentation - classes encoded as json + except json.decoder.JSONDecodeError: + class_names = ast.literal_eval(txt) # keys are integers instead of strings - use ast + + if isinstance(class_names, dict): + class_names = [class_names] + + sorted_by_key = [sorted(cn.items(), key=lambda kv: int(kv[0])) for cn in class_names] + + all_names = [] + + for output_index in range(len(sorted_by_key)): + output_names = [] + class_counter = 0 + + for key, value in sorted_by_key[output_index]: + if int(key) != class_counter: + raise Exception("Class names in the model metadata are not consecutive (missing class label)") + class_counter += 1 + output_names.append(value) + all_names.append(output_names) + + return all_names + + return None + + def get_channel_name(self, layer_id: int, channel_id: int) -> str: + """ Get channel name by id if exists in model metadata + + Parameters + ---------- + channel_id : int + Channel id (means index in the output tensor) + + Returns + ------- + str + Channel name or empty string if not found + """ + + channel_id_str = str(channel_id) + default_return = f'channel_{channel_id_str}' + + if self.outputs_names is None: + return default_return + + if layer_id >= len(self.outputs_names): + raise Exception(f'Layer id {layer_id} is out of range of the model outputs') + + if channel_id >= len(self.outputs_names[layer_id]): + raise Exception(f'Channel id {channel_id} is out of range of the model outputs') + + return f'{self.outputs_names[layer_id][channel_id]}' + + def get_metadata_model_type(self) -> Optional[str]: + """ Get model type from metadata + + Returns + ------- + Optional[str] + Model type or None if not found + """ + meta = self.sess.get_modelmeta() + name = 'model_type' + if name in meta.custom_metadata_map: + value = json.loads(meta.custom_metadata_map[name]) + return str(value).capitalize() + return None + + def get_metadata_standarization_parameters(self) -> Optional[StandardizationParameters]: + """ Get standardization parameters from metadata if exists + + Returns + ------- + Optional[StandardizationParameters] + Standardization parameters or None if not found + """ + meta = self.sess.get_modelmeta() + name_mean = 'standardization_mean' + name_std = 'standardization_std' + + param = StandardizationParameters(channels_number=self.get_input_shape()[-3]) + + if name_mean in meta.custom_metadata_map and name_std in meta.custom_metadata_map: + mean = json.loads(meta.custom_metadata_map[name_mean]) + std = json.loads(meta.custom_metadata_map[name_std]) + + mean = [float(x) for x in mean] + std = [float(x) for x in std] + + param.set_mean_std(mean=mean, std=std) + + return param + + return param # default, no standardization + + def get_metadata_resolution(self) -> Optional[float]: + """ Get resolution from metadata if exists + + Returns + ------- + Optional[float] + Resolution or None if not found + """ + meta = self.sess.get_modelmeta() + name = 'resolution' + if name in meta.custom_metadata_map: + value = json.loads(meta.custom_metadata_map[name]) + return float(value) + return None + + def get_metadata_tile_size(self) -> Optional[int]: + """ Get tile size from metadata if exists + + Returns + ------- + Optional[int] + Tile size or None if not found + """ + meta = self.sess.get_modelmeta() + name = 'tile_size' + if name in meta.custom_metadata_map: + value = json.loads(meta.custom_metadata_map[name]) + return int(value) + return None + + def get_metadata_tiles_overlap(self) -> Optional[int]: + """ Get tiles overlap from metadata if exists + + Returns + ------- + Optional[int] + Tiles overlap or None if not found + """ + meta = self.sess.get_modelmeta() + name = 'tiles_overlap' + if name in meta.custom_metadata_map: + value = json.loads(meta.custom_metadata_map[name]) + return int(value) + return None + + def get_metadata_segmentation_threshold(self) -> Optional[float]: + """ Get segmentation threshold from metadata if exists + + Returns + ------- + Optional[float] + Segmentation threshold or None if not found + """ + meta = self.sess.get_modelmeta() + name = 'seg_thresh' + if name in meta.custom_metadata_map: + value = json.loads(meta.custom_metadata_map[name]) + return float(value) + return None + + def get_metadata_segmentation_small_segment(self) -> Optional[int]: + """ Get segmentation small segment from metadata if exists + + Returns + ------- + Optional[int] + Segmentation small segment or None if not found + """ + meta = self.sess.get_modelmeta() + name = 'seg_small_segment' + if name in meta.custom_metadata_map: + value = json.loads(meta.custom_metadata_map[name]) + return int(value) + return None + + def get_metadata_regression_output_scaling(self) -> Optional[float]: + """ Get regression output scaling from metadata if exists + + Returns + ------- + Optional[float] + Regression output scaling or None if not found + """ + meta = self.sess.get_modelmeta() + name = 'reg_output_scaling' + if name in meta.custom_metadata_map: + value = json.loads(meta.custom_metadata_map[name]) + return float(value) + return None + + def get_metadata_detection_confidence(self) -> Optional[float]: + """ Get detection confidence from metadata if exists + + Returns + ------- + Optional[float] + Detection confidence or None if not found + """ + meta = self.sess.get_modelmeta() + name = 'det_conf' + if name in meta.custom_metadata_map: + value = json.loads(meta.custom_metadata_map[name]) + return float(value) + return None + + def get_detector_type(self) -> Optional[str]: + """ Get detector type from metadata if exists + + Returns string value of DetectorType enum or None if not found + ------- + Optional[str] + Detector type or None if not found + """ + meta = self.sess.get_modelmeta() + name = 'det_type' + if name in meta.custom_metadata_map: + value = json.loads(meta.custom_metadata_map[name]) + return str(value) + return None + + def get_metadata_detection_iou_threshold(self) -> Optional[float]: + """ Get detection iou threshold from metadata if exists + + Returns + ------- + Optional[float] + Detection iou threshold or None if not found + """ + meta = self.sess.get_modelmeta() + name = 'det_iou_thresh' + if name in meta.custom_metadata_map: + value = json.loads(meta.custom_metadata_map[name]) + return float(value) + return None + + def get_number_of_channels(self) -> int: + """ Returns number of channels in the input layer + + Returns + ------- + int + Number of channels in the input layer + """ + return self.input_shape[-3] + + def process(self, tiles_batched: np.ndarray): + """ Process a single tile image + + Parameters + ---------- + img : np.ndarray + Image to process ([TILE_SIZE x TILE_SIZE x channels], type uint8, values 0 to 255) + + Returns + ------- + np.ndarray + Single prediction + """ + input_batch = self.preprocessing(tiles_batched) + model_output = self.sess.run( + output_names=None, + input_feed={self.input_name: input_batch}) + res = self.postprocessing(model_output) + return res + + def preprocessing(self, tiles_batched: np.ndarray) -> np.ndarray: + """ Preprocess the batch of images for the model (resize, normalization, etc) + + Parameters + ---------- + image : np.ndarray + Batch of images to preprocess (N,H,W,C), RGB, 0-255 + + Returns + ------- + np.ndarray + Preprocessed batch of image (N,C,H,W), RGB, 0-1 + """ + + # imported here, to avoid isseue with uninstalled dependencies during the first plugin start + # in other places we use LazyPackageLoader, but here it is not so easy + import deepness.processing.models.preprocessing_utils as preprocessing_utils + + tiles_batched = preprocessing_utils.limit_channels_number(tiles_batched, limit=self.input_shape[-3]) + tiles_batched = preprocessing_utils.normalize_values_to_01(tiles_batched) + tiles_batched = preprocessing_utils.standardize_values(tiles_batched, params=self.standardization_parameters) + tiles_batched = preprocessing_utils.transpose_nhwc_to_nchw(tiles_batched) + + return tiles_batched + + def postprocessing(self, outs: List) -> np.ndarray: + """ Abstract method for postprocessing + + Parameters + ---------- + outs : List + Output from the model (depends on the model type) + + Returns + ------- + np.ndarray + Postprocessed output + """ + raise NotImplementedError('Base class not implemented!') + + def get_number_of_output_channels(self) -> List[int]: + """ Abstract method for getting number of classes in the output layer + + Returns + ------- + int + Number of channels in the output layer""" + raise NotImplementedError('Base class not implemented!') + + def check_loaded_model_outputs(self): + """ Abstract method for checking if the model outputs are valid + + """ + raise NotImplementedError('Base class not implemented!') diff --git a/zipdeepness/processing/models/model_types.py b/zipdeepness/processing/models/model_types.py new file mode 100644 index 0000000000000000000000000000000000000000..a0e2c7d6db23be65ad289462e740a94e00d564ee --- /dev/null +++ b/zipdeepness/processing/models/model_types.py @@ -0,0 +1,93 @@ +import enum +from dataclasses import dataclass + +from deepness.common.processing_parameters.detection_parameters import DetectionParameters +from deepness.common.processing_parameters.map_processing_parameters import MapProcessingParameters +from deepness.common.processing_parameters.recognition_parameters import RecognitionParameters +from deepness.common.processing_parameters.regression_parameters import RegressionParameters +from deepness.common.processing_parameters.segmentation_parameters import SegmentationParameters +from deepness.common.processing_parameters.superresolution_parameters import SuperresolutionParameters +from deepness.processing.map_processor.map_processor_detection import MapProcessorDetection +from deepness.processing.map_processor.map_processor_recognition import MapProcessorRecognition +from deepness.processing.map_processor.map_processor_regression import MapProcessorRegression +from deepness.processing.map_processor.map_processor_segmentation import MapProcessorSegmentation +from deepness.processing.map_processor.map_processor_superresolution import MapProcessorSuperresolution +from deepness.processing.models.detector import Detector +from deepness.processing.models.recognition import Recognition +from deepness.processing.models.regressor import Regressor +from deepness.processing.models.segmentor import Segmentor +from deepness.processing.models.superresolution import Superresolution +from deepness.processing.models.dual import DualModel + + +class ModelType(enum.Enum): + SEGMENTATION = Segmentor.get_class_display_name() + REGRESSION = Regressor.get_class_display_name() + DETECTION = Detector.get_class_display_name() + SUPERRESOLUTION = Superresolution.get_class_display_name() + RECOGNITION = Recognition.get_class_display_name() + + +@dataclass +class ModelDefinition: + model_type: ModelType + model_class: type + parameters_class: type + map_processor_class: type + + @classmethod + def get_model_definitions(cls): + return [ + cls( + model_type=ModelType.SEGMENTATION, + model_class=DualModel, + parameters_class=SegmentationParameters, + map_processor_class=MapProcessorSegmentation, + ), + cls( + model_type=ModelType.REGRESSION, + model_class=Regressor, + parameters_class=RegressionParameters, + map_processor_class=MapProcessorRegression, + ), + cls( + model_type=ModelType.DETECTION, + model_class=Detector, + parameters_class=DetectionParameters, + map_processor_class=MapProcessorDetection, + ), # superresolution + cls( + model_type=ModelType.SUPERRESOLUTION, + model_class=Superresolution, + parameters_class=SuperresolutionParameters, + map_processor_class=MapProcessorSuperresolution, + ), # recognition + cls( + model_type=ModelType.RECOGNITION, + model_class=Recognition, + parameters_class=RecognitionParameters, + map_processor_class=MapProcessorRecognition, + ) + + ] + + @classmethod + def get_definition_for_type(cls, model_type: ModelType): + model_definitions = cls.get_model_definitions() + for model_definition in model_definitions: + if model_definition.model_type == model_type: + return model_definition + raise Exception(f"Unknown model type: '{model_type}'!") + + @classmethod + def get_definition_for_params(cls, params: MapProcessingParameters): + """ get model definition corresponding to the specified parameters """ + model_definitions = cls.get_model_definitions() + for model_definition in model_definitions: + if type(params) == model_definition.parameters_class: + return model_definition + + for model_definition in model_definitions: + if isinstance(params, model_definition.parameters_class): + return model_definition + raise Exception(f"Unknown model type for parameters: '{params}'!") diff --git a/zipdeepness/processing/models/preprocessing_utils.py b/zipdeepness/processing/models/preprocessing_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..eff25fa163318356e2b1d95f8223c290cd4fd754 --- /dev/null +++ b/zipdeepness/processing/models/preprocessing_utils.py @@ -0,0 +1,41 @@ +import numpy as np + +from deepness.common.processing_parameters.standardization_parameters import StandardizationParameters + + +def limit_channels_number(tiles_batched: np.array, limit: int) -> np.array: + """ Limit the number of channels in the input image to the model + + :param tiles_batched: Batch of tiles + :param limit: Number of channels to keep + :return: Batch of tiles with limited number of channels + """ + return tiles_batched[:, :, :, :limit] + + +def normalize_values_to_01(tiles_batched: np.array) -> np.array: + """ Normalize the values of the input image to the model to the range [0, 1] + + :param tiles_batched: Batch of tiles + :return: Batch of tiles with values in the range [0, 1], in float32 + """ + return np.float32(tiles_batched * 1./255.) + + +def standardize_values(tiles_batched: np.array, params: StandardizationParameters) -> np.array: + """ Standardize the input image to the model + + :param tiles_batched: Batch of tiles + :param params: Parameters for standardization of type STANDARIZE_PARAMS + :return: Batch of tiles with standardized values + """ + return (tiles_batched - params.mean) / params.std + + +def transpose_nhwc_to_nchw(tiles_batched: np.array) -> np.array: + """ Transpose the input image from NHWC to NCHW + + :param tiles_batched: Batch of tiles in NHWC format + :return: Batch of tiles in NCHW format + """ + return np.transpose(tiles_batched, (0, 3, 1, 2)) diff --git a/zipdeepness/processing/models/recognition.py b/zipdeepness/processing/models/recognition.py new file mode 100644 index 0000000000000000000000000000000000000000..12233e4b006597ec8fcd0775ee1c2f7ff0ff8573 --- /dev/null +++ b/zipdeepness/processing/models/recognition.py @@ -0,0 +1,98 @@ +""" Module including the class for the recognition of the images +""" +import logging +from typing import List + +import numpy as np + +from deepness.common.lazy_package_loader import LazyPackageLoader +from deepness.processing.models.model_base import ModelBase + +cv2 = LazyPackageLoader('cv2') + + +class Recognition(ModelBase): + """Class implements recognition model + + Recognition model is used to predict class confidence per pixel of the image. + """ + + def __init__(self, model_file_path: str): + """ + + Parameters + ---------- + model_file_path : str + Path to the model file + """ + super(Recognition, self).__init__(model_file_path) + + def postprocessing(self, model_output: List) -> np.ndarray: + """Postprocess the model output. + Function returns the array of embeddings + + Parameters + ---------- + model_output : List + Output embeddings from the (Recognition) model + + Returns + ------- + np.ndarray + Same as input + """ + # TODO - compute cosine similarity to self.query_img_emb + # cannot, won't work for query image + + return np.array(model_output) + + def get_number_of_output_channels(self): + """Returns model's number of class + + Returns + ------- + int + Number of channels in the output layer + """ + logging.warning(f"outputs_layers: {self.outputs_layers}") + logging.info(f"outputs_layers: {self.outputs_layers}") + + if len(self.outputs_layers) == 1: + return [self.outputs_layers[0].shape[1]] + else: + raise NotImplementedError( + "Model with multiple output layers is not supported! Use only one output layer." + ) + + @classmethod + def get_class_display_name(cls): + """Returns the name of the class to be displayed in the GUI + + Returns + ------- + str + Name of the class + """ + return cls.__name__ + + def check_loaded_model_outputs(self): + """Checks if the model outputs are valid + + Valid means that: + - the model has only one output + - the output is 2D (N,C) + - the batch size is 1 + + """ + if len(self.outputs_layers) == 1: + shape = self.outputs_layers[0].shape + + if len(shape) != 2: + raise Exception( + f"Recognition model output should have 4 dimensions: (B,C,H,W). Has {shape}" + ) + + else: + raise NotImplementedError( + "Model with multiple output layers is not supported! Use only one output layer." + ) diff --git a/zipdeepness/processing/models/regressor.py b/zipdeepness/processing/models/regressor.py new file mode 100644 index 0000000000000000000000000000000000000000..f995938d981bf2425d9db242ad25fa734e260bf7 --- /dev/null +++ b/zipdeepness/processing/models/regressor.py @@ -0,0 +1,92 @@ +""" Module including Regression model definition +""" +from typing import List + +import numpy as np + +from deepness.processing.models.model_base import ModelBase + + +class Regressor(ModelBase): + """ Class implements regression model. + + Regression model is used to predict metric per pixel of the image. + """ + + def __init__(self, model_file_path: str): + """ + + Parameters + ---------- + model_file_path : str + Path to the model file + """ + super(Regressor, self).__init__(model_file_path) + + def postprocessing(self, model_output: List) -> np.ndarray: + """ Postprocess the model output. + + Parameters + ---------- + model_output : List + Output from the (Regression) model + + Returns + ------- + np.ndarray + Output from the (Regression) model + """ + return model_output + + def get_number_of_output_channels(self) -> List[int]: + """ Returns number of channels in the output layer + + Returns + ------- + int + Number of channels in the output layer + """ + channels = [] + + for layer in self.outputs_layers: + if len(layer.shape) != 4 and len(layer.shape) != 3: + raise Exception(f'Output layer should have 3 or 4 dimensions: (Bs, H, W) or (Bs, Channels, H, W). Actually has: {layer.shape}') + + if len(layer.shape) == 3: + channels.append(1) + elif len(layer.shape) == 4: + channels.append(layer.shape[-3]) + + return channels + + @classmethod + def get_class_display_name(cls) -> str: + """ Returns display name of the model class + + Returns + ------- + str + Display name of the model class + """ + return cls.__name__ + + def check_loaded_model_outputs(self): + """ Check if the model has correct output layers + + Correct means that: + - there is at least one output layer + - batch size is 1 or parameter + - each output layer regresses only one channel + - output resolution is square + """ + for layer in self.outputs_layers: + if len(layer.shape) != 4 and len(layer.shape) != 3: + raise Exception(f'Output layer should have 3 or 4 dimensions: (Bs, H, W) or (Bs, Channels, H, W). Actually has: {layer.shape}') + + if len(layer.shape) == 4: + if layer.shape[2] != layer.shape[3]: + raise Exception(f'Regression model can handle only square outputs masks. Has: {layer.shape}') + + elif len(layer.shape) == 3: + if layer.shape[1] != layer.shape[2]: + raise Exception(f'Regression model can handle only square outputs masks. Has: {layer.shape}') diff --git a/zipdeepness/processing/models/segmentor.py b/zipdeepness/processing/models/segmentor.py new file mode 100644 index 0000000000000000000000000000000000000000..6dabf26b2cc56d594eb1ed9615d45e6ce4882791 --- /dev/null +++ b/zipdeepness/processing/models/segmentor.py @@ -0,0 +1,104 @@ +""" Module including the class for the segmentation of the images +""" +from typing import List + +import numpy as np + +from deepness.processing.models.model_base import ModelBase + + +class Segmentor(ModelBase): + """Class implements segmentation model + + Segmentation model is used to predict class confidence per pixel of the image. + """ + + def __init__(self, model_file_path: str): + """ + + Parameters + ---------- + model_file_path : str + Path to the model file + """ + super(Segmentor, self).__init__(model_file_path) + + self.outputs_are_sigmoid = self.check_loaded_model_outputs() + + for idx in range(len(self.outputs_layers)): + if self.outputs_names is None: + continue + + if len(self.outputs_names[idx]) == 1 and self.outputs_are_sigmoid[idx]: + self.outputs_names[idx] = ['background', self.outputs_names[idx][0]] + + def postprocessing(self, model_output: List) -> np.ndarray: + """ Postprocess the model output. + Function returns the mask with the probability of the presence of the class in the image. + + Parameters + ---------- + model_output : List + Output from the (Segmentation) model + + Returns + ------- + np.ndarray + Output from the (Segmentation) model + """ + return model_output + + def get_number_of_output_channels(self) -> List[int]: + """ Returns model's number of class + + Returns + ------- + int + Number of channels in the output layer + """ + output_channels = [] + for layer in self.outputs_layers: + ls = layer.shape + + if len(ls) == 3: + output_channels.append(2) + elif len(ls) == 4: + chn = ls[-3] + if chn == 1: + output_channels.append(2) + else: + output_channels.append(chn) + + return output_channels + + @classmethod + def get_class_display_name(cls): + """ Returns the name of the class to be displayed in the GUI + + Returns + ------- + str + Name of the class + """ + return cls.__name__ + + def check_loaded_model_outputs(self) -> List[bool]: + """ Check if the model outputs are sigmoid (for segmentation) + + Parameters + ---------- + + Returns + ------- + List[bool] + List of booleans indicating if the model outputs are sigmoid + """ + outputs = [] + + for output in self.outputs_layers: + if len(output.shape) == 3: + outputs.append(True) + else: + outputs.append(output.shape[-3] == 1) + + return outputs diff --git a/zipdeepness/processing/models/superresolution.py b/zipdeepness/processing/models/superresolution.py new file mode 100644 index 0000000000000000000000000000000000000000..1fdaed87e097df953df8b9f5def0c353679b6460 --- /dev/null +++ b/zipdeepness/processing/models/superresolution.py @@ -0,0 +1,99 @@ +""" Module including Super Resolution model definition +""" +from typing import List + +import numpy as np + +from deepness.processing.models.model_base import ModelBase + + +class Superresolution(ModelBase): + """ Class implements super resolution model. + + Super Resolution model is used improve the resolution of an image. + """ + + def __init__(self, model_file_path: str): + """ + + Parameters + ---------- + model_file_path : str + Path to the model file + """ + super(Superresolution, self).__init__(model_file_path) + + def postprocessing(self, model_output: List) -> np.ndarray: + """ Postprocess the model output. + + Parameters + ---------- + model_output : List + Output from the (Regression) model + + Returns + ------- + np.ndarray + Postprocessed mask (H,W,C), 0-1 (one output channel) + + """ + return model_output[0] + + def get_number_of_output_channels(self) -> List[int]: + """ Returns number of channels in the output layer + + Returns + ------- + int + Number of channels in the output layer + """ + if len(self.outputs_layers) == 1: + return [self.outputs_layers[0].shape[-3]] + else: + raise NotImplementedError("Model with multiple output layers is not supported! Use only one output layer.") + + @classmethod + def get_class_display_name(cls) -> str: + """ Returns display name of the model class + + Returns + ------- + str + Display name of the model class + """ + return cls.__name__ + + def get_output_shape(self) -> List[int]: + """ Returns shape of the output layer + + Returns + ------- + List[int] + Shape of the output layer + """ + if len(self.outputs_layers) == 1: + return self.outputs_layers[0].shape + else: + raise NotImplementedError("Model with multiple output layers is not supported! Use only one output layer.") + + def check_loaded_model_outputs(self): + """ Check if the model has correct output layers + + Correct means that: + - there is only one output layer + - output layer has 1 channel + - batch size is 1 + - output resolution is square + """ + if len(self.outputs_layers) == 1: + shape = self.outputs_layers[0].shape + + if len(shape) != 4: + raise Exception(f'Regression model output should have 4 dimensions: (Batch_size, Channels, H, W). \n' + f'Actually has: {shape}') + + if shape[2] != shape[3]: + raise Exception(f'Regression model can handle only square outputs masks. Has: {shape}') + + else: + raise NotImplementedError("Model with multiple output layers is not supported! Use only one output layer.") diff --git a/zipdeepness/processing/processing_utils.py b/zipdeepness/processing/processing_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..25edbad477cbb4182a02c312b0c760ec16e9a3c8 --- /dev/null +++ b/zipdeepness/processing/processing_utils.py @@ -0,0 +1,557 @@ +""" +This file contains utilities related to processing of the ortophoto +""" + +import logging +from dataclasses import dataclass +from typing import List, Optional, Tuple + +import numpy as np +from qgis.core import (Qgis, QgsCoordinateTransform, QgsFeature, QgsGeometry, QgsPointXY, QgsRasterLayer, QgsRectangle, + QgsUnitTypes, QgsWkbTypes) + +from deepness.common.defines import IS_DEBUG +from deepness.common.lazy_package_loader import LazyPackageLoader +from deepness.common.processing_parameters.map_processing_parameters import MapProcessingParameters +from deepness.common.processing_parameters.segmentation_parameters import SegmentationParameters +from deepness.common.temp_files_handler import TempFilesHandler + +cv2 = LazyPackageLoader('cv2') + + +def convert_meters_to_rlayer_units(rlayer: QgsRasterLayer, distance_m: float) -> float: + """ How many map units are there in one meter. + :param rlayer: raster layer for which we want to convert meters to its units + :param distance_m: distance in meters + """ + if rlayer.crs().mapUnits() != QgsUnitTypes.DistanceUnit.DistanceMeters: + logging.warning(f"Map units are not meters but '{rlayer.crs().mapUnits()}'. It should be fine for most cases, but be aware.") + + # Now we support all units, but we need to convert them to meters + # to have a consistent unit for the distance + scaling_factor = QgsUnitTypes.fromUnitToUnitFactor(QgsUnitTypes.DistanceMeters, rlayer.crs().mapUnits()) + distance = distance_m * scaling_factor + assert distance != 0 + return distance + + +def get_numpy_data_type_for_qgis_type(data_type_qgis: Qgis.DataType): + """Conver QGIS data type to corresponding numpy data type + In [58]: Qgis.DataType? + implemented: Byte, UInt16, Int16, Float32, Float64 + """ + if data_type_qgis == Qgis.DataType.Byte: + return np.uint8 + if data_type_qgis == Qgis.DataType.UInt16: + return np.uint16 + if data_type_qgis == Qgis.DataType.UInt32: + return np.uint32 + if data_type_qgis == Qgis.DataType.Int16: + return np.int16 + if data_type_qgis == Qgis.DataType.Int32: + return np.int32 + if data_type_qgis == Qgis.DataType.Float32: + return np.float32 + if data_type_qgis == Qgis.DataType.Float64: + return np.float64 + raise Exception(f"Invalid input layer data type ({data_type_qgis})!") + + +def get_tile_image( + rlayer: QgsRasterLayer, + extent: QgsRectangle, + params: MapProcessingParameters) -> np.ndarray: + """_summary_ + + Parameters + ---------- + rlayer : QgsRasterLayer + raster layer from which the image will be extracted + extent : QgsRectangle + extent of the image to extract + params : MapProcessingParameters + map processing parameters + + Returns + ------- + np.ndarray + extracted image [SIZE x SIZE x CHANNELS]. Probably RGBA channels + """ + + expected_meters_per_pixel = params.resolution_cm_per_px / 100 + expected_units_per_pixel = convert_meters_to_rlayer_units( + rlayer, expected_meters_per_pixel) + expected_units_per_pixel_2d = expected_units_per_pixel, expected_units_per_pixel + # to get all pixels - use the 'rlayer.rasterUnitsPerPixelX()' instead of 'expected_units_per_pixel_2d' + image_size = round((extent.width()) / expected_units_per_pixel_2d[0]), \ + round((extent.height()) / expected_units_per_pixel_2d[1]) + + # sanity check, that we gave proper extent as parameter + assert image_size[0] == params.tile_size_px + assert image_size[1] == params.tile_size_px + + # enable resampling + data_provider = rlayer.dataProvider() + if data_provider is None: + raise Exception("Somehow invalid rlayer!") + data_provider.enableProviderResampling(True) + original_resampling_method = data_provider.zoomedInResamplingMethod() + data_provider.setZoomedInResamplingMethod( + data_provider.ResamplingMethod.Bilinear) + data_provider.setZoomedOutResamplingMethod( + data_provider.ResamplingMethod.Bilinear) + + def get_raster_block(band_number_): + raster_block = rlayer.dataProvider().block( + band_number_, + extent, + image_size[0], image_size[1]) + block_height, block_width = raster_block.height(), raster_block.width() + if block_height == 0 or block_width == 0: + raise Exception("No data on layer within the expected extent!") + return raster_block + + input_channels_mapping = params.input_channels_mapping + number_of_model_inputs = input_channels_mapping.get_number_of_model_inputs() + tile_data = [] + + if input_channels_mapping.are_all_inputs_standalone_bands(): + band_count = rlayer.bandCount() + for i in range(number_of_model_inputs): + image_channel = input_channels_mapping.get_image_channel_for_model_input( + i) + band_number = image_channel.get_band_number() + # we cannot obtain a higher band than the maximum in the image + assert band_number <= band_count + rb = get_raster_block(band_number) + raw_data = rb.data() + bytes_array = bytes(raw_data) + data_type = rb.dataType() + data_type_numpy = get_numpy_data_type_for_qgis_type(data_type) + a = np.frombuffer(bytes_array, dtype=data_type_numpy) + b = a.reshape((image_size[1], image_size[0], 1)) + tile_data.append(b) + elif input_channels_mapping.are_all_inputs_composite_byte(): + rb = get_raster_block(1) # the data are always in band 1 + raw_data = rb.data() + bytes_array = bytes(raw_data) + dt = rb.dataType() + number_of_image_channels = input_channels_mapping.get_number_of_image_channels() + # otherwise we did something wrong earlier... + assert number_of_image_channels == 4 + if dt != Qgis.DataType.ARGB32: + raise Exception("Invalid input layer data type!") + a = np.frombuffer(bytes_array, dtype=np.uint8) + b = a.reshape((image_size[1], image_size[0], number_of_image_channels)) + + for i in range(number_of_model_inputs): + image_channel = input_channels_mapping.get_image_channel_for_model_input( + i) + byte_number = image_channel.get_byte_number() + # we cannot get more bytes than there are + assert byte_number < number_of_image_channels + # last index to keep dimension + tile_data.append(b[:, :, byte_number:byte_number+1]) + else: + raise Exception("Unsupported image channels composition!") + + data_provider.setZoomedInResamplingMethod( + original_resampling_method) # restore old resampling method + img = np.concatenate(tile_data, axis=2) + return img + + +def erode_dilate_image(img, segmentation_parameters: SegmentationParameters): + """Apply to dilate and erode to the input image""" + if segmentation_parameters.postprocessing_dilate_erode_size: + size = (segmentation_parameters.postprocessing_dilate_erode_size // 2) ** 2 + 1 + kernel = np.ones((size, size), np.uint8) + img = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel) + img = cv2.morphologyEx(img, cv2.MORPH_CLOSE, kernel) + return img + + +def convert_cv_contours_to_features(features, + cv_contours, + hierarchy, + current_contour_index, + is_hole, + current_holes): + """ + Convert contour found with OpenCV to features accepted by QGis. + Called recursively. + """ + + if current_contour_index == -1: + return + + while True: + contour = cv_contours[current_contour_index] + if len(contour) >= 3: + first_child = hierarchy[current_contour_index][2] + internal_holes = [] + convert_cv_contours_to_features( + features=features, + cv_contours=cv_contours, + hierarchy=hierarchy, + current_contour_index=first_child, + is_hole=not is_hole, + current_holes=internal_holes) + + if is_hole: + current_holes.append(contour) + else: + feature = QgsFeature() + polygon_xy_vec_vec = [ + contour, + *internal_holes + ] + geometry = QgsGeometry.fromPolygonXY(polygon_xy_vec_vec) + feature.setGeometry(geometry) + + # polygon = shapely.geometry.Polygon(contour, holes=internal_holes) + features.append(feature) + + current_contour_index = hierarchy[current_contour_index][0] + if current_contour_index == -1: + break + + +def transform_points_list_xy_to_target_crs( + points: List[Tuple], + extent: QgsRectangle, + rlayer_units_per_pixel: float): + """ Transform points from xy coordinates to the target CRS system coordinates""" + x_left = extent.xMinimum() + y_upper = extent.yMaximum() + points_crs = [] + + for point_xy in points: + x_crs = point_xy[0] * rlayer_units_per_pixel + x_left + y_crs = -(point_xy[1] * rlayer_units_per_pixel - y_upper) + points_crs.append(QgsPointXY(x_crs, y_crs)) + return points_crs + + +def transform_contours_yx_pixels_to_target_crs( + contours, + extent: QgsRectangle, + rlayer_units_per_pixel: float): + """ Transform countours with points as yx pixels to the target CRS system coordinates""" + x_left = extent.xMinimum() + y_upper = extent.yMaximum() + + polygons_crs = [] + for polygon_3d in contours: + # https://stackoverflow.com/questions/33458362/opencv-findcontours-why-do-we-need-a- + # vectorvectorpoint-to-store-the-cont + polygon = polygon_3d.squeeze(axis=1) + + polygon_crs = [] + for i in range(len(polygon)): + yx_px = polygon[i] + x_crs = yx_px[0] * rlayer_units_per_pixel + x_left + y_crs = -(yx_px[1] * rlayer_units_per_pixel - y_upper) + polygon_crs.append(QgsPointXY(x_crs, y_crs)) + polygons_crs.append(polygon_crs) + return polygons_crs + + +@dataclass +class BoundingBox: + """ + Describes a bounding box rectangle. + Similar to cv2.Rect + """ + x_min: int + x_max: int + y_min: int + y_max: int + rot: float = 0.0 + + def get_shape(self) -> Tuple[int, int]: + """ Returns the shape of the bounding box as a tuple (height, width) + + Returns + ------- + tuple + (height, width) + """ + return [ + self.y_max - self.y_min + 1, + self.x_max - self.x_min + 1 + ] + + def get_xyxy(self) -> Tuple[int, int, int, int]: + """ Returns the bounding box as a tuple (x_min, y_min, x_max, y_max) + + Returns + ------- + Tuple[int, int, int, int] + (x_min, y_min, x_max, y_max) + """ + return [ + self.x_min, + self.y_min, + self.x_max, + self.y_max + ] + + def get_xyxy_rot(self) -> Tuple[int, int, int, int, float]: + """ Returns the bounding box as a tuple (x_min, y_min, x_max, y_max, rotation) + + Returns + ------- + Tuple[int, int, int, int, float] + (x_min, y_min, x_max, y_max, rotation) + """ + return [ + self.x_min, + self.y_min, + self.x_max, + self.y_max, + self.rot + ] + + def get_xywh(self) -> Tuple[int, int, int, int]: + """ Returns the bounding box as a tuple (x_min, y_min, width, height) + + Returns + ------- + Tuple[int, int, int, int] + (x_min, y_min, width, height) + """ + return [ + self.x_min, + self.y_min, + self.x_max - self.x_min, + self.y_max - self.y_min + ] + + def get_center(self) -> Tuple[int, int]: + """ Returns the center of the bounding box as a tuple (x, y) + + Returns + ------- + Tuple[int, int] + (x, y) + """ + return [ + (self.x_min + self.x_max) // 2, + (self.y_min + self.y_max) // 2 + ] + + def get_area(self) -> float: + """Calculate bounding box reactangle area + + Returns + ------- + float + Bounding box area + """ + shape = self.get_shape() + return shape[0] * shape[1] + + def calculate_overlap_in_pixels(self, other) -> float: + """Calculate overlap between two bounding boxes in pixels + + Parameters + ---------- + other : BoundingBox + Other bounding box + + Returns + ------- + float + Overlap in pixels + """ + dx = min(self.x_max, other.x_max) - max(self.x_min, other.x_min) + dy = min(self.y_max, other.y_max) - max(self.y_min, other.y_min) + if (dx >= 0) and (dy >= 0): + return dx * dy + return 0 + + def calculate_intersection_over_smaler_area(self, other) -> float: + """ Calculate intersection over smaler area (IoS) between two bounding boxes + + Parameters + ---------- + other : BoundingBox + Other bounding bo + + Returns + ------- + float + Value between 0 and 1 + """ + + Aarea = (self.x_max - self.x_min) * (self.y_max - self.y_min) + Barea = (other.x_max - other.x_min) * (other.y_max - other.y_min) + + xA = max(self.x_min, other.x_min) + yA = max(self.y_min, other.y_min) + xB = min(self.x_max, other.x_max) + yB = min(self.y_max, other.y_max) + + # compute the area of intersection rectangle + return max(0, xB - xA + 1) * max(0, yB - yA + 1) / min(Aarea, Barea) + + def get_slice(self) -> Tuple[slice, slice]: + """ Returns the bounding box as a tuple of slices (y_slice, x_slice) + + Returns + ------- + Tuple[slice, slice] + (y_slice, x_slice) + """ + roi_slice = np.s_[self.y_min:self.y_max + 1, self.x_min:self.x_max + 1] + return roi_slice + + def apply_offset(self, offset_x: int, offset_y: int): + """Apply (x,y) offset to keeping coordinates + + Parameters + ---------- + offset_x : int + x-axis offset in pixels + offset_y : int + y-axis offset in pixels + """ + self.x_min += offset_x + self.y_min += offset_y + self.x_max += offset_x + self.y_max += offset_y + + def get_4_corners(self) -> List[Tuple]: + """Get 4 points (corners) describing the detection rectangle, each point in (x, y) format + + Returns + ------- + List[Tuple] + List of 4 rectangle corners in (x, y) format + """ + if np.isclose(self.rot, 0.0): + return [ + (self.x_min, self.y_min), + (self.x_min, self.y_max), + (self.x_max, self.y_max), + (self.x_max, self.y_min), + ] + else: + x_center = (self.x_min + self.x_max) / 2 + y_center = (self.y_min + self.y_max) / 2 + + corners = np.array([ + [self.x_min, self.y_min], + [self.x_min, self.y_max], + [self.x_max, self.y_max], + [self.x_max, self.y_min], + ]) + + xys = x_center + np.cos(self.rot) * (corners[:, 0] - x_center) - np.sin(self.rot) * (corners[:, 1] - y_center) + yys = y_center + np.sin(self.rot) * (corners[:, 0] - x_center) + np.cos(self.rot) * (corners[:, 1] - y_center) + + return [(int(x), int(y)) for x, y in zip(xys, yys)] + + +def transform_polygon_with_rings_epsg_to_extended_xy_pixels( + polygons: List[List[QgsPointXY]], + extended_extent: QgsRectangle, + img_size_y_pixels: int, + rlayer_units_per_pixel: float) -> List[List[Tuple]]: + """ + Transform coordinates polygons to pixels contours (with cv2 format), in base_extent pixels system + :param polygons: List of tuples with two lists each (x and y points respoectively) + :param extended_extent: + :param img_size_y_pixels: + :param rlayer_units_per_pixel: + :return: 2D contours list + """ + xy_pixel_contours = [] + for polygon in polygons: + xy_pixel_contour = [] + + x_min_epsg = extended_extent.xMinimum() + y_min_epsg = extended_extent.yMinimum() + y_max_pixel = img_size_y_pixels - 1 # -1 to have the max pixel, not shape + for point_epsg in polygon: + x_epsg, y_epsg = point_epsg + x = round((x_epsg - x_min_epsg) / rlayer_units_per_pixel) + y = y_max_pixel - \ + round((y_epsg - y_min_epsg) / rlayer_units_per_pixel) + # NOTE: here we can get pixels +-1 values, because we operate on already rounded bounding boxes + xy_pixel_contour.append((x, y)) + + # Values: + # extended_extent.height() / rlayer_units_per_pixel, extended_extent.width() / rlayer_units_per_pixel + # are not integers, because extents are aligned to grid, not pixels resolution + + xy_pixel_contours.append(np.asarray(xy_pixel_contour)) + return xy_pixel_contours + + +def create_area_mask_image(vlayer_mask, + rlayer: QgsRasterLayer, + extended_extent: QgsRectangle, + rlayer_units_per_pixel: float, + image_shape_yx: Tuple[int, int], + files_handler: Optional[TempFilesHandler] = None) -> Optional[np.ndarray]: + """ + Mask determining area to process (within extended_extent coordinates) + None if no mask layer provided. + """ + + if vlayer_mask is None: + return None + + if files_handler is None: + img = np.zeros(shape=image_shape_yx, dtype=np.uint8) + else: + img = np.memmap(files_handler.get_area_mask_img_path(), + dtype=np.uint8, + mode='w+', + shape=image_shape_yx) + + features = vlayer_mask.getFeatures() + + if vlayer_mask.crs() != rlayer.crs(): + xform = QgsCoordinateTransform() + xform.setSourceCrs(vlayer_mask.crs()) + xform.setDestinationCrs(rlayer.crs()) + + # see https://docs.qgis.org/3.22/en/docs/pyqgis_developer_cookbook/vector.html#iterating-over-vector-layer + for feature in features: + print("Feature ID: ", feature.id()) + geom = feature.geometry() + + if vlayer_mask.crs() != rlayer.crs(): + geom.transform(xform) + + geom_single_type = QgsWkbTypes.isSingleType(geom.wkbType()) + + if geom.type() == QgsWkbTypes.PointGeometry: + logging.warning("Point geometry not supported!") + elif geom.type() == QgsWkbTypes.LineGeometry: + logging.warning("Line geometry not supported!") + elif geom.type() == QgsWkbTypes.PolygonGeometry: + polygons = [] + if geom_single_type: + polygon = geom.asPolygon() # polygon with rings + polygons.append(polygon) + else: + polygons = geom.asMultiPolygon() + + for polygon_with_rings in polygons: + polygon_with_rings_xy = transform_polygon_with_rings_epsg_to_extended_xy_pixels( + polygons=polygon_with_rings, + extended_extent=extended_extent, + img_size_y_pixels=image_shape_yx[0], + rlayer_units_per_pixel=rlayer_units_per_pixel) + # first polygon is actual polygon + cv2.fillPoly(img, pts=polygon_with_rings_xy[:1], color=255) + if len(polygon_with_rings_xy) > 1: # further polygons are rings + cv2.fillPoly(img, pts=polygon_with_rings_xy[1:], color=0) + else: + print("Unknown or invalid geometry") + + return img diff --git a/zipdeepness/processing/tile_params.py b/zipdeepness/processing/tile_params.py new file mode 100644 index 0000000000000000000000000000000000000000..bea0b9e2337d7beae34a34efff0119bad81b6052 --- /dev/null +++ b/zipdeepness/processing/tile_params.py @@ -0,0 +1,176 @@ +""" +This file contains utilities related to processing a tile. +Tile is a small part of the ortophoto, which is being processed by the model one by one. +""" + +from typing import Optional, Tuple + +import numpy as np +from qgis.core import QgsRectangle + +from deepness.common.lazy_package_loader import LazyPackageLoader +from deepness.common.processing_parameters.map_processing_parameters import MapProcessingParameters + +cv2 = LazyPackageLoader('cv2') + + +class TileParams: + """ Defines a single tile parameters - image that's being processed by model""" + + def __init__(self, + x_bin_number: int, + y_bin_number: int, + x_bins_number: int, + y_bins_number: int, + params: MapProcessingParameters, + rlayer_units_per_pixel, + processing_extent: QgsRectangle): + """ init + + Parameters + ---------- + x_bin_number : int + what is the tile number in a row, counting from left side + y_bin_number : int + what is the tile number in a column + x_bins_number : int + how many tiles are there in a row + y_bins_number : int + how many tiles are there in a column + params : MapProcessingParameters + processing parameters + rlayer_units_per_pixel : _type_ + How many rlayer crs units are in a pixel + processing_extent : QgsRectangle + """ + self.x_bin_number = x_bin_number + self.y_bin_number = y_bin_number + self.x_bins_number = x_bins_number + self.y_bins_number = y_bins_number + self.stride_px = params.processing_stride_px + self.start_pixel_x = x_bin_number * self.stride_px + self.start_pixel_y = y_bin_number * self.stride_px + self.params = params + self.rlayer_units_per_pixel = rlayer_units_per_pixel + + self.extent = self._calculate_extent(processing_extent) # type: QgsRectangle # tile extent in CRS cordinates + + def _calculate_extent(self, processing_extent): + tile_extent = QgsRectangle(processing_extent) # copy + x_min = processing_extent.xMinimum() + self.start_pixel_x * self.rlayer_units_per_pixel + y_max = processing_extent.yMaximum() - self.start_pixel_y * self.rlayer_units_per_pixel + tile_extent.setXMinimum(x_min) + # extent needs to be on the further edge (so including the corner pixel, hence we do not subtract 1) + tile_extent.setXMaximum(x_min + self.params.tile_size_px * self.rlayer_units_per_pixel) + tile_extent.setYMaximum(y_max) + y_min = y_max - self.params.tile_size_px * self.rlayer_units_per_pixel + tile_extent.setYMinimum(y_min) + return tile_extent + + def get_slice_on_full_image_for_entire_tile(self) -> Tuple[slice, slice]: + """ Obtain slice to get the entire tile from full final image, + including the overlapping parts. + + Returns + ------- + Tuple[slice, slice] + Slice to be used on the full image + """ + + # 'core' part of the tile (not overlapping with other tiles), for sure copied for each tile + x_min = self.start_pixel_x + x_max = self.start_pixel_x + self.params.tile_size_px - 1 + y_min = self.start_pixel_y + y_max = self.start_pixel_y + self.params.tile_size_px - 1 + + roi_slice = np.s_[:, y_min:y_max + 1, x_min:x_max + 1] + return roi_slice + + def get_slice_on_full_image_for_copying(self, tile_offset: int = 0): + """ + As we are doing processing with overlap, we are not going to copy the entire tile result to final image, + but only the part that is not overlapping with the neighbouring tiles. + Edge tiles have special handling too. + + :param tile_offset: how many pixels to cut from the tile result (to remove the padding) + + :return Slice to be used on the full image + """ + half_overlap = max((self.params.tile_size_px - self.stride_px) // 2 - 2*tile_offset, 0) + + # 'core' part of the tile (not overlapping with other tiles), for sure copied for each tile + x_min = self.start_pixel_x + half_overlap + x_max = self.start_pixel_x + self.params.tile_size_px - half_overlap - 1 + y_min = self.start_pixel_y + half_overlap + y_max = self.start_pixel_y + self.params.tile_size_px - half_overlap - 1 + + # edge tiles handling + if self.x_bin_number == 0: + x_min -= half_overlap + if self.y_bin_number == 0: + y_min -= half_overlap + if self.x_bin_number == self.x_bins_number-1: + x_max += half_overlap + if self.y_bin_number == self.y_bins_number-1: + y_max += half_overlap + + x_min += tile_offset + x_max -= tile_offset + y_min += tile_offset + y_max -= tile_offset + + roi_slice = np.s_[:, y_min:y_max + 1, x_min:x_max + 1] + return roi_slice + + def get_slice_on_tile_image_for_copying(self, roi_slice_on_full_image=None, tile_offset: int = 0): + """ + Similar to _get_slice_on_full_image_for_copying, but ROI is a slice on the tile + """ + if not roi_slice_on_full_image: + roi_slice_on_full_image = self.get_slice_on_full_image_for_copying(tile_offset=tile_offset) + + r = roi_slice_on_full_image + roi_slice_on_tile = np.s_[ + :, + r[1].start - self.start_pixel_y - tile_offset:r[1].stop - self.start_pixel_y - tile_offset, + r[2].start - self.start_pixel_x - tile_offset:r[2].stop - self.start_pixel_x - tile_offset + ] + return roi_slice_on_tile + + def is_tile_within_mask(self, mask_img: Optional[np.ndarray]): + """ + To check if tile is within the mask image + """ + if mask_img is None: + return True # if we don't have a mask, we are going to process all tiles + + roi_slice = self.get_slice_on_full_image_for_copying() + mask_roi = mask_img[roi_slice[0]] + # check corners first + if mask_roi[0, 0] and mask_roi[1, -1] and mask_roi[-1, 0] and mask_roi[-1, -1]: + return True # all corners in mask, almost for sure a good tile + + coverage_percentage = cv2.countNonZero(mask_roi) / (mask_roi.shape[0] * mask_roi.shape[1]) * 100 + return coverage_percentage > 0 # TODO - for training we can use tiles with higher coverage only + + def set_mask_on_full_img(self, full_result_img, tile_result): + if tile_result.shape[1] != self.params.tile_size_px or tile_result.shape[2] != self.params.tile_size_px: + tile_offset = (self.params.tile_size_px - tile_result.shape[1])//2 + + if tile_offset % 2 != 0: + raise Exception("Model output shape is not even, cannot calculate offset") + + if tile_offset < 0: + raise Exception("Model output shape is bigger than tile size, cannot calculate offset") + else: + tile_offset = 0 + + roi_slice_on_full_image = self.get_slice_on_full_image_for_copying(tile_offset=tile_offset) + roi_slice_on_tile_image = self.get_slice_on_tile_image_for_copying(roi_slice_on_full_image, tile_offset=tile_offset) + + full_result_img[roi_slice_on_full_image] = tile_result[roi_slice_on_tile_image] + + def get_entire_tile_from_full_img(self, full_result_img) -> np.ndarray: + roi_slice_on_full_image = self.get_slice_on_full_image_for_entire_tile() + img = full_result_img[roi_slice_on_full_image] + return img diff --git a/zipdeepness/python_requirements/requirements.txt b/zipdeepness/python_requirements/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3156cf5aa5dc05772f4868c1008f25b5fea709c1 --- /dev/null +++ b/zipdeepness/python_requirements/requirements.txt @@ -0,0 +1,10 @@ +## CORE ## +#numpy # available already from qgis repo +#pyqt5 # available already from qgis repo + + +# NOTE - for the time being - keep the same packages and versions in the `packages_installer_dialog.py` +numpy<2.0.0 +onnxruntime-gpu>=1.12.1,<=1.17.0 +opencv-python-headless>=4.5.5.64,<=4.9.0.80 +rasterio \ No newline at end of file diff --git a/zipdeepness/python_requirements/requirements_development.txt b/zipdeepness/python_requirements/requirements_development.txt new file mode 100644 index 0000000000000000000000000000000000000000..cb7da2e1bef6d85feffd344a2d1555bda3d6a4cc --- /dev/null +++ b/zipdeepness/python_requirements/requirements_development.txt @@ -0,0 +1,21 @@ +# Install main requirements +-r requirements.txt + +# Requirements required only to test and debug code while developing +pytest==7.4.0 +pytest-cov==4.1.0 +flake8==6.0.0 +onnx==1.14.0 + +# docs +sphinx==6.1.3 +sphinxcontrib-youtube==1.2.0 +sphinx-autodoc-typehints==1.22 +sphinx-autopackagesummary==1.3 +sphinx-rtd-theme==0.5.1 +m2r2==0.3.3 +sphinx-rtd-size==0.2.0 + +# vscode +pylint==2.17.4 +autopep8==2.0.2 diff --git a/zipdeepness/resources.py b/zipdeepness/resources.py new file mode 100644 index 0000000000000000000000000000000000000000..f434ed642379ecd17d8b9a73cc1eeeebcf82bcec --- /dev/null +++ b/zipdeepness/resources.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- + +# Resource object code +# +# Created by: The Resource Compiler for PyQt5 (Qt v5.15.3) +# +# WARNING! All changes made in this file will be lost! + +from PyQt5 import QtCore + +qt_resource_data = b"\ +\x00\x00\x02\x32\ +\x89\ +\x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52\x00\ +\x00\x00\x17\x00\x00\x00\x18\x08\x06\x00\x00\x00\x11\x7c\x66\x75\ +\x00\x00\x00\x04\x73\x42\x49\x54\x08\x08\x08\x08\x7c\x08\x64\x88\ +\x00\x00\x01\xe9\x49\x44\x41\x54\x48\x89\xa5\x95\xdf\x6a\x13\x51\ +\x10\xc6\x7f\x67\xb2\xa6\xa4\xa2\x5e\x14\x1a\x08\x8a\x4d\xad\x8b\ +\x15\xd4\x2b\x7d\x80\x7a\x21\x78\xe9\x75\x9f\xc1\x27\x28\xb4\x0f\ +\xe1\x23\xf4\xce\x27\x10\xb4\x0f\xa0\x05\x2f\x0a\x0d\x04\xd3\x82\ +\xa4\x42\x85\x52\x2c\x92\x54\x76\xcf\x1c\x2f\x36\xc5\xa4\xe7\x4f\ +\x9b\xf8\x41\x60\x99\x73\xe6\x9b\xef\x9b\x9d\xd9\x18\x2e\x43\x97\ +\x14\x63\x8c\x17\xbf\x80\xc3\x21\x87\x12\x3d\x1f\x83\x7f\xe9\xd4\ +\x9c\x26\x33\x0c\x06\xdb\xb6\xb3\x91\x2f\x1c\x2e\xa0\xe8\x15\x59\ +\x82\xb6\xb5\xfa\x2d\x29\xb6\x6d\x59\xbf\xb5\xee\xeb\xc8\x75\x44\ +\xa4\x4a\xd1\xef\x73\xb2\xb1\xc1\xd9\xf6\x36\xb6\x6d\x91\x40\xf1\ +\x14\x8e\xf4\x88\x96\xb4\xc6\xc8\x9d\xf3\x6f\x95\x25\xfb\xab\x42\ +\xcd\x4e\x47\x7e\x09\x91\xe4\x2c\xe3\xe3\xab\xff\x22\x4e\x90\x03\ +\x6f\xdf\xc1\x87\xd7\xe0\x4c\xc0\xd9\xf5\x60\x4c\xae\x0e\xc0\x11\ +\x9f\x3e\x3a\xb9\x22\xe5\x74\x4e\x1c\x4e\xb4\x2b\x68\x57\x58\x1b\ +\xee\x20\xb1\x21\x59\xed\x0a\xc5\x5c\x7a\x82\xc6\x48\xf9\xde\x1c\ +\xf2\xa8\x87\x71\x30\x61\xfb\x4e\xfe\x8b\x33\x6e\x5f\xdf\x81\x1b\ +\xcb\xff\x53\xb7\x3c\xdb\x17\x10\x01\x7c\x72\x80\x07\xcb\x3d\x0e\ +\xb2\xe5\x78\x01\x53\x56\x3d\x2c\x32\xe5\xc9\x5e\x09\xf5\x7a\x75\ +\x38\xb9\xd9\x41\x72\x80\x17\xf7\x3f\xf3\x65\xee\x79\xb8\x00\x45\ +\x01\x65\x09\x8d\x46\xe4\x42\x9a\x1c\xe0\xe5\xbd\x4f\xec\x34\xd6\ +\x52\xf9\x49\x18\xa5\x7a\x8b\x86\xf0\xb8\xa4\x1d\x5c\x41\x7e\xf1\ +\x30\x80\x41\x03\x82\x36\x9b\x2b\xc7\xfc\x94\xc5\xd9\xc9\x01\x14\ +\x34\xe6\x60\x3e\x1f\x30\x0c\xd7\x8e\x62\x62\xac\x36\x61\x33\x76\ +\x71\xd0\x9d\x8f\xef\x41\x04\x9e\xca\x94\x7a\x00\xc9\x35\xbd\xcd\ +\x7b\x8f\xe1\xc6\x39\x60\xa6\xfc\xa4\x02\x2d\xfb\x23\x7e\xd8\xc9\ +\xa1\x7e\x5e\x49\x33\xce\x27\xdf\x85\xdd\x14\x79\xbf\x77\x17\xe3\ +\x4d\xaf\x73\xbc\x7f\x03\x52\x4e\x44\x83\xfe\x66\x6a\x4d\xe7\xa1\ +\x43\xec\x44\x30\xd8\x96\x98\x7a\x05\x1d\x2d\x9d\xbf\x78\xc6\x7a\ +\x62\xa2\xea\x2c\x58\x09\x14\xb7\x60\xb3\x95\xe3\x13\x64\xf1\xdf\ +\xe0\x77\x72\xaf\x25\x51\xe5\x00\x5b\xb0\x15\x8a\xd7\xa0\xf6\xfb\ +\x5b\xf3\x26\x8c\xfe\x7b\xbf\x3e\x0d\x12\x03\xfc\x05\xb8\x94\xac\ +\x7e\xe6\x88\x91\xed\x00\x00\x00\x00\x49\x45\x4e\x44\xae\x42\x60\ +\x82\ +" + +qt_resource_name = b"\ +\x00\x07\ +\x07\x3b\xe0\xb3\ +\x00\x70\ +\x00\x6c\x00\x75\x00\x67\x00\x69\x00\x6e\x00\x73\ +\x00\x1b\ +\x02\xd8\xc9\xeb\ +\x00\x64\ +\x00\x65\x00\x65\x00\x70\x00\x5f\x00\x73\x00\x65\x00\x67\x00\x6d\x00\x65\x00\x6e\x00\x74\x00\x61\x00\x74\x00\x69\x00\x6f\x00\x6e\ +\x00\x5f\x00\x66\x00\x72\x00\x61\x00\x6d\x00\x65\x00\x77\x00\x6f\x00\x72\x00\x6b\ +\x00\x06\ +\x07\x03\x7d\xc3\ +\x00\x69\ +\x00\x6d\x00\x61\x00\x67\x00\x65\x00\x73\ +\x00\x08\ +\x0a\x61\x5a\xa7\ +\x00\x69\ +\x00\x63\x00\x6f\x00\x6e\x00\x2e\x00\x70\x00\x6e\x00\x67\ +" + +qt_resource_struct_v1 = b"\ +\x00\x00\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x01\ +\x00\x00\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x02\ +\x00\x00\x00\x14\x00\x02\x00\x00\x00\x01\x00\x00\x00\x03\ +\x00\x00\x00\x50\x00\x02\x00\x00\x00\x01\x00\x00\x00\x04\ +\x00\x00\x00\x62\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\ +" + +qt_resource_struct_v2 = b"\ +\x00\x00\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x01\ +\x00\x00\x00\x00\x00\x00\x00\x00\ +\x00\x00\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x02\ +\x00\x00\x00\x00\x00\x00\x00\x00\ +\x00\x00\x00\x14\x00\x02\x00\x00\x00\x01\x00\x00\x00\x03\ +\x00\x00\x00\x00\x00\x00\x00\x00\ +\x00\x00\x00\x50\x00\x02\x00\x00\x00\x01\x00\x00\x00\x04\ +\x00\x00\x00\x00\x00\x00\x00\x00\ +\x00\x00\x00\x62\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\ +\x00\x00\x01\x82\xb5\xcc\x01\x44\ +" + +qt_version = [int(v) for v in QtCore.qVersion().split('.')] +if qt_version < [5, 8, 0]: + rcc_version = 1 + qt_resource_struct = qt_resource_struct_v1 +else: + rcc_version = 2 + qt_resource_struct = qt_resource_struct_v2 + +def qInitResources(): + QtCore.qRegisterResourceData(rcc_version, qt_resource_struct, qt_resource_name, qt_resource_data) + +def qCleanupResources(): + QtCore.qUnregisterResourceData(rcc_version, qt_resource_struct, qt_resource_name, qt_resource_data) + +qInitResources() diff --git a/zipdeepness/resources.qrc b/zipdeepness/resources.qrc new file mode 100644 index 0000000000000000000000000000000000000000..5385347c5a77e41690a1a5361565f436d3be57bc --- /dev/null +++ b/zipdeepness/resources.qrc @@ -0,0 +1,5 @@ + + + images/icon.png + + diff --git a/zipdeepness/widgets/__init__.py b/zipdeepness/widgets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/zipdeepness/widgets/input_channels_mapping/input_channels_mapping_widget.py b/zipdeepness/widgets/input_channels_mapping/input_channels_mapping_widget.py new file mode 100644 index 0000000000000000000000000000000000000000..e9633d7cbbd3d4628b09c6036957b0090daffcb9 --- /dev/null +++ b/zipdeepness/widgets/input_channels_mapping/input_channels_mapping_widget.py @@ -0,0 +1,177 @@ +""" +This file contains a single widget, which is embedded in the main dockwiget - to select channels mapping +""" + + +import os +from typing import Optional, List + +from qgis.PyQt import QtWidgets, uic +from qgis.PyQt.QtWidgets import QComboBox +from qgis.PyQt.QtWidgets import QLabel +from qgis.core import Qgis, QgsRasterLayer + +from deepness.common.channels_mapping import ChannelsMapping, ImageChannelStandaloneBand, \ + ImageChannelCompositeByte +from deepness.common.channels_mapping import ImageChannel +from deepness.common.config_entry_key import ConfigEntryKey +from deepness.processing.models.model_base import ModelBase + +FORM_CLASS, _ = uic.loadUiType(os.path.join( + os.path.dirname(__file__), 'input_channels_mapping_widget.ui')) + + +class InputChannelsMappingWidget(QtWidgets.QWidget, FORM_CLASS): + """ + Widget responsible for mapping image channels to model input channels. + Allows to define the channels mapping (see `deepness.common.channels_mapping` for details). + + UI design defined in `input_channels_mapping_widget.ui` file. + """ + + def __init__(self, rlayer, parent=None): + super(InputChannelsMappingWidget, self).__init__(parent) + self.setupUi(self) + self._model_wrapper: Optional[ModelBase] = None + self._rlayer: QgsRasterLayer = rlayer + self._create_connections() + self._channels_mapping = ChannelsMapping() + + self._channels_mapping_labels: List[QLabel] = [] + self._channels_mapping_comboboxes: List[QComboBox] = [] + + self._selection_mode_changed() + + def load_ui_from_config(self): + is_advanced_mode = ConfigEntryKey.INPUT_CHANNELS_MAPPING__ADVANCED_MODE.get() + self.radioButton_advancedMapping.setChecked(is_advanced_mode) + + if is_advanced_mode: + mapping_list_str = ConfigEntryKey.INPUT_CHANNELS_MAPPING__MAPPING_LIST_STR.get() + mapping_list = [int(v) for v in mapping_list_str] + self._channels_mapping.load_mapping_from_list(mapping_list) + + def save_ui_to_config(self): + is_advanced_mode = self.radioButton_advancedMapping.isChecked() + ConfigEntryKey.INPUT_CHANNELS_MAPPING__ADVANCED_MODE.set(is_advanced_mode) + + if is_advanced_mode: + mapping_list = self._channels_mapping.get_mapping_as_list() + mapping_list_str = [str(v) for v in mapping_list] + ConfigEntryKey.INPUT_CHANNELS_MAPPING__MAPPING_LIST_STR.set(mapping_list_str) + + def get_channels_mapping(self) -> ChannelsMapping: + """ Get the channels mapping currently selected in the UI """ + if self.radioButton_defaultMapping.isChecked(): + return self._channels_mapping.get_as_default_mapping() + else: # advanced mapping + return self._channels_mapping + + def get_channels_mapping_for_training_data_export(self) -> ChannelsMapping: + """ Get the channels mapping to be used for the `training data export tool`. + It is not channels mapping exactly, because we do not have the model, but we can use it + """ + mapping = self._channels_mapping.get_as_default_mapping() + mapping.set_number_of_model_inputs_same_as_image_channels() + return mapping + + def _create_connections(self): + self.radioButton_defaultMapping.clicked.connect(self._selection_mode_changed) + self.radioButton_advancedMapping.clicked.connect(self._selection_mode_changed) + + def _selection_mode_changed(self): + is_advanced = self.radioButton_advancedMapping.isChecked() + self.widget_mapping.setVisible(is_advanced) + + def set_model(self, model_wrapper: ModelBase): + """ Set the model for which we are creating the mapping here """ + self._model_wrapper = model_wrapper + number_of_channels = self._model_wrapper.get_number_of_channels() + self.label_modelInputs.setText(f'{number_of_channels}') + self._channels_mapping.set_number_of_model_inputs(number_of_channels) + self.regenerate_mapping() + + def set_rlayer(self, rlayer: QgsRasterLayer): + """ Set the raster layer (ortophoto file) which is selected (for which we create the mapping here)""" + self._rlayer = rlayer + + if rlayer: + number_of_image_bands = rlayer.bandCount() + else: + number_of_image_bands = 0 + + image_channels = [] # type: List[ImageChannel] + + if number_of_image_bands == 1: + # if there is one band, then there is probably more "bands" hidden in a more complex data type (e.g. RGBA) + data_type = rlayer.dataProvider().dataType(1) + if data_type in [Qgis.DataType.Byte, Qgis.DataType.UInt16, Qgis.DataType.Int16, + Qgis.DataType.Float32]: + image_channel = ImageChannelStandaloneBand( + band_number=1, + name=rlayer.bandName(1)) + image_channels.append(image_channel) + elif data_type == Qgis.DataType.ARGB32: + # Alpha channel is at byte number 3, red is byte 2, ... - reversed order + band_names = [ + 'Alpha (band 4)', + 'Red (band 1)', + 'Green (band 2)', + 'Blue (band 3)', + ] + for i in [1, 2, 3, 0]: # We want order of model inputs as 'RGB' first and then 'A' + image_channel = ImageChannelCompositeByte( + byte_number=3 - i, # bytes are in reversed order + name=band_names[i]) + image_channels.append(image_channel) + else: + raise Exception("Invalid input layer data type!") + else: + for band_number in range(1, number_of_image_bands + 1): # counted from 1 + image_channel = ImageChannelStandaloneBand( + band_number=band_number, + name=rlayer.bandName(band_number)) + image_channels.append(image_channel) + + self.label_imageInputs.setText(f'{len(image_channels)}') + self._channels_mapping.set_image_channels(image_channels) + self.regenerate_mapping() + + def _combobox_index_changed(self, model_input_channel_number): + combobox = self._channels_mapping_comboboxes[model_input_channel_number] # type: QComboBox + image_channel_index = combobox.currentIndex() + # print(f'Combobox {model_input_channel_number} changed to {current_index}') + self._channels_mapping.set_image_channel_for_model_input( + model_input_number=model_input_channel_number, + image_channel_index=image_channel_index) + + def regenerate_mapping(self): + """ Regenerate the mapping after the model or ortophoto was changed or mapping was read from config""" + for combobox in self._channels_mapping_comboboxes: + self.gridLayout_mapping.removeWidget(combobox) + self._channels_mapping_comboboxes.clear() + for label in self._channels_mapping_labels: + self.gridLayout_mapping.removeWidget(label) + self._channels_mapping_labels.clear() + + for model_input_channel_number in range(self._channels_mapping.get_number_of_model_inputs()): + label = QLabel(self) + label.setText(f"Model input {model_input_channel_number}:") + combobox = QComboBox(self) + for image_channel in self._channels_mapping.get_image_channels(): + combobox.addItem(image_channel.name) + + if self._channels_mapping.get_number_of_image_channels() > 0: + # image channel witch is currently assigned to the current model channel + image_channel_index = self._channels_mapping.get_image_channel_index_for_model_input( + model_input_channel_number) + combobox.setCurrentIndex(image_channel_index) + + combobox.currentIndexChanged.connect( + lambda _, v=model_input_channel_number: self._combobox_index_changed(v)) + + self.gridLayout_mapping.addWidget(label, model_input_channel_number, 0) + self.gridLayout_mapping.addWidget(combobox, model_input_channel_number, 1, 1, 2) + + self._channels_mapping_comboboxes.append(combobox) + self._channels_mapping_labels.append(label) diff --git a/zipdeepness/widgets/input_channels_mapping/input_channels_mapping_widget.ui b/zipdeepness/widgets/input_channels_mapping/input_channels_mapping_widget.ui new file mode 100644 index 0000000000000000000000000000000000000000..c22efbe2bd93c5f5d3ae95ed9c7ade144e97f330 --- /dev/null +++ b/zipdeepness/widgets/input_channels_mapping/input_channels_mapping_widget.ui @@ -0,0 +1,100 @@ + + + Form + + + + 0 + 0 + 399 + 378 + + + + Form + + + + + + + + Model inputs (channels): + + + + + + + Image inputs (bands): + + + + + + + ... + + + + + + + ... + + + + + + + + + Qt::Horizontal + + + + + + + Default (image channels passed +in sequence as input channels) + + + true + + + + + + + Advanced (manually select which input image +channel is assigned to each model input) + + + + + + + + 0 + + + 0 + + + 0 + + + 0 + + + + + + + + + + + + diff --git a/zipdeepness/widgets/training_data_export_widget/training_data_export_widget.py b/zipdeepness/widgets/training_data_export_widget/training_data_export_widget.py new file mode 100644 index 0000000000000000000000000000000000000000..0854aa2a55453b25ce2c7a4fb48d0fc7a79cc7ea --- /dev/null +++ b/zipdeepness/widgets/training_data_export_widget/training_data_export_widget.py @@ -0,0 +1,96 @@ +""" +This file contains a single widget, which is embedded in the main dockwiget - to select the training data export parameters +""" + +import os + +from qgis.PyQt import QtWidgets, uic +from qgis.PyQt.QtWidgets import QFileDialog +from qgis.core import QgsMapLayerProxyModel +from qgis.core import QgsProject + +from deepness.common.config_entry_key import ConfigEntryKey +from deepness.common.processing_parameters.map_processing_parameters import MapProcessingParameters +from deepness.common.processing_parameters.training_data_export_parameters import \ + TrainingDataExportParameters + +FORM_CLASS, _ = uic.loadUiType(os.path.join( + os.path.dirname(__file__), 'training_data_export_widget.ui')) + + +class TrainingDataExportWidget(QtWidgets.QWidget, FORM_CLASS): + """ + Widget responsible for defining the parameters for the Trainign Data Export process (not doing the actual export). + + UI design defined in the `training_data_export_widget.ui` file. + """ + + def __init__(self, rlayer, parent=None): + super(TrainingDataExportWidget, self).__init__(parent) + self.setupUi(self) + self._create_connections() + self._setup_misc_ui() + + def load_ui_from_config(self): + layers = QgsProject.instance().mapLayers() + + self.lineEdit_outputDirPath.setText(ConfigEntryKey.DATA_EXPORT_DIR.get()) + self.checkBox_exportImageTiles.setChecked( + ConfigEntryKey.DATA_EXPORT_TILES_ENABLED.get()) + self.checkBox_exportMaskEnabled.setChecked( + ConfigEntryKey.DATA_EXPORT_SEGMENTATION_MASK_ENABLED.get()) + + segmentation_layer_id = ConfigEntryKey.DATA_EXPORT_SEGMENTATION_MASK_ID.get() + if segmentation_layer_id and segmentation_layer_id in layers: + self.mMapLayerComboBox_inputLayer.setLayer(layers[segmentation_layer_id]) + + def save_ui_to_config(self): + ConfigEntryKey.DATA_EXPORT_DIR.set(self.lineEdit_outputDirPath.text()) + ConfigEntryKey.DATA_EXPORT_TILES_ENABLED.set( + self.checkBox_exportImageTiles.isChecked()) + ConfigEntryKey.DATA_EXPORT_SEGMENTATION_MASK_ENABLED.set( + self.checkBox_exportMaskEnabled.isChecked()) + ConfigEntryKey.DATA_EXPORT_SEGMENTATION_MASK_ID.set(self.get_segmentation_mask_layer_id()) + + def _browse_output_directory(self): + current_directory = self.lineEdit_outputDirPath.text() + if not current_directory: + current_directory = os.path.expanduser('~') + new_directory = QFileDialog.getExistingDirectory( + self, + "Select Directory", + current_directory, + QFileDialog.ShowDirsOnly) + self.lineEdit_outputDirPath.setText(new_directory) + + def _enable_disable_mask_layer_selection(self): + is_enabled = self.checkBox_exportMaskEnabled.isChecked() + self.mMapLayerComboBox_maskLayer.setEnabled(is_enabled) + + def _create_connections(self): + self.checkBox_exportMaskEnabled.toggled.connect(self._enable_disable_mask_layer_selection) + self.pushButton_browseOutputDirectory.clicked.connect(self._browse_output_directory) + + def _setup_misc_ui(self): + self.mMapLayerComboBox_maskLayer.setFilters(QgsMapLayerProxyModel.VectorLayer) + self._enable_disable_mask_layer_selection() + + def get_segmentation_mask_layer_id(self): + if not self.checkBox_exportMaskEnabled.isChecked(): + return None + return self.mMapLayerComboBox_maskLayer.currentLayer().id() + + def get_training_data_export_parameters(self, map_processing_parameters: MapProcessingParameters): + """ Get the parameters from the UI for the data exporting process""" + if self.checkBox_exportMaskEnabled.isChecked(): + segmentation_mask_layer_id = self.mMapLayerComboBox_maskLayer.currentLayer().id() + else: + segmentation_mask_layer_id = None + + params = TrainingDataExportParameters( + **map_processing_parameters.__dict__, + export_image_tiles=self.checkBox_exportImageTiles.isChecked(), + segmentation_mask_layer_id=segmentation_mask_layer_id, + output_directory_path=self.lineEdit_outputDirPath.text(), + ) + return params diff --git a/zipdeepness/widgets/training_data_export_widget/training_data_export_widget.ui b/zipdeepness/widgets/training_data_export_widget/training_data_export_widget.ui new file mode 100644 index 0000000000000000000000000000000000000000..9540d968c4003af591b9701aac707dba90ff2069 --- /dev/null +++ b/zipdeepness/widgets/training_data_export_widget/training_data_export_widget.ui @@ -0,0 +1,133 @@ + + + Form + + + + 0 + 0 + 461 + 378 + + + + Form + + + + + + + + + + + Resolution [cm/px]: + + + + + + + Path to directory where the generated data will be saved. + + + + + + + Whether tiles from the "Input layer" (the main ortophoto) should be created. + + + Export image tiles: + + + true + + + + + + + Browse... + + + + + + + Input layer selected in +"Input Layer" section + + + + + + + Tile size [px]: + + + + + + + Tiles overlap [%]: + + + + + + + Output dir path: + + + + + + + <html><head/><body><p>Whether tiles from the segmentation mask should be created.</p><p>Can be used to create training images for the model, if you already have manually annotated the image.</p></body></html> + + + Export segmentation +mask for layer: + + + + + + + Selected in section +"Processing parameters" + + + + + + + Selected in section +"Processing parameters" + + + + + + + Selected in section +"Processing parameters" + + + + + + + + + + QgsMapLayerComboBox + QComboBox +
qgsmaplayercombobox.h
+ + + + +