SightLinks-Dev-main/DevelopmentTools/experimental_resources/FeatureExtractionMethods.py

# Feature Extraction Experimental Methods - Our method actually does not signficiantly benefit from 
# common preprocessing techniques, so we created a set of more uncommon/ niche methods as well as some
#  other untested methods to experiment with which features best represent the crosswalk features.

# This hopefully should be of great use to anyone trying to train their own specialised classifier using 
# our pipeline - hopefully one of these works for you!

# All these methods work on a numpy array - if you want to use them convert your image to numpy first
# Most of the methods assume the channels of the image are at the end

import numpy as np
from scipy.signal import convolve2d
import skimage.feature as skf
from cv2 import NMSBoxes

from scipy.ndimage import maximum_filter
from collections import deque



# Was required for keeping several of the methods optimised.
def matrixConvolution(image, kernel, padding=True):
    convolvedImage = np.zeros_like(image)

    # Again, this assumes the channels are placed at the end of the image
    for c in range(image.shape[2]):
        convolvedImage[..., c] = convolve2d(image[..., c], kernel, mode='same', boundary='wrap')

    return convolvedImage


# Different more traditional image processing methods, these may not be of particular use in our project, 
# but could be useful for other developers using this pipeline.
class ImageProcessingFeatures:
    def __init__(self):
        self.gaussianKernel = self.generateGaussianKernel(5, 1)

    # Converts an image to grayscale - but using binary thresholding of the averaged colour channels
    # Image in format [n, m, k] where k is the channels and n, m are the dimensions
    def binaryThresholding(self, image, threshold):

        averaged = np.mean(image, axis=2)
        thresholded = averaged > threshold

        return thresholded
    
    # A bit unintuitively named, but this takes an already grayscale image and applies binary thresholding to it
    def grayscaleBinaryThresholding(self, grayscaleImage, threshold):
        return grayscaleImage > threshold

    
    # Converts an image to grayscale - it has several possible schema it can use but defaults to the lightness method
    def grayscaleConversion(self, image, schema="lightness"):
        # https://tannerhelland.com/2011/10/01/grayscale-image-algorithm-vb6.html
        grayImage = np.zeros(np.shape(image)[:-1]) 

        if schema == "average":
            grayImage = np.mean(image, axis=2)

        if schema == "lightness":
            # (max(R, G, B) + min(R, G, B)) / 2 --> Sometimes called desaturation
            grayImage = (np.max(image, axis=2) + np.min(image, axis=2)) / 2

        if schema == "luma":
            # I assume RGB colour ordering in the image here -- feel free to overwrite in your implementation
            # (Red * 0.2126 + Green * 0.7152 + Blue * 0.0722)
            colourWeighting = np.array([0.2126, 0.7152, 0.0722])
            grayImage = np.dot(image[..., :3], colourWeighting)

        if schema == "decomposition":
            # This is a maximum decomposition - minumum decomposition can be implemented by just switching our for min...
            grayImage = np.max(image, axis=2)

        return grayImage


    # Sharpens the image by convolving a preset laplace kernel with it
    # My chosen kernel is a pretty standard one that takes into consideration the corners - the 8 adjacencies one
    # ​[-1, -1, -1], [-1, 8, -1], [-1, -1, -1]
    def laplaceTransform(self, image):
        laplaceKernel=[[-1, -1, -1], [-1, 8, -1], [-1, -1, -1]]
        convolvedImage = None

        # RGB image
        if len(np.shape(image)) == 3:
            convolvedImage = matrixConvolution(image, laplaceKernel)

        # GrayScale image
        if len(np.shape(image)) == 2:
            convolvedImage = convolve2d(image, laplaceKernel, mode='same', boundary='wrap')

        return convolvedImage

    # Assumes Grayscale images
    # https://en.wikipedia.org/wiki/Sobel_operator - naive implementation by me, can probably be massively improved
    # Basically takes the vertical and horizontal gradients using convolution with a sobel operator and combines them.
    def sobelConvolution(self, image):
        # Gx
        verticalKernel = [[-1, 0, 1],
                            [-2, 0, 2],
                            [-1, 0, 1]]

        # Gy
        horizontalKernel = [[-1, -2, -1],
                        [0, 0, 0],
                        [1, 2, 1]] 

        horizontalGrad = convolve2d(image, horizontalKernel, mode='same', boundary='wrap')
        verticalGrad = convolve2d(image, verticalKernel, mode='same', boundary='wrap')

        sobelConvolvedImage = np.sqrt(np.square(horizontalGrad) + np.square(verticalGrad))

        # You could go further to find the gradient direction by calculating angle with some trigonometry - atan2(Gy, Gx)
        return sobelConvolvedImage
    
    # Part of the calculations for canny edge detection, a tri-threshold operations. Requires greyscale images.
    # Classifies into strong edges and weak edges based on pixel intensity
    def doubleFiltering(self, image, weakThreshold=75, strongThreshold=200):
        weak, strong = 125, 255

        strong_edges = image >= strongThreshold
        weak_edges = (image >= weakThreshold) & (image < strongThreshold)
        
        result = np.zeros_like(image, dtype=np.uint8)
        result[strong_edges] = strong
        result[weak_edges] = weak
        
        return result, strong, weak
    
    # Technical name: edge tracking by hysteresis. This is an 'optimised' dequeue approach (to the best of my ability)
    def followEdges(self, weakEdges, strongEdges):
        h, w = weakEdges.shape
        directions = [(-1, -1), (-1, 0), (0, -1), (1, -1), (-1, 1), (0, 0), (0, 1), (1, 0), (1, 1)]
        finalEdges = (strongEdges.copy() > 0) # must include strong edges, it's guaranteed, and we want a binary output.
        edgeQueue = deque(np.argwhere(strongEdges == 1)) 
        
        while edgeQueue:
            y, x = edgeQueue.popleft()  
            for dy, dx in directions:
                ny, nx = y + dy, x + dx
                
                # Expand the branch if they're connected to a weak edge, else kill this branch
                if 0 <= ny < h and 0 <= nx < w and weakEdges[ny, nx] > 0:
                    weakEdges[ny, nx] = 0
                    finalEdges[ny, nx] = 1  
                    edgeQueue.append((ny, nx))
        
        return finalEdges


    # Much better than sobel in terms of accuracy and removing false edges produced due to noise, but slower.
    # https://en.wikipedia.org/wiki/Canny_edge_detector + a lot of chatGPT prompts
    def cannyEdgeDetection(self, image):
        # Ensures that the image is in greyscale so we don't have any issues.
        if (np.shape(image) == 3):
            image = ImageProcessingFeatures.grayscaleConversion(image, schema="average")

        # Takes a greyscale image and returns a set of strong and weak edges (see wikipedia page)
        blurred = convolve2d(image, self.gaussianKernel, mode='same', boundary='wrap')
        gradientMagnitude = self.sobelConvolution(blurred)
        maxFiltered = maximum_filter(gradientMagnitude, size=3, mode='constant')
        suppressed = np.where(gradientMagnitude==maxFiltered, gradientMagnitude, 0)
        thresholded, strong, weak = self.doubleFiltering(suppressed)

        final_edges = self.followEdges(strong, weak)
        # We expand the definition of strong edges to include weak edges that are adjacent to strong edges

        return final_edges

    
    # Convolve this with the image to produce a Gaussian Blur effect 
    # In a real application you should precompute this, it's really unoptimised
    def generateGaussianKernel(self, size, sigma):
        center = size // 2
        kernel = np.zeros((size, size))

        for i in range(size):
            for j in range(size):
                sides = np.sqrt((i - center) ** 2 + (j - center) ** 2)
                kernel[i, j] = np.exp(-(sides ** 2) / (2 * sigma ** 2))  # According to the formula

        return kernel / np.sum(kernel)


    # This should be applied to a grayscale image! 
    def differenceOfGaussians(self, image, size=5, sigmaOne=1, sigmaTwo=1.5, brightFeatureFocus=True):
        # This by default uses a 5x5 kernel with sigmaOne = 1 and sigmaTwo = 2. These are not finetuned values but follow the general
        # principle that sigma should not be greater than approx. 3*dims. Be careful to consider the relationship between the dimension
        # and sigma values (and between the two sigmas themselves) to preserve the gaussian property!
        kernelOne, kernelTwo = self.generateGaussianKernel(size, sigmaOne), self.generateGaussianKernel(size, sigmaTwo)

        primaryImage = convolve2d(image, kernelOne, mode='same', boundary='wrap')
        backgroundImage = convolve2d(image, kernelTwo, mode='same', boundary='wrap')

        diffOfGaussians = primaryImage - backgroundImage
        
        # Enhances bright feature edges (our focus in crosswalks), but this is a thing that might vary based on your focus. Can be disabled!
        if brightFeatureFocus:
            diffOfGaussians = diffOfGaussians[diffOfGaussians > 0]

        return diffOfGaussians


# Different feature extraction methods that attempt to quantify the complexity in an image. 
# Generally seperated into local complexity per region, and complexity of an image as a whole.
# Intended to potentially detect the occlusion in an image, e.g. more treees --> more complexity hopefully.
class ComplexityFeatures:
    def __init__(self):
        pass

    # Definition: Mathematical measure of how complex an image or pattern is
    # In our case (Grayscale image), it measures how much the detail in an image changes with the scale it is perceived at.
    # The particular method we will the basic box-counting method
    # Our definition of a useful feature map is a binary-thresholded line detection method using difference of Gaussians
    def fractalDimension(self, image, minimumBoxSize=2, imageStructureThreshold=0.9):

        if len(image.shape) > 2:
            image = ImageProcessingFeatures.grayscaleConversion(image, schema="average")

        image = ImageProcessingFeatures.sobelConvolution(image) # Could also go for DoG or Laplace too - most edge detectors work
        image = np.int_(255 * (image - np.min(image)) / (np.max(image) - np.min(image)))
        image = ImageProcessingFeatures.grayscaleBinaryThresholding(image, imageStructureThreshold) 

        N, M = image.shape

        startingBoxSize = min(N, M) // 4
        boxSizes = [s for s in range(minimumBoxSize, startingBoxSize + 1, 2) if min(N, M) % s == 0]
        logSizes, logCounts = [], []

        for boxSize in boxSizes:
            numBoxes = 0

            for i in range(0, N, boxSize):
                for j in range(0, M, boxSize):
                    if np.any(image[i:i+boxSize, j:j+boxSize]):  # Check if box contains a 1 (part of the image structure)
                        numBoxes += 1
                        
            if numBoxes > 0:
                logSizes.append(np.log(1.0 / boxSizes)) 
                logCounts.append(np.log(numBoxes))
        
        slope, _ = np.polyfit(logSizes, logCounts, 1)
        # Finds the relationship of the sizes and the image structure contained as the image scale decreases

        return slope

    # Come back to
    def waveletBasedFractalTransform(self, image):
        pass


# A set of feature extraction methods inspired by texture analysis methods
# just a wild throw in the dark, not sure how useful these could be as a feature
# Many of these should be applied in sliding window approaches or in regions, or to the whole image if you have a feature vector
class TextureFeatures:
    def __init__(self):
        pass

    def lbpCompare(self, threshold, value):
        return 0 if value < threshold else 1

    # A matter of personal preference, but this is a spiral concatenation for the local binary pattern signature generation
    # You could do row by row, but this is my preferred method. Feel free to overwrite, it shouldn't make a difference as long as you're consistent
    def spiral_concatenation(self, vals, dims):
        summed = []
        dirs = [(0, 1), (1, 0), (0, -1), (-1, 0)] 
        cur_dir = 0  
        cur_x, cur_y = 0, 0  
        bounds = dims  

        while bounds > 0:
            for i in range(bounds):
                summed.append(str(vals[cur_y * dims + cur_x])) 
                cur_y, cur_x = cur_y + dirs[cur_dir][0], cur_x + dirs[cur_dir][1]  
            cur_dir = (cur_dir + 1) % 4 
            
            bounds -= 1

            if bounds > 0:
                for i in range(bounds):
                    summed.append(str(vals[cur_y * dims + cur_x]))
                    cur_y, cur_x = cur_y + dirs[cur_dir][0], cur_x + dirs[cur_dir][1]
                cur_dir = (cur_dir + 1) % 4
                bounds -= 1

        return "".join(summed)

    # Basically captures the local binary changes in texture in an image - a potentially useful feature for our crosswalk detector that
    # works on a very similar principle with the local regions of interest found by the first.
    # It generates a signature for each local region that can be used to compare them quite easily in applications like texture analysis.
    # CAREFUL - THIS IS A STRING FEATURE, NOT A NUMERICAL ONE
    def localBinaryPattern(self, image, dims):
        imgWidth, imgLength = len(image[0]), len(image)
        edge = dims // 2 
        lbpList = []
        for row in range(edge, imgLength - edge):
            for pixel in range(edge, imgWidth - edge):
                neighborhood = image[row - edge: row + edge + 1, pixel - edge: pixel + edge + 1]
                
                centralPixel = image[row][pixel]
                binaryVals = [self.lbpCompare(centralPixel, val) for val in neighborhood.flatten()]

                lbpSignature = self.spiral_concatenation(binaryVals, dims)
                lbpList.append(lbpSignature)

        return lbpList

    # https://medium.com/@girishajmera/feature-extraction-of-images-using-glcm-gray-level-cooccurrence-matrix-e4bda8729498
    # Link above explains the function quite well and succinctly. This takes in both coloured (n, m, k) and grayscale arrays (n, m).
    # Captures the spatial relationships between neighbouring gray levels/ Intensities
    def grayLevelCoOccurenceMatrix(self, image, pixelOffset=5, preserveMatrix=False):
        if not (np.shape(image) == 2):
            # P.S - this assumes that RGB and [n, m, k] formats are followed. Can throw errors otherwise.
            image = ImageProcessingFeatures.grayscaleConversion(image, schema="average")

        transformedArray = np.int_(255 * (image - np.min(image)) / (np.max(image) - np.min(image)))

        distances = [pixelOffset]
        angles = [0, np.pi/4, np.pi/2, 3*np.pi/4]

        # A bit of a cop out, but this just does it all for us. The metrics you choose to extract from this depend.
        glcm = skf.graycomatrix(image, distances=distances, angles=angles, levels=256, symmetric=True, normed=True)

        # This is a set of features that I personally thought might be useful for the crosswalks - but there are many other extractable features
        contrast = skf.graycoprops(glcm, 'contrast')
        energy = skf.graycoprops(glcm, 'energy')
        homogeneity = skf.graycoprops(glcm, 'homogeneity')
        correlation = skf.graycoprops(glcm, 'correlation')

        # Single value metrics for the image
        metrics = (np.mean(contrast.flatten()), np.mean(energy.flatten()), np.mean(homogeneity.flatten()), np.mean(correlation.flatten()))

        if preserveMatrix:
            return metrics, (contrast, energy, homogeneity, correlation)
        
        return metrics