tools/metrics.py

import open3d as o3d
import torch
import numpy as np
import matplotlib.pyplot as plt
from math import atan2
from scipy.spatial.transform import Rotation as R
import warnings


def get_rotaion(est):
    # Extract the rotation matrices
    try:
        estimated_rotation = est[:3, :3]
    except TypeError:
        try:
            estimation_transformation = est.transformation # for open3d
        except AttributeError:
            try: 
                estimation_transformation = est.rot  # for Probreg
            except: # for learning 3d
                detached_est = est['est_T'].detach().cpu().numpy()[0]
                estimation_transformation = detached_est.reshape(4,4)
                
        estimated_rotation = estimation_transformation[:3, :3]

    return estimated_rotation

def get_translation(est):
    # Extract the translation vectors
    try:
        estimated_translation = est[:3, 3]
    except TypeError:
        try:
            estimated_translation = est.t  # for Probreg
        except AttributeError:
            try: 
                estimation_transformation = est.transformation # for open3d
            except: # for learning 3d
                detached_est = est['est_T'].detach().cpu().numpy()[0]
                estimation_transformation = detached_est.reshape(4,4)
            
            estimated_translation = estimation_transformation[:3, 3]

    return estimated_translation

def compute_rmse(result, target, corres):
    '''
    This function computes the root-mean-square error (RMSE) between the (transforemed) source and target point clouds based on the correspondences.
    Based on C++ code in Open3D https://github.com/isl-org/Open3D/blob/c8856fc0d4ec89f8d53591db245fd29ad946f9cb/cpp/open3d/pipelines/registration/TransformationEstimation.cpp#L20

    args:
        result (point cloud data): the transformed point cloud
        target (point cloud data): the target point cloud
        corres (Vector2iVector): the point-to-point correspondences between the source and target point clouds
    returns:
        rmse: the root-mean-square error (centimeters or meters based on the source and target point clouds)
    '''
    err = 0.0
    for c in corres:
        diff = np.asarray(result.points)[c[0]] - np.asarray(target.points)[c[1]] # Euclidean distance
        err += np.sum(diff**2)  # sum of squared distances
    rmse = np.sqrt(err / len(corres))  
    
    return rmse

def rotation_error(gt_transformation, est):
    ''' 
    This function computes the rotation error as the Geodesic distance between the estimated and the ground truth rotation matrices in 3D space.
    Based on formula on this page http://www.boris-belousov.net/2016/12/01/quat-dist/

    args:
        gt_transformation: the ground truth transformation (rotation matrix will be extracted from this)
        est: the estimated transformation (rotation matrix will be extracted from this)

    returns:
        rotation_error_deg: the rotation error (degrees)
    '''
    estimated_rotation = get_rotaion(est)
    ground_truth_rotation = gt_transformation[:3, :3]

    # Compute the angular distance between the two rotation matrices
    R = np.dot(ground_truth_rotation, estimated_rotation.T)  # Compute the relative rotation matrix (np.matmul(gt.T, est))       
    trace = np.trace(R)
    normalized_trace = min(max(((trace - 1) / 2), -1.0), 1.0) # Normalize and clamp the trace to avoid numerical errors
    theta = np.arccos(normalized_trace) 
    rotation_error = abs(theta)
    rotation_error_deg = np.rad2deg(rotation_error) # Convert to degrees

    return rotation_error_deg

def translation_error(gt_transformation, est):
    ''' 
    This function computes the translation error as the Euclidean distance between Ground Truth & Estimated translation.
    Based on definitions in this paper https://arxiv.org/pdf/2103.02690

    args:
        gt_transformation: the ground truth transformation (translation vector will be extracted from this)
        est: the estimated transformation (translation vector will be extracted from this)

    returns:
        abs(translation_error): the absolute translation error (centimeters)
    '''
    estimated_translation = get_translation(est)
    ground_truth_translation = gt_transformation[:3, 3]

    # Compute the translation error 
    translation_error = np.linalg.norm(estimated_translation - ground_truth_translation)

    return abs(translation_error)

def residual_error(cloud1: o3d.geometry.PointCloud, cloud2: o3d.geometry.PointCloud) -> float:
    '''
    This metric combines the rotation and translation errors so that different approaches can be compared.
    It uses root mean squared distance between homologous points of the source point cloud, after the execution of an algorithm, and the same point cloud at the ground truth pose.
    Based on papers: 
        https://www.sciencedirect.com/science/article/pii/S0921889021000191?via=ihub (3.3. Error metric) (main paper) -> https://github.com/iralabdisco/point_clouds_registration_benchmark/tree/master
        https://link.springer.com/article/10.1007/s11370-024-00562-1 [14]
    '''
    if len(cloud1.points) != len(cloud2.points):
        if len(cloud1.points) > len(cloud2.points):
            cloud1.points = cloud1.points[:len(cloud2.points)]
        else:
            cloud2.points = cloud2.points[:len(cloud1.points)]
    
    assert len(cloud1.points) == len(cloud2.points), "len(cloud1.points) != len(cloud2.points)"
    
    centroid, _ = cloud1.compute_mean_and_covariance()
    weights = np.linalg.norm(np.asarray(cloud1.points) - centroid, 2, axis=1)
    distances = np.linalg.norm(np.asarray(cloud1.points) - np.asarray(cloud2.points), 2, axis=1)/len(weights)
    return np.sum(distances/weights)

def chamfer_distance(pcd1, pcd2):
    '''
    Computes the Chamfer Distance between two point clouds using Open3D and PyTorch.

    The Chamfer Distance is a metric that measures the similarity between two point clouds.
    It is calculated as the sum of the mean squared distances from each point in one point cloud
    to its nearest neighbor in the other point cloud, in both directions.

    Args:
        pcd1 (o3d.geometry.PointCloud): The first point cloud.
        pcd2 (o3d.geometry.PointCloud): The second point cloud.

    Returns:
        chamfer_distance (float): The Chamfer distance between the two point clouds.
    '''
    dist1 = pcd1.compute_point_cloud_distance(pcd2)
    dist2 = pcd2.compute_point_cloud_distance(pcd1)
    dist1 = torch.tensor(np.asarray(dist1), dtype=torch.float32)
    dist2 = torch.tensor(np.asarray(dist2), dtype=torch.float32)
    
    chamfer_distance = torch.mean(dist1) + torch.mean(dist2)
    chamfer_distance = chamfer_distance.item()

    return chamfer_distance


def all_evaluations(source, target, result, time, gt_transformation = None, est_transformation = None, corres = None):

    cd = chamfer_distance(result, target)
    error = residual_error(target, result)
    computation_time = time

    max_treshold = 0.5
    inlier_rmse = o3d.pipelines.registration.evaluate_registration(result, target, max_treshold, np.eye(4)).inlier_rmse
    fitness = o3d.pipelines.registration.evaluate_registration(result, target, max_treshold, np.eye(4)).fitness

    if gt_transformation is not None:
        rmse = 1
        rotation_err = rotation_error(gt_transformation, est_transformation)
        translation_err = translation_error(gt_transformation, est_transformation)

        return rmse, rotation_err, translation_err, computation_time, cd, error, fitness, inlier_rmse #8
        
    else:
        return cd, fitness, inlier_rmse, computation_time #4
    

def summerize_results(results: np.ndarray) -> dict:
    if results.shape[2] == 8:
        mean_rmse = np.round(np.mean(results[:, :, 0]), 4)
        mean_rotation_error = np.round(np.mean(results[:, :, 1]), 4)
        mean_translation_error = np.round(np.mean(results[:, :, 2]), 4)
        mean_computation_time = np.round(np.mean(results[:, :, 3]), 4)
        mean_cd = np.round(np.mean(results[:, :, 4]), 4)
        mean_error = np.round(np.mean(results[:, :, 5]), 4)
        mean_fitness = np.round(np.mean(results[:, :, 6]), 4)
        mean_inlier_rmse = np.round(np.mean(results[:, :, 7]), 4)
        return {
            'mean_rmse': mean_rmse,
            'mean_rotation_error': mean_rotation_error,
            'mean_translation_error': mean_translation_error,
            'mean_computation_time': mean_computation_time,
            'mean_cd': mean_cd,
            'mean_error': mean_error,
            'mean_fitness': mean_fitness,
            'mean_inlier_rmse': mean_inlier_rmse
        }
    elif results.shape[2] == 5:
        mean_cd = np.round(np.mean(results[:, :, 0]), 4)
        mean_error = np.round(np.mean(results[:, :, 1]), 4)
        mean_fitness = np.round(np.mean(results[:, :, 2]), 4)
        mean_inlier_rmse = np.round(np.mean(results[:, :, 3]), 4)
        mean_computation_time = np.round(np.mean(results[:, :, 4]), 4)
        return {
            'mean_cd': mean_cd,
            'mean_error': mean_error,
            'mean_fitness': mean_fitness,
            'mean_inlier_rmse': mean_inlier_rmse,
            'mean_computation_time': mean_computation_time
        }
    else:   
        raise ValueError('Invalid results shape. Expected shape (N, M, 8) or (N, M, 5).')


def inlier_ratio(pcd, all_errors, threshold = 5):
    '''
    This function calculates the inlier ratio based on a given threshold.
    
    args:
        pcd (point cloud data): the point cloud
        all_errors (list): the errors between the two point clouds (from error_histogram)
        threshold (float): the threshold for inliers
        
    returns:
        inlier_ratio (float): the ratio of inliers (set at 5 cm)
        '''
    inliers = []
    for error in all_errors:
        # if the error is below the threshold, the pair is an inlier
        if error < threshold:
            inliers.append(error)
    inlier_ratio = len(inliers) / len(pcd.points)

    return inlier_ratio


def calculate_snr(clean_pcd, noisy_pcd):
    '''
    This function calculates the signal-to-noise ratio (SNR) between two point clouds.
    SNR is defined as the ratio of the RMS power of the signal (clean point cloud) to the RMS power of the noise (difference between clean and noisy point clouds).

    args:
        clean_pcd (point cloud): the clean point cloud
        noisy_pcd (point cloud): the noisy point cloud

    returns:
        snr_db (float): the signal-to-noise ratio in decibels
    '''
    # Convert point clouds to numpy arrays
    clean_points = np.asarray(clean_pcd.points)
    noisy_points = np.asarray(noisy_pcd.points)

    # Calculate the RMS power of the signal (clean point cloud)
    signal_amplitude = np.sqrt(np.mean(np.sum(clean_points**2, axis=1)))

    # Calculate the RMS power of the noise (difference between clean and noisy point clouds)
    noise_amplitude = np.sqrt(np.mean(np.sum((clean_points - noisy_points)**2, axis=1)))

    # Compute the SNR
    snr = (signal_amplitude / noise_amplitude)**2
    snr_db = 10 * np.log10(snr)

    return snr_db

def rotation_error_along_axis(gt_transformation, est_transformation, convention = 'zyx', verbose = False):
    '''
    This function calculates the rotation error along the each axis (Euler angle) between the estimated and ground truth rotation matrices.
    It gives a warning if gimbal lock is detected (90 or 270 degrees).

    Based on formulas and discussions in:
        https://www.youtube.com/watch?v=wg9bI8-Qx2Q (10:29)
    Args:
        gt_transformation: the ground truth transformation
        est_transformation: the estimated transformation
        convention: the convention for the Euler angles (default and only one supported is 'zyx')
        verbose: if True, the Euler angles are printed

    Returns:
        theta_x_deg: the rotation error along the x-axis (degrees) rounded to 3 decimal places
        theta_y_deg: the rotation error along the y-axis (degrees) rounded to 3 decimal places
        theta_z_deg: the rotation error along the z-axis (degrees) rounded to 3 decimal places
    '''
    if convention != 'zyx':
        raise ValueError("Invalid convention. Only 'zyx' is supported.")
    
    estimated_rotation = get_rotaion(est_transformation)
    ground_truth_rotation = gt_transformation[:3, :3]

    R_relative = np.dot(ground_truth_rotation, estimated_rotation.T) 

    theta_z = atan2(R_relative[1, 0], R_relative[0, 0]) # Euler angles = atan2(r21, r11)
    theta_x = atan2(R_relative[2, 1], R_relative[2, 2]) # Euler angles = atan2(r32, r33)

    if np.isclose(np.cos(theta_z), 0.0, atol=1e-6):
        second_term = R_relative[1, 0]/np.sin(theta_z)
        theta_y = atan2(-R_relative[2, 0], second_term)
    else:
        second_term = R_relative[0, 0]/np.cos(theta_z) 
        theta_y = atan2(-R_relative[2, 0], second_term)

    # Convert to degrees
    theta_x_deg = np.round(abs(np.rad2deg(theta_x)), 3)
    theta_y_deg = np.round(abs(np.rad2deg(theta_y)), 3)
    theta_z_deg = np.round(abs(np.rad2deg(theta_z)), 3)

    if verbose:
        if np.round(theta_y_deg, 3) == 90 or np.round(theta_y_deg, 3) == 270:
            print("Warning: Gimbal lock detected! It might not be possible to uniquely and accurately determine all angles.")

        print(f'{theta_x_deg}° error along x-axis, {theta_y_deg}° error along y-axis and {theta_z_deg}° error along z-axis.')

    return theta_x_deg, theta_y_deg, theta_z_deg

def angle_diff(a, b):
    '''
    This function calculates the smallest unsigned angle difference in degrees.
    '''
    diff = abs(a - b) % 360
    return min(diff, 360 - diff)

def signed_angle_diff(a, b):
    '''
    This function calculates the signed angle difference in degrees.
    '''
    diff = (b - a + 180) % 360 - 180
    return diff

def decompose_rotation_error(gt_transformation, est_transformation, signed = True, convention = 'zyx', verbose = False):
    '''
    This fuction calculates Euler angles (rotation) differences between the estimated and ground truth transformation matrices.
    It uses the scipy.spatial.transform.Rotation class to extract the Euler angles from the rotation matrices.
    If Gimbal lock is detected, it uses a manual calculation of the angles.

    Args:
        gt_transformation: the ground truth transformation matrix
        est: the estimated transformation matrix
        signed: if True, the signed angle difference is calculated
        convention: the convention for the Euler angles (default and only supported is 'zyx')
                    This restriction is because the rotation matrix applied to simulate data is 'xyz' order which is the inverse of 'zyx' - because initrinsic and extrinsic rotations are inverted. Source: https://dominicplein.medium.com/extrinsic-intrinsic-rotation-do-i-multiply-from-right-or-left-357c38c1abfd
        verbose: if True, the Euler angles are printed
    Returns:
        x_diff: the difference in the x-axis rotation (degrees)
        y_diff: the difference in the y-axis rotation (degrees)
        z_diff: the difference in the z-axis rotation (degrees)

        OR if Gimbal lock is detected:
        re_along_x: the rotation error along the x-axis (degrees)
        re_along_y: the rotation error along the y-axis (degrees)
        re_along_z: the rotation error along the z-axis (degrees)
    
    Raises:
        RuntimeError: if Gimbal lock is detected.
    '''
    if convention != 'zyx':
        raise ValueError("Invalid convention. Only 'zyx' is supported.")
    
    # Get rotation matrices
    estimated_rotation = get_rotaion(est_transformation)
    gt_rotation = gt_transformation[:3, :3]

    # Get Euler degrees using scipy
    try:
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter("always")

            gt_euler = R.from_matrix(gt_rotation).as_euler(convention, degrees=True)
            gt_z, gt_y, gt_x = np.round(gt_euler, 3)
            est_euler = R.from_matrix(estimated_rotation).as_euler(convention, degrees=True)
            est_z, est_y, est_x = np.round(est_euler, 3)

            # Check for Gimbal lock warning
            gimal_lock_warning = any("Gimbal lock detected" in str(warn.message) for warn in w)
            if gimal_lock_warning:
                raise RuntimeError("Gimbal lock detected!")

        if verbose:
            print(f"Extracted Ground Truth Euler angles (deg): x={gt_x}, y={gt_y}, z={gt_z}")
            print(f"Extracted Estimated Euler angles (deg): x={est_x}, y={est_y}, z={est_z}")
    
        # compute differences
        diff_fn = signed_angle_diff if signed else angle_diff
        x_diff = diff_fn(gt_x, est_x)  
        y_diff = diff_fn(gt_y, est_y)  
        z_diff = diff_fn(gt_z, est_z)

    # Use manual calculation of angles if Gimbal lock is detected
    except RuntimeError as e:
        print(f'{e} - Trying manual calculation of angles.') if verbose else None
        x_diff, y_diff, z_diff = rotation_error_along_axis(gt_transformation, est_transformation, convention)

    if verbose:
        print(f'{x_diff}° error along x-axis, {y_diff}° error along y-axis and {z_diff}° error along z-axis.')

    return x_diff, y_diff, z_diff


# --------- UNUSED FUNCTIONS ---------
def error_histogram(source, target, result, dis_type ='mse', bins = 200):
    '''
    This function creates a histogram of the errors between the target and result point clouds.

    args:
        target: the target point cloud
        result: the result point cloud
        dis_type: the type of distance to calculate the error (mse or mae)
        bins: the number of bins for the histogram
    
    returns:
        histogram (Plot): the histogram plot of the errors 
        all_errors (list): the errors between the source and target point clouds (for inlier ratio calculation)
    '''
    # Calcualte error between the correspondences
    all_errors = [] 

    for i in range(len(source.points)):
        if dis_type =='mse':
            # Error as root mean square error
            error = np.sqrt(np.sum((np.asarray(result.points[i]) - np.asarray(target.points[i]))**2))
        elif dis_type == 'mae':
            # Error as mean absolute error
            error = np.mean(np.abs(np.asarray(result.points[i]) - np.asarray(target.points[i])))
                
        all_errors.append(error)

    # Make a histogram of the errors
    plt.hist(all_errors, bins = bins)
    plt.xlabel('Error')
    plt.ylabel('Frequency')
    plt.show()

    return all_errors


def rotation_error_along_z (gt_transformation, est, symmetry = None):
    '''
    This function calculates the rotation error along the z-axis (Euler angle) between the estimated and ground truth rotation matrices.
    Based on formulas and discussions in:
        https://math.stackexchange.com/questions/31001/finding-the-cos-angle-between-two-matrices-using-the-euclidean-inner-product
        https://stackoverflow.com/questions/15022630/how-to-calculate-the-angle-from-rotation-matrix
        https://www.youtube.com/watch?v=wg9bI8-Qx2Q (10:29)

    For C2 and C4 symmetries, the explanation is here:
        https://www2.math.upenn.edu/~mlazar/math170/notes07-4.pdf
        https://web.stanford.edu/~kaleeg/chem32/groupT/
    '''
    estimated_rotation = get_rotaion(est)
    ground_truth_rotation = gt_transformation[:3, :3]

    R = np.dot(ground_truth_rotation, estimated_rotation.T) 
    theta_z = atan2(R[1, 0], R[0, 0]) # Euler angles 
    theta_z_deg = abs(np.rad2deg(theta_z))

    if symmetry == 'C2':
        theta_z_deg = min(theta_z_deg, abs(180 - theta_z_deg))
    elif symmetry == 'C4':
        theta_z_deg = min(theta_z_deg, abs(90 - theta_z_deg), abs(180 - theta_z_deg), abs(270 - theta_z_deg))

    return np.round(theta_z_deg, 3)

def rotation_error_along_x (gt_transformation, est):
    '''
    This function calculates the rotation error along the x-axis (Euler angle) between the estimated and ground truth rotation matrices.
    Based on formulas and discussions in:
        https://www.youtube.com/watch?v=wg9bI8-Qx2Q (10:29)

    Args:
        est: the estimated transformation
        gt_transformation: the ground truth transformation
    Returns:
        theta_x_deg: the rotation error along the x-axis (degrees) rounded to 3 decimal places
    '''
    estimated_rotation = get_rotaion(est)
    ground_truth_rotation = gt_transformation[:3, :3]

    R = np.dot(ground_truth_rotation, estimated_rotation.T) 
    theta_x = atan2(R[2, 1], R[2, 2]) # Euler angles = atan2(r32, r33)
    theta_x_deg = abs(np.rad2deg(theta_x))

    return np.round(theta_x_deg, 3)

def rotation_error_along_y (gt_transformation, est, theta_z_deg):
    '''
    This function calculates the rotation error along the y-axis (Euler angle) between the estimated and ground truth rotation matrices.
    It gives a warning if gimbal lock is detected (90 or 270 degrees).
    Based on formulas and discussions in:
        https://www.youtube.com/watch?v=wg9bI8-Qx2Q
    
    Args:
        est: the estimated transformation
        gt_transformation: the ground truth transformation
        theta_z_deg: the rotation error along the z-axis (degrees)
    Returns:
        theta_y_deg: the rotation error along the y-axis (degrees) rounded to 3 decimal places
    '''
   
    estimated_rotation = get_rotaion(est)
    ground_truth_rotation = gt_transformation[:3, :3]

    R = np.dot(ground_truth_rotation, estimated_rotation.T) 
    if np.cos(np.deg2rad(theta_z_deg)) == 0:
        second_term = R[1, 0]/np.sin(np.deg2rad(theta_z_deg))
        theta_y = atan2(-R[2, 0], second_term)
    else:
        second_term = R[0, 0]/np.cos(np.deg2rad(theta_z_deg)) 
        theta_y = atan2(-R[2, 0], second_term)

    theta_y_deg = abs(np.rad2deg(theta_y))

    if np.round(theta_y_deg, 3) == 90 or np.round(theta_y_deg, 3) == 270:
        print("Warning: Gimbal lock detected! It might not be possible to uniquely and accurately determine all angles.")

    return np.round(theta_y_deg, 3)

# Construct the transformation matrix from the rotation matrix, translation vector, and scale (for Probreg)
def reconstruct_transformation_propreg(rot, t, scale):
    scaled_rotation = scale * rot
    T = np.eye(4)
    T[:3, :3] = scaled_rotation
    T[:3, 3] = t   
    return T

def get_transformation(est):
    # Extract the transformation matrices
    try:
        estimation_transformation = est.transformation # for open3d
    except AttributeError:
        try: 
            estimation_transformation = reconstruct_transformation_propreg(est.rot, est.t, est.scale) # for Probreg
        except: # for learning 3d
            estimation_transformation = est['est_T'].detach().cpu().numpy()[0]
            estimation_transformation = estimation_transformation.reshape(4,4)

    return estimation_transformation

def transformation_error(est, gt_transformation):  # ATTENTION: not a good metric because transformations consist of rotation and translation, which have different metrics (one is radian, the other centimeters).
    ''' 
    This function calculates transformation error as the root-mean-square error between estimated transformation and ground truth transformation 
    Based on definitions in this paper https://arxiv.org/pdf/2103.02690
    
    args:
        est: the estimation object
        gt_transformation: the ground truth transformation

    returns:
        transformation_error: the transformation error
    '''
    estimation_transformation = get_transformation(est)
    rmseT = np.sqrt(np.mean(np.square(estimation_transformation - gt_transformation)))
    
    return rmseT


def remove_outliers(data):
    '''
    This function removes outliers from the data based on the interquartile range (IQR).
    Based on https://medium.com/@davidnh8/outlier-detection-101-median-and-interquartile-range-cc9dde94c0ac

    Args:
        data (np.array): The data to remove outliers from.
    Returns:
        clear_data (np.array): The data without outliers.
        outlier_index (np.array): The indices of the outliers.
    '''
    q1 = np.percentile(data, 25)
    q3 = np.percentile(data, 75)
    iqr = q3 - q1
    lower_bound = q1 - 1.5 * iqr
    upper_bound = q3 + 1.5 * iqr
    outlier_index = np.where((data <= lower_bound) | (data >= upper_bound))
    clear_data = np.delete(data, outlier_index)
    return clear_data, outlier_index


def get_overlap_ratio(source,target,threshold):
    """
    - Overlap is defined as the ratio of the number of points in each point cloud that cover a region of the scene, 
    which is also covered by the other point cloud, to the total number of points in the point cloud.
    
    - Overlap is computed as the ratio of the number of points in the source point cloud that are within a distance 
    threshold to the target point cloud to the total number of points in the source point cloud.
    
    Taken from https://github.com/prs-eth/OverlapPredator/blob/main/scripts/cal_overlap.py
    Based on https://www.open3d.org/docs/latest/tutorial/Basic/kdtree.html
    """
    pcd_tree = o3d.geometry.KDTreeFlann(target)
    
    match_count=0
    for i, point in enumerate(source.points):
        [count, _, _] = pcd_tree.search_radius_vector_3d(point, threshold)
        if(count!=0):
            match_count+=1

    overlap_ratio = match_count / len(source.points) *100
    return overlap_ratio