# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% # Copyright (c) 2023 Image Processing Research Group of University Federico II of Naples ('GRIP-UNINA'). # # All rights reserved. # This work should only be used for nonprofit purposes. # # By downloading and/or using any of these files, you implicitly agree to all the # terms of the license, as specified in the document LICENSE.txt # (included in this package) and online at # http://www.grip.unina.it/download/LICENSE_OPEN.txt """ @author: davide.cozzolino """ import os import numpy as np def extractGTs(gt, erodeKernSize=15, dilateKernSize=11): from scipy.ndimage.filters import minimum_filter, maximum_filter gt1 = minimum_filter(gt, erodeKernSize) gt0 = np.logical_not(maximum_filter(gt, dilateKernSize)) return gt0, gt1 def computeMetricsContinue(values, gt0, gt1): values = values.flatten().astype(np.float32) gt0 = gt0.flatten().astype(np.float32) gt1 = gt1.flatten().astype(np.float32) inds = np.argsort(values) inds = inds[(gt0[inds]+gt1[inds])>0] vet_th = values[inds] gt0 = gt0[inds] gt1 = gt1[inds] TN = np.cumsum(gt0) FN = np.cumsum(gt1) FP = np.sum(gt0) - TN TP = np.sum(gt1) - FN msk = np.pad(vet_th[1:]>vet_th[:-1], (0,1), mode='constant', constant_values=True) FP = FP[msk] TP = TP[msk] FN = FN[msk] TN = TN[msk] vet_th = vet_th[msk] return FP, TP, FN, TN, vet_th def computeMetrics_th(values, gt, gt0, gt1, th): values = values>th values = values.flatten().astype(np.uint8) gt = gt.flatten().astype(np.uint8) gt0 = gt0.flatten().astype(np.uint8) gt1 = gt1.flatten().astype(np.uint8) gt = gt[(gt0+gt1)>0] values = values[(gt0+gt1)>0] from sklearn.metrics import confusion_matrix cm = confusion_matrix(gt, values) TN = cm[0, 0] FN = cm[1, 0] FP = cm[0, 1] TP = cm[1, 1] return FP, TP, FN, TN def computeMCC(FP, TP, FN, TN): FP = np.float64(FP) TP = np.float64(TP) FN = np.float64(FN) TN = np.float64(TN) return np.abs(TP*TN - FP*FN) / np.maximum(np.sqrt((TP + FP)*(TP + FN)*(TN + FP)*(TN + FN) ), 1e-32) def computeF1(FP, TP, FN, TN): return 2*TP / np.maximum((2*TP + FN + FP), 1e-32) def computeLocalizationMetrics(map, gt): gt0, gt1 = extractGTs(gt) # best threshold try: FP, TP, FN, TN, _ = computeMetricsContinue(map, gt0, gt1) f1 = computeF1(FP, TP, FN, TN) f1i = computeF1(TN, FN, TP, FP) F1_best = max(np.max(f1), np.max(f1i)) except: import traceback traceback.print_exc() F1_best = np.nan # fixed threshold try: FP, TP, FN, TN = computeMetrics_th(map, gt, gt0, gt1, 0.5) f1 = computeF1(FP, TP, FN, TN) f1i = computeF1(TN, FN, TP, FP) F1_th = max(f1, f1i) except: import traceback traceback.print_exc() F1_th = np.nan return F1_best, F1_th def computeDetectionMetrics(scores, labels): lbl = np.array(labels) lbl = lbl[np.isfinite(scores)] scores = np.array(scores, dtype='float32') scores[scores==np.PINF] = np.nanmax(scores[scores0.5) return AUC, bACC # ---------------------------------------------------------------------------- # # DETECTION METRICS # ---------------------------------------------------------------------------- # ''' path = '/path-to-DSO1-dataset/' from glob import glob scores = [] labels = [] for map_path in glob(path + 'normal*'): scores.append(np.load(map_path)['score']) labels.append(0) for map_path in glob(path + 'splicing*'): scores.append(np.load(map_path)['score']) labels.append(1) print(computeDetectionMetrics(scores, labels)) ''' # ---------------------------------------------------------------------------- # # LOCALIZATION METRICS # ---------------------------------------------------------------------------- # ''' path = '/path-to-DSO1-dataset/' gt_path = '/path-to-DSO1-masks/' from glob import glob from tqdm import tqdm from PIL import Image import os F1_best_list = [] F1_th_list = [] for map_path in tqdm(glob(path + 'splicing*')): map = np.load(map_path)['map'] # gt can be <0.1 or >0.1 depending on the dataset. DSO-1 has inverted masks, so < gt = np.array(Image.open(gt_path + os.path.basename(map_path[:-4])).convert('L')) < 0.1 assert gt.shape == map.shape F1_best, F1_th = computeLocalizationMetrics(map, gt) F1_best_list.append(F1_best) F1_th_list.append(F1_th) print('skipped:', np.count_nonzero(np.isnan(F1_best_list))) print(np.nanmean(F1_best_list), np.nanmean(F1_th_list)) '''