combine_metrics_files.py

-----------------------------
Combine metrics.csv files from the different models into a single file, to be used by the statistics.p script

compare_ai_vs_human_annots.py
-----------------------------
Comparison between the AI and humman made segmentations on the OSLO-COMET dataset.

EvaluationScripts/combine_metrics_files.py

@@ -0,0 +1,105 @@
+import pandas as pd
+import os
+from argparse import ArgumentParser
+import re
+
+
+DICT_MODEL_NAMES = {'BASELINE': 'BL',
+                    'SEGGUIDED': 'SG',
+                    'UW': 'UW'}
+
+DICT_METRICS_NAMES = {'NCC': 'N',
+                      'SSIM': 'S',
+                      'DICE': 'D',
+                      'DICE MACRO': 'D',
+                      'HD': 'H', }
+
+
+DF_COLS = ['SSIM', 'NCC', 'MSE', 'DICE_MACRO', 'HD', 'HD95', 'Time', 'TRE', 'Experiment', 'Model']
+
+
+def get_model_name(in_path: str) -> str:
+    model = re.search('((UW|SEGGUIDED|BASELINE).*)_\d', in_path)
+    if model:
+        model = model.group(1).rstrip('_')
+        model = model.replace('_Lsim', '')
+        model = model.replace('_Lseg', '')
+        model = model.replace('_L', '')
+        model = model.replace('_', ' ')
+        model = model.upper()
+        elements = model.split()
+        model = elements[0]
+        metrics = list()
+        model = DICT_MODEL_NAMES[model]
+        for m in elements[1:]:
+            if m != 'MACRO':
+                metrics.append(DICT_METRICS_NAMES[m])
+
+        return '{}-{}'.format(model, ''.join(metrics))
+    else:
+        try:
+            model = re.search('(SyNCC|SyN)', in_path).group(1)
+        except AttributeError:
+            raise ValueError('Unknown folder name/model: '+ in_path)
+        return model
+
+
+def find_metric_files(root_path: str, folder_filter: str) -> dict:
+    metric_files = dict()
+    starting_level = root_path.count(os.sep)
+    for r, d, f in os.walk(root_path):
+        level = r.count(os.sep) - starting_level
+        if level < 3:
+            for name in f:
+                if 'metrics.csv' == name and folder_filter in r.split(os.sep):
+                    model = get_model_name(os.path.join(r, name))
+                    metric_files[model] = os.path.join(r, name)
+    return metric_files
+
+
+def read_metrics_files(metrics: dict, experiment: str) -> pd.DataFrame:
+    df = pd.DataFrame(columns=DF_COLS)
+    for k in metrics.keys():
+        csv = pd.read_csv(metrics[k], sep=';')
+        csv['Experiment'] = experiment
+        csv['Model'] = k
+        df = df.append(csv[DF_COLS], ignore_index=True)
+    return df
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser()
+    parser.add_argument('--output-dir', help='Output directory', default='./')
+    parser.add_argument('--ixi-runs', help='Directory were the evaluation outputs are stored')
+    parser.add_argument('--comet-runs', help='Directory were the evaluation outputs are stored')
+    parser.add_argument('--comet-tl-freezenone-runs', help='Directory were the evaluation outputs are stored')
+    parser.add_argument('--comet-tl-encoder-runs', help='Directory were the evaluation outputs are stored')
+    parser.add_argument('--ants-runs', help='Directory were the evaluation outputs are stored')
+    parser.add_argument('--folder-filter', default='Evaluate')
+
+    args = parser.parse_args()
+
+    assert os.path.exists(args.ixi_runs), 'IXI directory not found'
+    assert os.path.exists(args.comet_runs), 'COMET directory not found'
+    assert os.path.exists(args.comet_tl_freezenone_runs), 'COMET TL Fine Tuned Froze None directory not found'
+    assert os.path.exists(args.comet_tl_encoder_runs), 'COMET TL Fine Tuned in 2 Steps directory not found'
+    assert os.path.exists(args.ants_runs), 'COMET TL Fine Tuned in 2 Steps directory not found'
+
+    IXI_metrics = find_metric_files(args.ixi_runs, args.folder_filter)
+    COMET_metrics = find_metric_files(args.comet_runs, args.folder_filter)
+    COMET_TL_FTFN_metrics = find_metric_files(args.comet_tl_freezenone_runs, args.folder_filter)
+    COMET_TL_FT2S_metrics = find_metric_files(args.comet_tl_encoder_runs, args.folder_filter)
+    ANTS_metrics = find_metric_files(args.ants_runs, args.folder_filter)
+
+    IXI_df = read_metrics_files(IXI_metrics, 'IXI')
+    COMET_df = read_metrics_files(COMET_metrics, 'COMET')
+    COMET_TL_FtFn_df = read_metrics_files(COMET_TL_FTFN_metrics, 'COMET_TL_FtFn')
+    COMET_TL_Ft2S_df = read_metrics_files(COMET_TL_FT2S_metrics, 'COMET_TL_Ft2Stp')
+    ANTS_df = read_metrics_files(ANTS_metrics, 'ANTs')
+
+    df = pd.concat([IXI_df, COMET_df, COMET_TL_FtFn_df, COMET_TL_Ft2S_df, ANTS_df], ignore_index=True)
+    out_file_path = os.path.join(args.output_dir, 'Combined_metrics.csv')
+    if os.path.exists(out_file_path):
+        os.remove(out_file_path)
+    df.to_csv(out_file_path, sep=';', index=False)
+    print('Output file: ' + out_file_path)

EvaluationScripts/compare_ai_vs_human_annots.py

@@ -0,0 +1,116 @@
+import os, sys
+currentdir = os.path.dirname(os.path.realpath(__file__))
+parentdir = os.path.dirname(currentdir)
+sys.path.append(parentdir)  # PYTHON > 3.3 does not allow relative referencing
+
+import nibabel as nib
+import re
+import medpy.metric as medpy_metrics
+
+import pandas as pd
+
+import numpy as np
+
+from tqdm import tqdm
+
+import multiprocessing as mp
+IMG_DIRECTORY = '/mnt/EncryptedData1/Laparoscopy/OSLO_COMET_dataset/OSLO_COMET_CT/Volumes_nii/test_set'
+VES_SEG_DIRECTORY = '/mnt/EncryptedData1/Laparoscopy/OSLO_COMET_dataset/OSLO_COMET_CT/Vessels'
+PAR_SEG_DIRECTORY = '/mnt/EncryptedData1/Laparoscopy/OSLO_COMET_dataset/OSLO_COMET_CT/Parenchyma'
+
+LM_SEG_DIRECTORY = '/mnt/EncryptedData1/Laparoscopy/OSLO_COMET_dataset/OSLO_COMET_CT/LiverMask/test_set'
+
+nii_FILENAME_PATTERN = '(.*)_CT.nii'
+LV_VESSEL_FILENAME_PATTERN = '(.*)-vessels.nii'
+LV_PARENCHYMA_FILENAME_PATTERN = '(.*)-livermask.nii'
+
+OUT_DIRECTORY = '/mnt/EncryptedData1/Users/javier/ext_datasets/COMET_ai_vs_human'
+
+
+def process_group(file_list):
+    # print('Got: ' + ','.join(file_list))
+    img_file, par_file, ves_file, lm_par_file, lm_ves_file = file_list
+    img_num = int(re.match(nii_FILENAME_PATTERN, os.path.split(img_file)[-1])[1])
+
+    human_par = nib.load(par_file)
+    header = human_par.header
+    human_par = np.asarray(human_par.dataobj)
+    human_ves = np.asarray(nib.load(ves_file).dataobj)
+
+    lm_par = np.asarray(nib.load(lm_par_file).dataobj)
+    lm_ves = np.asarray(nib.load(lm_ves_file).dataobj)
+    lm_ves[lm_ves > 0] = 1
+
+    dsc_par = medpy_metrics.dc(human_par, lm_par)
+    dsc_ves = medpy_metrics.dc(human_ves, lm_ves)
+
+    hd_par = medpy_metrics.hd(human_par, lm_par, voxelspacing=header['pixdim'][1:4])
+    hd_ves = medpy_metrics.hd(human_ves, lm_ves, voxelspacing=header['pixdim'][1:4])
+
+    hd95_par = medpy_metrics.hd95(human_par, lm_par, voxelspacing=header['pixdim'][1:4])
+    hd95_ves = medpy_metrics.hd95(human_ves, lm_ves, voxelspacing=header['pixdim'][1:4])
+
+    return img_num, dsc_par, dsc_ves, hd_par, hd_ves, hd95_par, hd95_ves
+
+
+if __name__ == '__main__':
+    img_list = [os.path.join(IMG_DIRECTORY, f) for f in os.listdir(IMG_DIRECTORY) if f.endswith('.nii')]
+    img_list.sort()
+
+    ves_seg_list = [os.path.join(VES_SEG_DIRECTORY, os.path.split(f)[-1]+'.gz') for f in img_list]
+    ves_seg_list.sort()
+
+    par_seg_list = [os.path.join(PAR_SEG_DIRECTORY, os.path.split(f)[-1]+'.gz') for f in img_list]
+    par_seg_list.sort()
+
+    lm_ves_seg_list = [os.path.join(LM_SEG_DIRECTORY, f) for f in os.listdir(LM_SEG_DIRECTORY) if f.endswith('-vessels.nii')]
+    lm_ves_seg_list.sort()
+
+    lm_par_seg_list = [os.path.join(LM_SEG_DIRECTORY, f) for f in os.listdir(LM_SEG_DIRECTORY) if f.endswith('-livermask.nii')]
+    lm_par_seg_list.sort()
+
+    os.makedirs(OUT_DIRECTORY, exist_ok=True)
+
+    zip_lists = zip(img_list, par_seg_list, ves_seg_list, lm_par_seg_list, lm_ves_seg_list)
+    mets = ['DSC', 'DSC', 'HD', 'HD', 'H95', 'H95']
+    segs = ['Parenchyma', 'Vessels', 'Parenchyma', 'Vessels', 'Parenchyma', 'Vessels']
+    cols = list(zip(*[mets, segs]))
+    idx = pd.MultiIndex.from_tuples(cols, names=['Metrics', 'Labels'])
+    df = pd.DataFrame(index=idx)
+
+    print('\nLaunching processes...')
+    with mp.Pool(11, maxtasksperchild=1) as p:
+        results = p.map_async(process_group, zip_lists)
+        for v in results.get():
+            df[v[0]] = v[1:]
+
+    # for i, (img_file, par_file, ves_file, lm_par_file, lm_ves_file) in tqdm(enumerate(zip_lists), total=len(img_list)):
+    #     img_name = re.match(nii_FILENAME_PATTERN, os.path.split(img_file)[-1])[1]
+    #
+    #     img = nib.load(img_file)
+    #     human_par = nib.load(par_file)
+    #     header = human_par.header
+    #     human_par = np.asarray(human_par.dataobj)
+    #     human_ves = np.asarray(nib.load(ves_file).dataobj)
+    #
+    #     lm_par = np.asarray(nib.load(lm_par_file).dataobj)
+    #     lm_ves = np.asarray(nib.load(lm_ves_file).dataobj)
+    #     lm_ves[lm_ves > 0] = 1
+    #
+    #     dsc_par = medpy_metrics.dc(human_par, lm_par)
+    #     dsc_ves = medpy_metrics.dc(human_ves, lm_ves)
+    #
+    #     hd_par = medpy_metrics.hd(human_par, lm_par, voxelspacing=header['pixdim'][1:4])
+    #     hd_ves = medpy_metrics.hd(human_ves, lm_ves, voxelspacing=header['pixdim'][1:4])
+    #
+    #     hd95_par = medpy_metrics.hd95(human_par, lm_par, voxelspacing=header['pixdim'][1:4])
+    #     hd95_ves = medpy_metrics.hd95(human_ves, lm_ves, voxelspacing=header['pixdim'][1:4])
+    #
+    #     df[i] = [dsc_par, dsc_ves, hd_par, hd_ves, hd95_par, hd95_ves]
+    print('\nResults...')
+    print(df)
+
+    print('\nSummary...')
+    print(df.describe())
+
+    df.to_csv('comparison.csv', sep=';')