1

.gitignore

@@ -0,0 +1,16 @@
+!.gitignore
+/logs
+__pycache__*
+/*pycache*
+/model/*pycache*
+**/*pycache*
+*.out
+venv*
+helix_log
+checkpoints
+MONAI
+*.svg
+data
+generative-models
+datasets
+notuse

README.md

@@ -1,20 +1 @@
----
-title: Frankenstein
-emoji: 🚀
-colorFrom: red
-colorTo: red
-sdk: docker
-app_port: 8501
-tags:
-- streamlit
-pinned: false
-short_description: Artificial Life
-license: mit
----
-
-# Welcome to Streamlit!
-
-Edit `/src/streamlit_app.py` to customize this app to your heart's desire. :heart:
-
-If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
-forums](https://discuss.streamlit.io).
+# 

dataprocesser/.gitignore

@@ -0,0 +1 @@
+__pycache__*

dataprocesser/Preprocess_CT_Mask_generation.py

@@ -0,0 +1,267 @@
+import os
+import pandas as pd
+import numpy as np
+import nrrd
+import SimpleITK as sitk
+import cv2
+
+import numpy as np
+
+def shift_to_min_zero(arr):
+    """
+    Shifts the input NumPy array so that the minimum value becomes 0.
+    
+    Parameters:
+        arr (numpy.ndarray): The input array to shift.
+    
+    Returns:
+        numpy.ndarray: The shifted array with the minimum value as 0.
+    """
+    min_value = np.min(arr)  # Find the minimum value
+    shifted_array = arr - min_value  # Subtract the minimum value from all elements
+    return shifted_array
+
+
+def create_body_mask(numpy_img, body_threshold=-500, min_contour_area=10000):
+    """
+    Create a binary body mask from a CT image tensor, using a specific threshold for the body parts.
+
+    Args:
+    tensor_img (torch.Tensor): A tensor representation of a grayscale CT image, with intensity values from -1024 to 1500.
+
+    Returns:
+    torch.Tensor: A binary mask tensor where the entire body region is 1 and the background is 0.
+    """
+    # Convert tensor to numpy array
+    numpy_img = np.ascontiguousarray(numpy_img.astype(np.int16))  # Ensure we can handle negative values correctly
+    #numpy_img = numpy_img.astype(np.int16)
+
+    # Threshold the image at -500 to separate potential body from the background
+    binary_img = np.where(numpy_img > body_threshold, 1, 0).astype(np.uint8)
+
+    # Find contours from the binary image
+    contours, _ = cv2.findContours(binary_img, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
+
+    # Create an empty mask
+    mask = np.zeros_like(binary_img)
+
+    VERBOSE = False
+    # Fill all detected body contours
+    if contours:
+        for contour in contours:
+            if cv2.contourArea(contour) >= min_contour_area:
+                if VERBOSE:
+                    print('current contour area: ', cv2.contourArea(contour), 'threshold: ', min_contour_area)
+                cv2.drawContours(mask, [contour], -1, 1, thickness=cv2.FILLED)
+
+    return mask
+
+def apply_mask(normalized_image_array, mask_array):
+    return normalized_image_array * mask_array
+
+def print_all_info(data, title):
+    print(f'min, max of {title}:', np.min(data), np.max(data))
+
+def process_CT_segmentation_numpy(mask, csv_simulation_values):
+    #df = pd.read_csv(csv_file)
+    df = csv_simulation_values
+    # Create a dictionary to map organ index to HU values
+    hu_values = dict(zip(df['Order Number'], df['HU Value']))
+    order_begin_from_0 = True if df['Order Number'].min()==0 else False
+    
+    hu_mask = np.zeros_like(mask)
+    # Value Assigment
+    hu_mask[mask == 0] = -1000 # background
+    for organ_index, hu_value in hu_values.items():
+        assert isinstance(hu_value, int), f"Expected mask value an integer, but got {hu_value}. Ensure the mask is created by fine mode of totalsegmentator"
+        assert isinstance(organ_index, int), f"Expected organ_index an integer, but got {organ_index}. Ensure the mask is created by fine mode of totalsegmentator"
+        if order_begin_from_0:
+            hu_mask[mask == (organ_index+1)] = hu_value # mask value begin from 1 as body value, other than 0 in TA2 table, so organ_index+1
+        else:
+            hu_mask[mask == (organ_index)] = hu_value
+    return hu_mask
+
+
+# 处理单个图像和分割图
+def process_image(input_path, contour_path, seg_path, seg_tissue_path, csv_simulation_values, output_path1, output_path2, output_path3, body_threshold):
+    # 读取原始 MR 图像和分割图
+    if input_path.endswith('.nrrd'):
+        img, header = nrrd.read(input_path)
+        segmentation_img, header_seg = nrrd.read(seg_path)
+        seg_tissue_img, header_seg_tissue = nrrd.read(seg_tissue_path)
+    elif input_path.endswith('.nii.gz') or input_path.endswith('.nii'):
+        import nibabel as nib
+        img_metadata = nib.load(input_path)
+        img = img_metadata.get_fdata()
+        affine = img_metadata.affine
+
+        seg_metadata = nib.load(seg_path)
+        segmentation_img = seg_metadata.get_fdata()
+        affine_seg = seg_metadata.affine
+
+        seg_tissue_metadata = nib.load(seg_tissue_path)
+        seg_tissue_img = seg_tissue_metadata.get_fdata()
+        
+    # extract contour
+    body_contour = np.zeros_like(img, dtype=np.int16)
+    for i in range(img.shape[-1]):
+        slice_data = img[:, :, i]
+        body_contour[:, :, i] = create_body_mask(slice_data, body_threshold=body_threshold)
+    
+    # CT images don't need additional normalization
+    # 
+    
+    # normalize to 0-1
+    img_normalized = shift_to_min_zero(img)
+    # img_normalized = img_normalized/2000 # scale factor
+    
+    # apply mask to ct img
+    masked_image = apply_mask(img_normalized, body_contour)
+    
+    # process the mask image
+    seg = segmentation_img
+    tissue = seg_tissue_img
+    tissue[tissue!=0] += 200
+    # Create a mask for overlapping areas
+    overlap_mask = (seg > 0) & (tissue > 0)
+    
+    # For overlapping areas, keep the lower value (organ values in seg)
+    merged_mask = tissue.copy()
+    merged_mask[overlap_mask] = seg[overlap_mask]
+    
+    # Keep all non-overlapping areas
+    merged_mask[seg > 0] = seg[seg > 0]
+
+    combined_array = merged_mask + body_contour
+    
+    processed_segmentation = combined_array
+
+    # assign simulation value to ct segmentation mask
+    assigned_segmentation = process_CT_segmentation_numpy(combined_array, csv_simulation_values)
+    
+    if input_path.endswith('.nrrd'):
+        # 保存处理后的 MR 图像
+        nrrd.write(output_path1, masked_image, header)
+        
+        # 保存处理后的分割图
+        nrrd.write(output_path2, processed_segmentation, header_seg)
+
+        # save the body contour mask
+
+    elif input_path.endswith('.nii.gz') or input_path.endswith('.nii'):
+        img_processed = nib.Nifti1Image(masked_image, affine)
+        nib.save(img_processed, output_path1)
+        seg_processed = nib.Nifti1Image(processed_segmentation, affine_seg)
+        nib.save(seg_processed, output_path2)
+        contour_processed = nib.Nifti1Image(body_contour, affine_seg)
+        assigned_segmentation_processed  = nib.Nifti1Image(assigned_segmentation, affine_seg)
+        # Split the path into directory and filename
+        directory, filename = os.path.split(output_path2)
+        contour_filename = filename.replace('_seg_merged', '_contour')
+        contour_path = os.path.join(directory, contour_filename)
+        nib.save(contour_processed, contour_path)
+
+        nib.save(assigned_segmentation_processed, output_path3)
+        
+    return processed_segmentation
+
+def analyse_hist(input_path):
+    if input_path.endswith('.nrrd'):
+        img, header = nrrd.read(input_path)
+    elif input_path.endswith('.nii.gz'):
+        import nibabel as nib
+        img_metadata = nib.load(input_path)
+        img = img_metadata.get_fdata()
+        affine = img_metadata.affine
+    import numpy as np
+    import matplotlib.pyplot as plt
+
+    # Plot the histogram
+    print('shape of img: ', img.shape)
+    plt.hist(img[:, :, 50], bins=30, edgecolor='black', alpha=0.7)
+    plt.xlabel('Value')
+    plt.ylabel('Frequency')
+    plt.title('Value Distribution')
+    plt.show()
+
+
+def process_csv(csv_file, output_root, csv_simulation_file, body_threshold=-500):
+    # read csv to get simulation value
+    csv_simulation_values = pd.read_csv(csv_simulation_file) #.to_numpy()
+    #csv_simulation_values = pd.read_csv(csv_simulation_file)
+
+    # check 2-dimensional csv_simulation_values 
+    if csv_simulation_values.ndim == 1:
+        raise ValueError("CSV should contain two columns: organ_index and simulation_value")
+
+    if not os.path.exists(csv_file):
+        print('csv:', csv_file)
+        raise ValueError('csv_file must input a available csv file in simplified form: id, Aorta_diss, seg, img!')
+    else:
+        print(f'use csv: {csv_file}')
+    
+    data_frame = pd.read_csv(csv_file)
+    if len(data_frame) == 0:
+        raise RuntimeError(f"Found 0 images in: {csv_file}")
+    patient_IDs = data_frame.iloc[:, 0].tolist()
+    Aorta_diss = data_frame.iloc[:, 1].tolist()
+    segs =  data_frame.iloc[:, 2].tolist()
+    images = data_frame.iloc[:, 3].tolist()
+
+    from tqdm import tqdm
+    dataset_list = []
+    for idx in tqdm(range(len(images))):
+        if (images[idx].endswith('.nii.gz') and segs[idx].endswith('.nii.gz')) or \
+            (images[idx].endswith('.nii') and segs[idx].endswith('.nii')):
+            input_file_path = images[idx]
+            seg_file_path = segs[idx]
+            patient_id = patient_IDs[idx]
+            ad = Aorta_diss[idx]
+            seg_tissue_file_path = seg_file_path.replace("_seg","_seg_tissue")
+
+            root_dir = os.path.dirname(input_file_path)
+            
+            # Get root path (directory path)
+            root_path = os.path.dirname(seg_file_path)
+            ct_processed_file_name = f"{patient_id}_ct_processed.nii.gz"
+            seg_merged_file_name = f"{patient_id}_ct_seg_merged.nii.gz"
+            seg_merged_assigned_mask_file_name = f"{patient_id}_ct_seg_merged_assigned_mask.nii.gz"
+            
+            os.makedirs(output_root, exist_ok=True)
+            output_file_path1 = os.path.join(output_root, ct_processed_file_name)
+            output_file_path2 = os.path.join(output_root, seg_merged_file_name)
+            output_file_path3 = os.path.join(output_root, seg_merged_assigned_mask_file_name)
+            print(f"Processing {input_file_path} with segmentation {seg_file_path}")
+            print(f"Save results to {output_file_path1} and {output_file_path2} and {output_file_path3} \n")
+            
+            
+            processed_seg = process_image(input_file_path, None, seg_file_path, seg_tissue_file_path, csv_simulation_values, output_file_path1, output_file_path2, output_file_path3, body_threshold)
+
+            # processed_mr_csv_file = ...
+            csv_mr_line = [patient_id,ad, output_file_path2, output_file_path1, output_file_path3]
+            dataset_list.append(csv_mr_line)
+
+    import csv
+    output_csv_file=os.path.join(output_root, 'processed_csv_file.csv')
+    with open(output_csv_file, 'w', newline='') as f:
+        csvwriter = csv.writer(f)
+        csvwriter.writerow(['id', 'Aorta_diss', 'seg', 'img', 'seg_mask']) 
+        csvwriter.writerows(dataset_list) 
+
+if __name__ == "__main__":
+    import argparse
+    csv_file = r'E:\Projects\yang_proj\SynthRad_GAN\synthrad_conversion\datacsv\ct_synthrad_test_newserver.csv'
+    output_root = r'E:\Projects\yang_proj\data\synthrad\processed'
+    csv_simulation_file = r'E:\Projects\yang_proj\SynthRad_GAN\synthrad_conversion\TA2_CT_from1.csv'
+    process_csv(csv_file, output_root, csv_simulation_file, body_threshold=-500)
+
+    '''parser = argparse.ArgumentParser(description="Process MR images and segmentation maps, apply masks and replace grayscale values.")
+    parser.add_argument('--input_folder1', required=True, help="Path to the folder containing input MR .nrrd files.")
+    parser.add_argument('--input_folder2', required=True, help="Path to the folder containing segmentation .nrrd files.")
+    parser.add_argument('--output_folder1', required=True, help="Path to the folder to save the output MR files.")
+    parser.add_argument('--output_folder2', required=True, help="Path to the folder to save the output segmentation files.")
+    parser.add_argument('--csv_simulation_file', required=True, help="CSV file containing simulated CT grayscale values.")
+    parser.add_argument('--body_threshold', type=int, default=50, help="Threshold to separate body from background.")
+    args = parser.parse_args()
+
+    process_folder(args.input_folder1, args.input_folder2, args.output_folder1, args.output_folder2, args.csv_simulation_file, args.body_threshold)'''

dataprocesser/Preprocess_MRCT_mask_conversion.py

@@ -0,0 +1,294 @@
+import pandas as pd
+import numpy as np
+import nibabel as nib
+import torch
+import os
+from tqdm import tqdm
+#from dataprocesser.customized_transforms import create_body_contour
+from dataprocesser.Preprocess_MR_Mask_generation import process_segmentation
+from dataprocesser.Preprocess_CT_Mask_generation import process_CT_segmentation_numpy
+import difflib
+
+def find_best_match_smart(organ_name, target_names):
+    # 完全匹配
+    if organ_name in target_names:
+        return organ_name
+    # 精确 startswith 匹配（如 vertebrae → vertebrae_Lx）
+    matches = [t for t in target_names if t.startswith(organ_name)]
+    if matches:
+        return matches[0]
+    # 再 fallback 到 difflib，但严格一些
+    import difflib
+    closes = difflib.get_close_matches(organ_name, target_names, n=1, cutoff=0.8)
+    return closes #[0] if close else None
+
+
+def convert_segmentation_mask_torch(source_mask, source_csv, target_csv, body_contour_value=1):
+    """
+    Converts segmentation mask values from source modality to target modality based on organ name mapping.
+
+    Parameters:
+    - source_mask (torch.Tensor): The source segmentation mask tensor.
+    - source_csv (str): Path to the CSV file of the source modality (CT or MR).
+    - target_csv (str): Path to the CSV file of the target modality (MR or CT).
+    - body_contour_value (int): The class value for "body contour" in the target modality.
+
+    Returns:
+    - target_mask (torch.Tensor): The converted segmentation mask tensor.
+    """
+    # Load the source and target anatomy lists
+    source_df = pd.read_csv(source_csv)
+    target_df = pd.read_csv(target_csv)
+
+    # Create dictionaries mapping class values to organ names
+    source_mapping = {}
+    for _, row in source_df.iterrows():
+        organ_name = row['Organ Name']
+        class_value = row.iloc[0]
+        source_mapping.setdefault(organ_name, []).append(class_value)
+
+    target_mapping = {}
+    for _, row in target_df.iterrows():
+        organ_name = row['Organ Name']
+        class_value = row.iloc[0]
+        target_mapping.setdefault(organ_name, []).append(class_value)
+
+    # Create a reverse mapping from class values to organ names for the source modality
+    class_to_organ = {class_value: organ_name for organ_name, class_values in source_mapping.items() for class_value in class_values}
+
+    # Initialize the target mask with zeros
+    target_mask = torch.zeros_like(source_mask, dtype=source_mask.dtype)
+
+    # Convert each unique class in the source mask
+    unique_classes = torch.unique(source_mask)
+    for class_value in unique_classes:
+        # Find the corresponding organ name in the source modality
+        organ_name = class_to_organ.get(class_value.item(), None)
+        
+        if class_value.item() == 0:  # Preserve background as is
+            target_value = 0
+        else:
+            # If organ name exists, find the corresponding target class values
+            if organ_name and organ_name in target_mapping:
+                # Pick the first target class value (or handle overlaps if needed)
+                target_value = target_mapping[organ_name][0]
+            else:
+                # Use body contour class value for unmapped organs
+                target_value = body_contour_value
+                #print(f'Processing for class {class_value.item()}')
+                #print(f'Not found {organ_name} in target mapping, replaced with body contour.')
+
+        # Replace class values in the target mask
+        target_mask[source_mask == class_value] = target_value
+
+    return target_mask
+
+def convert_segmentation_mask(source_mask, source_csv, target_csv, body_contour_value=1000):
+    """
+    Converts segmentation mask values from source modality to target modality based on organ name mapping.
+
+    Parameters:
+    - source_mask (ndarray): The source segmentation mask array.
+    - source_csv (str): Path to the CSV file of the source modality (CT or MR).
+    - target_csv (str): Path to the CSV file of the target modality (MR or CT).
+    - body_contour_value (int): The class value for "body contour" in the target modality.
+
+    Returns:
+    - target_mask (ndarray): The converted segmentation mask.
+    """
+    # Load the source and target anatomy lists
+    source_df = pd.read_csv(source_csv)
+    target_df = pd.read_csv(target_csv)
+
+    # Create dictionaries mapping class values to organ names and vice versa
+    source_mapping = {}
+    for _, row in source_df.iterrows():
+        organ_name = row['Organ Name']
+        class_value = row.iloc[0]
+        source_mapping.setdefault(organ_name, []).append(class_value)
+
+    target_mapping = {}
+    for _, row in target_df.iterrows():
+        organ_name = row['Organ Name']
+        class_value = row.iloc[0]
+        target_mapping.setdefault(organ_name, []).append(class_value)
+
+    # Create a reverse mapping from class values to organ names for the source modality
+    class_to_organ = {class_value: organ_name for organ_name, class_values in source_mapping.items() for class_value in class_values}
+    # Initialize the target mask
+    target_organ_names = list(target_mapping.keys())
+    target_mask = np.full_like(source_mask, 0, dtype=source_mask.dtype)
+
+    # Convert each unique class in the source mask
+    for class_value in np.unique(source_mask):
+        # Find the corresponding organ name in the source modality
+        organ_name = class_to_organ.get(class_value, None)
+        if class_value == 0:
+            target_value = 0
+        else:
+            # If organ name exists, find the corresponding target class values
+            if organ_name and organ_name in target_mapping:
+                # Pick the first target class value (or handle overlaps if needed)
+                target_value = target_mapping[organ_name][0]
+            else:
+                # Manual mapping: source organ name → target organ name
+                manual_mapping = {
+                    'intervertebral_discs': 'spinal_cord',
+                    'quadriceps_femoris_left':'gluteus_maximus_left',
+                    'quadriceps_femoris_right':'gluteus_maximus_right',
+                    'thigh_medial_compartment_left': 'gluteus_maximus_left',
+                    'thigh_medial_compartment_right': 'gluteus_maximus_right',
+                    'thigh_posterior_compartment_left': 'gluteus_maximus_left',
+                    'thigh_posterior_compartment_right': 'gluteus_maximus_right',
+                    'sartorius_left': 'gluteus_maximus_left',
+                    'sartorius_right': 'gluteus_maximus_right',
+                    # Add more mappings here as needed
+                }
+                # Check manual mapping first
+                if organ_name in manual_mapping and manual_mapping[organ_name] in target_mapping:
+                    matched_name = manual_mapping[organ_name]
+                    target_value = target_mapping[matched_name][0]
+                    print(f"[Manual match] '{organ_name}' → '{matched_name}' → label {target_value}")
+                else:
+                    # Fuzzy match fallback
+                    close_matches = difflib.get_close_matches(organ_name, target_organ_names, n=1, cutoff=0.4)
+                    if close_matches:
+                        matched_name = close_matches[0]
+                        target_value = target_mapping[matched_name][0]
+                        print(f"[Fuzzy match] '{organ_name}' → '{matched_name}' → label {target_value}")
+                    else:
+                        print(f"[Warning] No match for '{organ_name}', using body contour value.")
+                        target_value = body_contour_value
+                '''close_matches = difflib.get_close_matches(organ_name, target_organ_names, n=1, cutoff=0.4)
+                if close_matches:
+                    matched_name = close_matches[0]
+                    target_value = target_mapping[matched_name][0]
+                    print(f"[Fuzzy match] '{organ_name}' → '{matched_name}' → label {target_value}")
+                else:
+                    print(f"[Warning] No match for '{organ_name}', using body contour value.")
+                    target_value = body_contour_value'''
+        # Replace class values in the target mask
+        target_mask[source_mask == class_value] = target_value
+
+    return target_mask
+
+def run_mask_conversion(
+    mask = r'E:\Projects\yang_proj\data\synthrad\Task1\pelvis\1PA001\ct_seg.nii.gz',
+    img = r'E:\Projects\yang_proj\data\synthrad\Task1\pelvis\1PA001\ct.nii.gz',
+    MR_csv = r'E:\Projects\yang_proj\SynthRad_GAN\synthrad_conversion\TA2_MR_for_convert.csv',
+    CT_csv = r'E:\Projects\yang_proj\SynthRad_GAN\synthrad_conversion\TA2_CT_for_convert.csv',
+    output_path = r'mr_mask_from_ct.nii.gz', # output_path = r'ct_mask_from_mr.nii.gz'
+    mode = 'ct2mr'
+):
+    if mode == 'ct2mr':
+        body_threshold=-500
+        source_csv = CT_csv
+        target_csv = MR_csv
+    elif mode == 'mr2ct':
+        body_threshold=5
+        source_csv = MR_csv
+        target_csv = CT_csv
+
+    source_mask = mask
+    img = img
+    
+    seg_metadata = nib.load(source_mask)
+    seg = seg_metadata.get_fdata()
+    affine = seg_metadata.affine
+
+    img_metadata = nib.load(img)
+    img = img_metadata.get_fdata()
+    affine = img_metadata.affine
+
+    '''body_contour = np.zeros_like(img, dtype=np.int16)
+    for i in range(img.shape[2]):
+        slice_data = img[:, :, i]
+        body_contour[:, :, i] = create_body_contour(slice_data, body_threshold)
+    contour = body_contour
+    seg_with_contour = seg+contour'''
+    seg_with_contour = seg
+    target_mask = convert_segmentation_mask(seg_with_contour, source_csv, target_csv, body_contour_value=1)
+    if mode == 'ct2mr':
+        csv_simulation_file = MR_csv
+        csv_values = pd.read_csv(csv_simulation_file, header=None).to_numpy()
+        target_mask = process_segmentation(target_mask, csv_values)
+    elif mode == 'mr2ct':
+        csv_simulation_file = CT_csv
+        target_mask = process_CT_segmentation_numpy(target_mask, csv_simulation_file)
+    img_processed = nib.Nifti1Image(target_mask, affine)
+    nib.save(img_processed, output_path)
+
+
+def run_mask_conversion_synthrad_test(synthrad_root = r'E:\Projects\yang_proj\data\synthrad\Task1\pelvis', patient_list=['1PA001'], mode = 'ct2mr', output_csv_file = 'ct2mr_conversion.csv'):
+    dataset_list = []
+    for patient in tqdm(patient_list):
+        mr_mask = os.path.join(synthrad_root, patient, 'mr_merged_seg.nii.gz')
+        mr_img = os.path.join(synthrad_root, patient, 'mr.nii.gz')
+        ct_mask = os.path.join(synthrad_root, patient, 'ct_seg.nii.gz') 
+        ct_img = os.path.join(synthrad_root, patient, 'ct.nii.gz')
+        MR_csv = r'synthrad_conversion/TA2_MR_for_convert.csv'
+        CT_csv = r'synthrad_conversion/TA2_CT_for_convert.csv'
+        if mode == 'ct2mr':
+            preprocessed_mr_path = r'E:\Projects\yang_proj\data\anika\MR_processed'
+            preprocessed_mr_img = os.path.join(preprocessed_mr_path, f'mr_{patient}.nii.gz')
+            output_path = os.path.join(synthrad_root, patient, 'mr_mask_from_ct.nii.gz')  
+            csv_mr_line = [patient,0,output_path,preprocessed_mr_img]
+        
+        elif mode == 'mr2ct':
+            output_path = os.path.join(synthrad_root, patient, 'ct_mask_from_mr.nii.gz')
+            csv_mr_line = [patient,0,output_path,ct_img]
+        run_mask_conversion(mr_mask, mr_img, ct_mask, ct_img, MR_csv, CT_csv, output_path, mode)
+        dataset_list.append(csv_mr_line)
+        
+    import csv
+    with open(output_csv_file, 'w', newline='') as f:
+        csvwriter = csv.writer(f)
+        csvwriter.writerow(['id', 'Aorta_diss', 'seg', 'img'])
+        csvwriter.writerows(dataset_list) 
+
+def run_mask_conversion_csv(csv_file = r'E:\Projects\yang_proj\data\synthrad\processed\processed_ct_csv_file.csv', mode = 'ct2mr', output_csv_file = 'ct2mr_conversion.csv'):
+    data_frame = pd.read_csv(csv_file)
+    if len(data_frame) == 0:
+        raise RuntimeError(f"Found 0 images in: {csv_file}")
+    patient_IDs = data_frame.iloc[:, 0].tolist()
+    Aorta_diss = data_frame.iloc[:, 1].tolist()
+    segs =  data_frame.iloc[:, 2].tolist()
+    images = data_frame.iloc[:, 3].tolist()
+    aligned_segs = data_frame.iloc[:, 4].tolist()
+    dataset_list = []
+    synthrad_root = r"E:\Projects\yang_proj\data\synthrad\Task1\pelvis"
+    from tqdm import tqdm
+    for idx in tqdm(range(len(images))):
+        MR_csv = r'synthrad_conversion/TA2_MR_for_convert.csv'
+        CT_csv = r'synthrad_conversion/TA2_CT_for_convert.csv'
+        patient = patient_IDs[idx]
+        if mode == 'ct2mr':
+            ct_mask = segs[idx]
+            ct_img = images[idx]
+            preprocessed_mr_path = r'E:\Projects\yang_proj\data\anika\MR_processed'
+            preprocessed_mr_img = os.path.join(preprocessed_mr_path, f'mr_{patient}.nii.gz')
+
+            mr_mask_from_ct_folder = r'E:\Projects\yang_proj\data\synthrad\mr_mask_from_ct'
+            output_path = os.path.join(mr_mask_from_ct_folder, f'{patient}_mr_mask_from_ct.nii.gz')   
+            csv_mr_line = [patient,0,output_path,preprocessed_mr_img]
+            run_mask_conversion(ct_mask, ct_img, MR_csv, CT_csv, output_path, mode)
+        
+        elif mode == 'mr2ct':
+            mr_mask = os.path.join(synthrad_root, patient, 'mr_merged_seg.nii.gz')
+            mr_img = os.path.join(synthrad_root, patient, 'mr.nii.gz')
+            output_path = os.path.join(synthrad_root, patient, 'ct_mask_from_mr.nii.gz')
+            csv_mr_line = [patient,0,output_path,ct_img]
+            run_mask_conversion(mr_mask, mr_img, MR_csv, CT_csv, output_path, mode)
+        dataset_list.append(csv_mr_line)
+        
+    import csv
+    with open(output_csv_file, 'w', newline='') as f:
+        csvwriter = csv.writer(f)
+        csvwriter.writerow(['id', 'Aorta_diss', 'seg', 'img'])
+        csvwriter.writerows(dataset_list) 
+
+if __name__ == "__main__":
+    csv_file = r'E:\Projects\yang_proj\data\synthrad\processed\processed_csv_file.csv'
+    mode = 'ct2mr'
+    output_csv_file = r'E:\Projects\yang_proj\SynthRad_GAN\synthrad_conversion\datacsv\ct2mr_conversion.csv'
+    run_mask_conversion_csv(csv_file = csv_file, mode = mode, output_csv_file = output_csv_file)

dataprocesser/Preprocess_MR_Mask_generation.py

@@ -0,0 +1,306 @@
+import os
+import pandas as pd
+import numpy as np
+import nrrd
+import SimpleITK as sitk
+import cv2
+
+from dataprocesser.preprocess_MR import step3_vibe_resetsignal
+"""
+该代码用于处理一组 MR 图像和对应的分割图，应用掩膜、进行归一化，并根据 CSV 文件中的仿真 MR 灰度值对分割图进行替换。最后将处理后的 MR 图像和分割图保存。
+
+主要步骤：
+1. **读取数据**：从指定的文件夹中读取 MR 图像和对应的分割图。
+2. **归一化处理**：对 MR 图像进行归一化，将其值范围映射到 0 到 255 之间。
+3. **轮廓提取**：从归一化后的 MR 图像中提取出主体区域的轮廓（根据给定的阈值分割），创建掩膜。
+4. **掩膜应用**：将提取出的掩膜应用到归一化后的 MR 图像上，保留主体区域，抑制背景。
+5. **分割图处理**：读取对应的分割图，并与提取出的轮廓进行叠加，之后根据 CSV 文件中的仿真 CT 值替换分割图中的灰度值。
+6. **图像保存**：将处理后的 MR 图像和修改后的分割图保存到指定的输出文件夹中，保证其空间属性和几何信息与输入图像一致。
+7. **输出**：在 ITK-SNAP 等医学图像工具中打开时, MR 图像和分割图能够保持同步和正确的比例显示。
+
+函数简介：
+- `normalize`: 对 MR 图像进行归一化处理，将像素值范围映射到 [0, 255]。
+- `create_body_mask`: 从图像中提取出身体的轮廓，生成二值掩膜。
+- `apply_mask`: 将提取的掩膜应用到 MR 图像上，保留轮廓内部的区域。
+- `process_segmentation`: 读取分割图，并根据 CSV 文件中的仿真 CT 值对其灰度值进行替换。
+- `process_image`: 处理单个 MR 图像及其对应的分割图，包括归一化、轮廓提取、掩膜应用、分割图处理等。
+- `process_folder`: 处理整个文件夹中的 MR 图像和分割图，逐一处理所有图像并保存结果。
+"""
+
+# 归一化函数
+def normalize(img, vmin_out=0, vmax_out=1, norm_min_v=None, norm_max_v=None, epsilon=1e-5):
+    if norm_min_v is None and norm_max_v is None:
+        norm_min_v = np.min(img)
+        norm_max_v = np.max(img)
+    img = np.clip(img, norm_min_v, norm_max_v)
+    img = (img - norm_min_v) / (norm_max_v - norm_min_v + epsilon)
+    img = img * (vmax_out - vmin_out) + vmin_out
+    return img
+
+# 创建轮廓掩膜
+def create_body_mask_simple(numpy_img, body_threshold=50):
+    numpy_img = numpy_img.astype(np.int16)
+    body_mask = np.where(numpy_img > body_threshold, 1, 0).astype(np.uint8)
+    contours, _ = cv2.findContours(body_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    mask = np.zeros_like(body_mask, dtype=np.uint8)
+
+    if contours:
+        largest_contour = max(contours, key=cv2.contourArea)
+        mask = np.ascontiguousarray(mask)
+        largest_contour = np.ascontiguousarray(largest_contour)
+        cv2.drawContours(mask, [largest_contour], -1, 1, thickness=cv2.FILLED)
+
+    return mask
+
+def create_body_mask(numpy_img, body_threshold=-500, min_contour_area=10000):
+    """
+    Create a binary body mask from a CT image tensor, using a specific threshold for the body parts.
+
+    Args:
+    tensor_img (torch.Tensor): A tensor representation of a grayscale CT image, with intensity values from -1024 to 1500.
+
+    Returns:
+    torch.Tensor: A binary mask tensor where the entire body region is 1 and the background is 0.
+    """
+    # Convert tensor to numpy array
+    numpy_img = np.ascontiguousarray(numpy_img.astype(np.int16))  # Ensure we can handle negative values correctly
+    #numpy_img = numpy_img.astype(np.int16)
+
+    # Threshold the image at -500 to separate potential body from the background
+    binary_img = np.where(numpy_img > body_threshold, 1, 0).astype(np.uint8)
+
+    # Find contours from the binary image
+    contours, _ = cv2.findContours(binary_img, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
+
+    # Create an empty mask
+    mask = np.zeros_like(binary_img)
+
+    VERBOSE = False
+    # Fill all detected body contours
+    if contours:
+        for contour in contours:
+            if cv2.contourArea(contour) >= min_contour_area:
+                if VERBOSE:
+                    print('current contour area: ', cv2.contourArea(contour), 'threshold: ', min_contour_area)
+                cv2.drawContours(mask, [contour], -1, 1, thickness=cv2.FILLED)
+
+    return mask
+
+def apply_mask(normalized_image_array, mask_array):
+    return normalized_image_array * mask_array
+
+def print_all_info(data, title):
+    print(f'min, max of {title}:', np.min(data), np.max(data))
+
+# process the segmentation, replace the classes with simulated MR values
+def process_segmentation(combined_array, csv_simulation_values, mr_signal_formula=step3_vibe_resetsignal.calculate_signal_vibe):
+    combined_array = combined_array.astype(np.int16)
+    print_all_info(combined_array, 'combine')
+    # two columns of  unique value 和 simulation value
+    # the first element will not be included
+    organ_indexs = csv_simulation_values[1:, 0]  # first column: organ index
+    T1_values = csv_simulation_values[1:, 1]  # second column: simulate MRI value
+    T2_values = csv_simulation_values[1:, 2]
+    Rho_values = csv_simulation_values[1:, 3]
+    order_begin_from_0 = True if organ_indexs.astype(int).min()==0 else False
+    #print('organ order number begin from 0:', order_begin_from_0)
+    #print(organ_indexs)
+    assign_value_mask = np.zeros_like(combined_array)
+
+    step=0
+    for step in range(len(organ_indexs)):
+        organ_index = organ_indexs[step] # in csv file, organs begin with 1       
+        t1 = float(T1_values[step])
+        t2 = float(T2_values[step])
+        rho = float(Rho_values[step])
+
+        simulation_value = mr_signal_formula(t1, t2, rho)
+        organ_index = int(organ_index)
+        if order_begin_from_0:
+            #print("order in csv begin from 0")
+            assign_value_mask[combined_array == organ_index+1] = simulation_value #  organ_index+ 1
+        else:
+            #print("order in csv begin from 1")
+            assign_value_mask[combined_array == organ_index] = simulation_value
+        step+=1
+    print_all_info(assign_value_mask, 'assignment')
+    return assign_value_mask
+
+# 处理单个图像和分割图
+def process_image(input_path, contour_path, seg_path, csv_simulation_values, output_path1, output_path2, body_threshold):
+    # 读取原始 MR 图像和分割图
+    if input_path.endswith('.nrrd'):
+        img, header = nrrd.read(input_path)
+        segmentation_img, header_seg = nrrd.read(seg_path)
+    elif input_path.endswith('.nii.gz') or input_path.endswith('.nii'):
+        import nibabel as nib
+        img_metadata = nib.load(input_path)
+        img = img_metadata.get_fdata()
+        affine = img_metadata.affine
+        
+        seg_metadata = nib.load(seg_path)
+        segmentation_img = seg_metadata.get_fdata()
+    
+    # 归一化处理
+    norm_max=255 #255
+    low_percentile = 5
+    high_percentile = 90
+    img_normalized = normalize(img, 0, norm_max, np.percentile(img, low_percentile), np.percentile(img, high_percentile), epsilon=0)
+    
+    # 提取轮廓图
+    body_contour = np.zeros_like(img, dtype=np.int16)
+    for i in range(img.shape[2]):
+        slice_data = img[:, :, i]
+        body_contour[:, :, i] = create_body_mask(slice_data, body_threshold=body_threshold)
+    
+    # 应用掩膜到归一化 MR 图像
+    masked_image = apply_mask(img_normalized, body_contour)
+    
+    # 处理分割图
+    # add contour background to the segmentation (all region inside body + 1)
+    combined_array = segmentation_img + body_contour
+    combined_array = np.clip(combined_array, 0, np.max(segmentation_img) + 1)
+    print_all_info(segmentation_img, 'seg')
+    processed_segmentation = process_segmentation(combined_array, csv_simulation_values)
+    
+    # normalize to 0-1
+    # masked_image = masked_image/norm_max
+    # processed_segmentation = processed_segmentation/norm_max
+
+    if input_path.endswith('.nrrd'):
+        # 保存处理后的 MR 图像
+        nrrd.write(output_path1, masked_image, header)
+        
+        # 保存处理后的分割图
+        nrrd.write(output_path2, processed_segmentation, header_seg)
+
+        # save the body contour mask
+
+    elif input_path.endswith('.nii.gz') or input_path.endswith('.nii'):
+        img_processed = nib.Nifti1Image(masked_image, affine)
+        nib.save(img_processed, output_path1)
+        seg_processed = nib.Nifti1Image(processed_segmentation, affine)
+        nib.save(seg_processed, output_path2)
+        contour_processed = nib.Nifti1Image(body_contour, affine)
+        
+        # Split the path into directory and filename
+        directory, filename = os.path.split(output_path2)
+        new_filename = filename.replace('seg', 'contour')
+        contour_path = os.path.join(directory, new_filename)
+
+        nib.save(contour_processed, contour_path)
+    return processed_segmentation
+
+# 处理文件夹
+def process_folder(input_folder1, input_folder2, output_folder1, output_folder2, csv_simulation_file, body_threshold=50):
+    # 读取CSV文件获取仿真CT灰度值 (两列)
+    csv_simulation_values = pd.read_csv(csv_simulation_file, header=None).to_numpy()
+
+    # 检查 csv_simulation_values 是否是二维数组
+    if csv_simulation_values.ndim == 1:
+        raise ValueError("CSV 文件格式不正确，应该包含两列：organ_index 和 simulation_value")
+
+    # 确保输出文件夹存在
+    os.makedirs(output_folder1, exist_ok=True)
+    os.makedirs(output_folder2, exist_ok=True)
+
+    for filename in os.listdir(input_folder1):
+        if filename.endswith('.nrrd'):
+            input_file_path = os.path.join(input_folder1, filename)
+            seg_file_path = os.path.join(input_folder2, filename)
+            output_file_path1 = os.path.join(output_folder1, filename)
+            output_file_path2 = os.path.join(output_folder2, filename)
+
+            print(f"Processing {input_file_path} with segmentation {seg_file_path}")
+            processed_seg = process_image(input_file_path, None, seg_file_path, csv_simulation_values, output_file_path1, output_file_path2, body_threshold)
+
+def analyse_hist(input_path):
+    if input_path.endswith('.nrrd'):
+        img, header = nrrd.read(input_path)
+    elif input_path.endswith('.nii.gz'):
+        import nibabel as nib
+        img_metadata = nib.load(input_path)
+        img = img_metadata.get_fdata()
+        affine = img_metadata.affine
+    import numpy as np
+    import matplotlib.pyplot as plt
+
+    # Plot the histogram
+    print('shape of img: ', img.shape)
+    plt.hist(img[:, :, 50], bins=30, edgecolor='black', alpha=0.7)
+    plt.xlabel('Value')
+    plt.ylabel('Frequency')
+    plt.title('Value Distribution')
+    plt.show()
+
+
+def process_csv(csv_file, output_folder1, output_folder2, csv_simulation_file, body_threshold=50, output_mr_csv_file='processed_mr_csv_file.csv'):
+    # 读取CSV文件获取仿真CT灰度值 (两列)
+    csv_simulation_values = pd.read_csv(csv_simulation_file, header=None).to_numpy()
+    #csv_simulation_values = pd.read_csv(csv_simulation_file)
+
+    # 检查 csv_simulation_values 是否是二维数组
+    if csv_simulation_values.ndim == 1:
+        raise ValueError("CSV 文件格式不正确，应该包含两列：organ_index 和 simulation_value")
+
+    # 确保输出文件夹存在
+    os.makedirs(output_folder1, exist_ok=True)
+    os.makedirs(output_folder2, exist_ok=True)
+    
+    from step1_init_data_list import list_img_seg_ad_pIDs_from_new_simplified_csv
+    patient_IDs, Aorta_diss, segs, images = list_img_seg_ad_pIDs_from_new_simplified_csv(csv_file)
+    from tqdm import tqdm
+    dataset_list = []
+    for idx in tqdm(range(len(images))):
+        if (images[idx].endswith('.nii.gz') and segs[idx].endswith('.nii.gz')) or \
+            (images[idx].endswith('.nii') and segs[idx].endswith('.nii')):
+            input_file_path = images[idx]
+            seg_file_path = segs[idx]
+            patient_id = patient_IDs[idx]
+            ad = Aorta_diss[idx]
+            root_dir = os.path.dirname(input_file_path)
+            
+            output_file_path1 = os.path.join(output_folder1, os.path.relpath(input_file_path, start=root_dir))
+            
+            synthrad_basic_mr_name = 'mr'
+            synthrad_basic_seg_name = 'mr_merged_seg'
+            if os.path.basename(output_file_path1) == f'{synthrad_basic_mr_name}.nii.gz' or \
+                os.path.basename(output_file_path1) == f'{synthrad_basic_mr_name}.nii':
+                # Insert the patient ID in the filename
+                output_file_path1 = output_file_path1.replace(f'{synthrad_basic_mr_name}', f'mr_{patient_id}')
+
+            output_file_path2 = os.path.join(output_folder2, os.path.relpath(seg_file_path, start=root_dir))
+            
+            if os.path.basename(output_file_path2) == f'{synthrad_basic_seg_name}.nii.gz' or \
+                os.path.basename(output_file_path2) == f'{synthrad_basic_seg_name}.nii':
+                # Insert the patient ID in the filename
+                output_file_path2 = output_file_path2.replace(f'{synthrad_basic_seg_name}', f'mr_seg_{patient_id}')
+
+            print(f"Processing {input_file_path} with segmentation {seg_file_path}")
+            print(f"Save results to {output_file_path1} and {output_file_path2}")
+            
+            processed_seg = process_image(input_file_path, None, seg_file_path, csv_simulation_values, output_file_path1, output_file_path2, body_threshold)
+
+            # processed_mr_csv_file = ...
+            csv_mr_line = [patient_id,ad,output_file_path2,output_file_path1]
+            dataset_list.append(csv_mr_line)
+
+    import csv
+    with open(output_mr_csv_file, 'w', newline='') as f:
+        csvwriter = csv.writer(f)
+        csvwriter.writerow(['id', 'Aorta_diss', 'seg', 'img'])
+        csvwriter.writerows(dataset_list) 
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Process MR images and segmentation maps, apply masks and replace grayscale values.")
+    parser.add_argument('--input_folder1', required=True, help="Path to the folder containing input MR .nrrd files.")
+    parser.add_argument('--input_folder2', required=True, help="Path to the folder containing segmentation .nrrd files.")
+    parser.add_argument('--output_folder1', required=True, help="Path to the folder to save the output MR files.")
+    parser.add_argument('--output_folder2', required=True, help="Path to the folder to save the output segmentation files.")
+    parser.add_argument('--csv_simulation_file', required=True, help="CSV file containing simulated CT grayscale values.")
+    parser.add_argument('--body_threshold', type=int, default=50, help="Threshold to separate body from background.")
+    args = parser.parse_args()
+
+    process_folder(args.input_folder1, args.input_folder2, args.output_folder1, args.output_folder2, args.csv_simulation_file, args.body_threshold)

dataprocesser/Preprocess_MR_Masks_overlay.py

@@ -0,0 +1,78 @@
+import os
+os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"  
+import nrrd
+import numpy as np
+
+def load_nifti_file(input_path):
+    if input_path.endswith('.nrrd'):
+        data, header = nrrd.read(input_path)
+        return data, header
+    elif input_path.endswith('.nii.gz') or input_path.endswith(".nii"):
+        import nibabel as nib
+        img_metadata = nib.load(input_path)
+        img = img_metadata.get_fdata()
+        affine = img_metadata.affine
+        return img, affine
+
+def save_nrrd_file(data, HeaderOrAffine, input_path, save_path):
+    #nrrd.write(save_path, data, header)
+    if input_path.endswith('.nrrd'):
+        nrrd.write(save_path, data, HeaderOrAffine)
+        
+    elif input_path.endswith('.nii.gz') or input_path.endswith(".nii"):
+        import nibabel as nib
+        img_processed = nib.Nifti1Image(data, HeaderOrAffine)
+        nib.save(img_processed, save_path)
+
+def overlay_images(mask_data, organ_data):
+    # Combine the images by adding the pixel values
+    organ_data = np.where(organ_data == 1, organ_data + 98, organ_data)
+    organ_data = np.where(organ_data == 2, organ_data + 197, organ_data)
+    organ_data = np.where(organ_data == 3, organ_data + 296, organ_data)
+    combined_data = mask_data + organ_data
+    return combined_data
+
+def main(files1, files2, output_folder=None):
+    # files is the list including all basic MR segmentations
+    # files is the list including all basic MR tissue segmentations
+        
+    print("preprocess length of seg files: ", len(files1))
+    print("preprocess length of tissue seg files: ", len(files2))
+
+    files2 = [file.replace('seg_tissue', 'seg') for file in files2]
+
+    files1 = set(files1)
+    files2 = set(files2)
+
+    common_files = files1.intersection(files2)
+
+    from tqdm import tqdm
+    for filename in tqdm(common_files):
+        if filename.endswith(".nrrd") or filename.endswith(".nii.gz") or filename.endswith(".nii"):
+            nrrd_path1 =  filename
+            nrrd_path2 = filename.replace('seg', 'seg_tissue')
+            
+            '''
+            if os.path.basename(filename) == 'mr_seg.nii.gz':
+                patient_ID = os.path.basename(os.path.dirname(filename))
+                output_file_name = os.path.basename(filename).replace("seg", f"seg_{patient_ID}") 
+            else:
+                output_file_name = os.path.basename(filename)
+                '''
+            
+            output_file_name = os.path.basename(filename)
+            output_file_name = output_file_name.replace("seg", "merged_seg")
+            if output_folder == None:
+                output_folder_current_patient = os.path.dirname(filename)
+            else:
+                output_folder_current_patient = output_folder
+            save_path = os.path.join(output_folder_current_patient, output_file_name)
+
+            print(f"Processing {nrrd_path1} and {nrrd_path2}, saving to {save_path}")
+
+            data1, header1 = load_nifti_file(nrrd_path1)
+            data2, header2 = load_nifti_file(nrrd_path2)
+            
+            combined_data = overlay_images(data1, data2)
+            save_nrrd_file(combined_data, header1, nrrd_path1, save_path)
+            

dataprocesser/__init__.py

@@ -0,0 +1,8 @@
+from dataprocesser.dataset_registry import DATASET_REGISTRY
+# register all the dataset in DATASET_REGISTRY
+import dataprocesser.dataset_anish 
+import dataprocesser.dataset_combined_csv
+import dataprocesser.dataset_combined_synthrad_anish
+import dataprocesser.dataset_csv_slice
+import dataprocesser.dataset_json
+import dataprocesser.dataset_synthrad

dataprocesser/archive/archiv.py

@@ -0,0 +1,236 @@
+from ..basics import get_file_list,crop_volumes, load_volumes
+def load_batch_slices(train_volume_ds,val_volume_ds, train_batch_size=5,val_batch_size=1,window_width=1,ifcheck=True):
+    patch_func = monai.data.PatchIterd(
+        keys=["source", "target"],
+        patch_size=(None, None, window_width),  # dynamic first two dimensions
+        start_pos=(0, 0, 0)
+    )
+    if window_width==1:
+        patch_transform = Compose(
+            [
+                SqueezeDimd(keys=["source", "target"], dim=-1),  # squeeze the last dim
+            ]
+        )
+    else:
+        patch_transform = None
+    # for training
+    train_patch_ds = monai.data.GridPatchDataset(
+        data=train_volume_ds, patch_iter=patch_func, transform=patch_transform, with_coordinates=False)
+    train_loader = DataLoader(
+        train_patch_ds,
+        batch_size=train_batch_size,
+        num_workers=2,
+        pin_memory=torch.cuda.is_available(),
+    )
+
+    # for validation
+    val_loader = DataLoader(
+        val_volume_ds, 
+        num_workers=1, 
+        batch_size=val_batch_size,
+        pin_memory=torch.cuda.is_available())
+    
+    if ifcheck:
+        check_batch_data(train_loader,val_loader,train_patch_ds,val_volume_ds,train_batch_size,val_batch_size)
+    return train_loader,val_loader
+
+def load_batch_slices3D(train_volume_ds,val_volume_ds, train_batch_size=5,val_batch_size=1,ifcheck=True):
+    patch_func = monai.data.PatchIterd(
+        keys=["source", "target"],
+        patch_size=(None, None,32),  # dynamic first two dimensions
+        start_pos=(0, 0, 0)
+    )
+
+    # for training
+    train_patch_ds = monai.data.GridPatchDataset(
+        data=train_volume_ds, patch_iter=patch_func, with_coordinates=False)
+    train_loader = DataLoader(
+        train_patch_ds,
+        batch_size=train_batch_size,
+        num_workers=2,
+        pin_memory=torch.cuda.is_available(),
+    )
+
+    # for validation
+    val_loader = DataLoader(
+        val_volume_ds, 
+        num_workers=1, 
+        batch_size=val_batch_size,
+        pin_memory=torch.cuda.is_available())
+    
+    if ifcheck:
+        check_batch_data(train_loader,val_loader,train_patch_ds,val_volume_ds,train_batch_size,val_batch_size)
+    return train_loader,val_loader
+
+
+
+def mydataloader_3d(data_pelvis_path,
+                   train_number,
+                   val_number,
+                   train_batch_size,
+                   val_batch_size,
+                   saved_name_train='./train_ds_2d.csv',
+                   saved_name_val='./val_ds_2d.csv',
+                   resized_size=(600,400,150),
+                   div_size=(16,16,16),
+                   ifcheck_volume=True,):
+    # volume-level transforms for both image and segmentation
+    normalize='zscore'
+    train_transforms = get_transforms(normalize,resized_size,div_size)
+    
+    train_ds, val_ds = get_file_list(data_pelvis_path, 
+                                     train_number, 
+                                     val_number)
+    #train_volume_ds, val_volume_ds 
+    
+    train_volume_ds,val_volume_ds = load_volumes(train_transforms=train_transforms, 
+                                                train_ds=train_ds, 
+                                                val_ds=val_ds, 
+                                                saved_name_train=saved_name_train, 
+                                                saved_name_val=saved_name_train,
+                                                ifsave=True,
+                                                ifcheck=ifcheck_volume)
+    '''
+    train_loader = DataLoader(train_volume_ds, batch_size=train_batch_size)
+    val_loader = DataLoader(val_volume_ds, batch_size=val_batch_size)
+    '''
+    ifcheck_sclices=False
+    train_loader,val_loader = load_batch_slices3D(train_volume_ds, 
+                                                val_volume_ds, 
+                                                train_batch_size,
+                                                val_batch_size=val_batch_size,
+                                                ifcheck=ifcheck_sclices)
+                                                
+    return train_loader,val_loader,train_transforms
+
+
+from torchvision.utils import save_image
+def save_dataset_as_png(train_ds, train_volume_ds,saved_img_folder,saved_label_folder):    
+    train_loader = DataLoader(train_volume_ds, batch_size=1)
+    for idx, train_check_data in enumerate(train_loader):
+        image_volume = train_check_data['image']
+        label_volume = train_check_data['label']
+        current_item = train_ds[idx]
+        file_name_prex = os.path.basename(os.path.dirname(current_item['image']))
+        slices_num=image_volume.shape[-1]
+        for i in range(slices_num):
+            image_i=image_volume[0,0,:,:,i]
+            label_i=label_volume[0,0,:,:,i]
+            #print(label_volume.shape)
+            #SaveImage(output_dir=saved_img_folder, output_postfix=f'{file_name_prex}_image', output_ext='.png', resample=True)(image_volume[0,:,:,:,0])
+            save_image(image_i, f'{saved_img_folder}\{file_name_prex}_image_{i}.png')
+            save_image(label_i, f'{saved_label_folder}\{file_name_prex}_label_{i}.png')
+
+def pre_dataset_for_stylegan(data_pelvis_path,
+                            normalize,
+                            train_number,
+                            val_number,
+                            saved_img_folder,
+                            saved_label_folder,
+                            saved_name_train='./train_ds_2d.csv',
+                            saved_name_val='./val_ds_2d.csv',
+                            resized_size=(600,400,None),
+                            div_size=(16,16,None),):
+    train_transforms = get_transforms(normalize,resized_size,div_size)
+    train_ds, val_ds = get_file_list(data_pelvis_path, 
+                                     train_number, 
+                                     val_number)
+    train_volume_ds, _ = load_volumes(train_transforms, 
+                                                train_ds, 
+                                                val_ds, 
+                                                saved_name_train, 
+                                                saved_name_val,
+                                                ifsave=False,
+                                                ifcheck=False)
+    save_dataset_as_png(train_ds, train_volume_ds,saved_img_folder,saved_label_folder)
+    return train_ds,train_volume_ds
+
+def sum_slices(data_pelvis_path, num=180):
+    train_ds, val_ds=get_file_list(data_pelvis_path, 0, num)
+    train_ds_2d, val_ds_2d,\
+    all_slices_train,all_slices_val,\
+    shape_list_train,shape_list_val = transform_datasets_to_2d(train_ds, val_ds, 
+                                                            saved_name_train='./train_ds_2d.csv', 
+                                                            saved_name_val='./val_ds_2d.csv', 
+                                                            ifsave=False)
+    print(all_slices_val)
+    return all_slices_val
+
+def transform_datasets_to_2d(train_ds, val_ds, saved_name_train, saved_name_val,ifsave=True):
+    # Load 2D slices of CT images
+    train_ds_2d = []
+    val_ds_2d = []
+    shape_list_train = []
+    shape_list_val = []
+    all_slices_train=0
+    all_slices_val=0
+
+    # Load 2D slices for training
+    for sample in train_ds:
+        train_ds_2d_image = LoadImaged(keys=["source","target"],image_only=True, ensure_channel_first=False, simple_keys=True)(sample)
+        name = os.path.basename(os.path.dirname(sample['image']))
+        num_slices = train_ds_2d_image["source"].shape[-1]
+        shape_list_train.append({'patient': name, 'shape': train_ds_2d_image["image"].shape})
+        for i in range(num_slices):
+            train_ds_2d.append({'image': train_ds_2d_image['image'][:,:,i], 'label': train_ds_2d_image['label'][:,:,i]})
+        all_slices_train += num_slices
+
+    # Load 2D slices for validation
+    for sample in val_ds:
+        val_ds_2d_image = LoadImaged(keys=["source","target"],image_only=True, ensure_channel_first=False, simple_keys=True)(sample)
+        name = os.path.basename(os.path.dirname(sample['image']))
+        shape_list_val.append({'patient': name, 'shape': val_ds_2d_image["image"].shape})
+        num_slices = val_ds_2d_image["image"].shape[-1]
+        for i in range(num_slices):
+            val_ds_2d.append({'image': val_ds_2d_image['image'][:,:,i], 'label': val_ds_2d_image['label'][:,:,i]})
+        all_slices_val += num_slices
+    # Save shape list to csv
+    if ifsave:
+        np.savetxt(saved_name_train,shape_list_train,delimiter=',',fmt='%s',newline='\n') # f means format, r means raw string
+        np.savetxt(saved_name_val,shape_list_val,delimiter=',',fmt='%s',newline='\n') # f means format, r means raw string
+    return train_ds_2d, val_ds_2d, all_slices_train, all_slices_val, shape_list_train, shape_list_val
+
+def get_train_val_loaders(train_ds_2d, val_ds_2d, batch_size, val_batch_size,normalize, resized_size=(600,400), div_size=(16,16,None),):
+    # Define transforms
+    train_transforms = get_transforms(normalize,resized_size,div_size)
+    train_transforms_list=train_transforms.__dict__['transforms']
+    batch_size = batch_size
+    # Create training dataset and data loader
+    train_dataset = Dataset(data=train_ds_2d, transform=train_transforms)
+    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=False, num_workers=1, pin_memory=True)
+
+    val_batch_size = val_batch_size
+    # Create validation dataset and data loader
+    val_dataset = Dataset(data=val_ds_2d, transform=train_transforms)
+    val_loader = DataLoader(val_dataset, batch_size=val_batch_size, shuffle=False, num_workers=1, pin_memory=True)
+    return train_loader, val_loader, train_transforms_list,train_transforms
+
+def mydataloader(data_pelvis_path,
+                 train_number,
+                 val_number,
+                 batch_size,
+                 val_batch_size,
+                 saved_name_train='./train_ds_2d.csv',
+                 saved_name_val='./val_ds_2d.csv',
+                 resized_size=(600,400)): 
+    train_ds, val_ds = get_file_list(data_pelvis_path, 
+                                     train_number, 
+                                     val_number)
+    train_ds_2d, val_ds_2d,\
+    all_slices_train,all_slices_val,\
+    shape_list_train,shape_list_val = transform_datasets_to_2d(train_ds, val_ds, 
+                                                            saved_name_train, 
+                                                            saved_name_val,ifsave=True)
+    
+    train_loader, val_loader, \
+    train_transforms_list,train_transforms = get_train_val_loaders(train_ds_2d, 
+                                                                val_ds_2d, 
+                                                                batch_size=batch_size, 
+                                                                val_batch_size=val_batch_size,
+                                                                normalize='zscore',
+                                                                resized_size=resized_size,
+                                                                div_size=(16,16,None),)
+    return train_loader,val_loader,\
+            train_transforms_list,train_transforms,\
+            all_slices_train,all_slices_val,\
+            shape_list_train,shape_list_val

dataprocesser/archive/basics.py

@@ -0,0 +1,167 @@
+import monai
+import os
+import numpy as np
+from monai.transforms import (
+    Compose,
+    LoadImaged,
+    Rotate90d,
+    ScaleIntensityd,
+    EnsureChannelFirstd,
+    ResizeWithPadOrCropd,
+    DivisiblePadd,
+    ThresholdIntensityd,
+    NormalizeIntensityd,
+    SqueezeDimd,
+    ShiftIntensityd,
+    Identityd,
+    CenterSpatialCropd,
+    ScaleIntensityRanged,
+    Spacingd,
+)
+from torch.utils.data import DataLoader
+from .checkdata import check_volumes, save_volumes
+def get_file_list(data_pelvis_path, train_number, val_number, source='mr', target='ct'):
+    #list all files in the folder
+    file_list=[i for i in os.listdir(data_pelvis_path) if 'overview' not in i]
+    file_list_path=[os.path.join(data_pelvis_path,i) for i in file_list]
+    #list all ct and mr files in folder
+    source_file_list=[os.path.join(j,f'{source}.nii.gz') for j in file_list_path]
+    target_file_list=[os.path.join(j,f'{target}.nii.gz') for j in file_list_path] #mr
+    # Dict Version
+    # source -> image
+    # target -> label
+    train_ds = [{'source': i, 'target': j, 'A_paths': i, 'B_paths': j} for i, j in zip(source_file_list[0:train_number], target_file_list[0:train_number])]
+    val_ds = [{'source': i, 'target': j, 'A_paths': i, 'B_paths': j} for i, j in zip(source_file_list[-val_number:], target_file_list[-val_number:])]
+    print('all files in dataset:',len(file_list))
+    return train_ds, val_ds
+
+def load_volumes(train_transforms,val_transforms,
+                 train_crop_ds, val_crop_ds, 
+                 train_ds, val_ds, 
+                 saved_name_train=None, saved_name_val=None,
+                 ifsave=False,ifcheck=False):
+    train_volume_ds = monai.data.Dataset(data=train_crop_ds, transform=train_transforms) 
+    val_volume_ds = monai.data.Dataset(data=val_crop_ds, transform=val_transforms)
+    if ifsave:
+        save_volumes(train_ds, val_ds, saved_name_train, saved_name_val)
+    if ifcheck:
+        check_volumes(train_ds, train_volume_ds, val_volume_ds, val_ds)
+    return train_volume_ds,val_volume_ds
+
+def crop_volumes(train_ds, val_ds,center_crop,resized_size=(512,512,None),pad='minimum'):
+    if center_crop>0:
+        crop=Compose([LoadImaged(keys=["source", "target"]),
+                    EnsureChannelFirstd(keys=["source", "target"]),
+                    CenterSpatialCropd(keys=["source", "target"], roi_size=(-1,-1,center_crop)),
+                    
+                    ])
+        train_crop_ds = monai.data.Dataset(data=train_ds, transform=crop)
+        val_crop_ds = monai.data.Dataset(data=val_ds, transform=crop)
+        print('center crop:',center_crop)
+    else:
+        crop=Compose([LoadImaged(keys=["source", "target"]),
+            EnsureChannelFirstd(keys=["source", "target"]),
+            ])
+        train_crop_ds = monai.data.Dataset(data=train_ds, transform=crop)
+        val_crop_ds = monai.data.Dataset(data=val_ds, transform=crop)
+    return train_crop_ds, val_crop_ds 
+
+def get_transforms(configs, mode='train'):
+    normalize=configs.dataset.normalize
+    pad=configs.dataset.pad
+    resized_size=configs.dataset.resized_size
+    WINDOW_WIDTH=configs.dataset.WINDOW_WIDTH
+    WINDOW_LEVEL=configs.dataset.WINDOW_LEVEL
+    prob=configs.dataset.augmentationProb
+    background=configs.dataset.background
+
+    transform_list=[]
+    min, max=WINDOW_LEVEL-(WINDOW_WIDTH/2), WINDOW_LEVEL+(WINDOW_WIDTH/2)
+    transform_list.append(ThresholdIntensityd(keys=["target"], threshold=min, above=True, cval=background))
+    #transform_list.append(ThresholdIntensityd(keys=["target"], threshold=max, above=False, cval=-1000))
+    # filter the source images
+    # transform_list.append(ThresholdIntensityd(keys=["source"], threshold=configs.dataset.MRImax, above=False, cval=0))
+    if normalize=='zscore':
+        transform_list.append(NormalizeIntensityd(keys=["source", "target"], nonzero=False, channel_wise=True))
+        print('zscore normalization')
+    elif normalize=='minmax':
+        transform_list.append(ScaleIntensityd(keys=["source", "target"], minv=-1, maxv=1.0))
+        print('minmax normalization')
+
+    elif normalize=='scale4000':
+        transform_list.append(ScaleIntensityd(keys=["source"], minv=-1, maxv=1))
+        transform_list.append(ScaleIntensityd(keys=["target"], minv=0))
+        transform_list.append(ScaleIntensityd(keys=["target"], factor=-0.99975)) # x=x(1+factor)
+        print('scale1000 normalization')
+
+    elif normalize=='scale1000':
+        transform_list.append(ScaleIntensityd(keys=["source"], minv=0, maxv=1))
+        transform_list.append(ScaleIntensityd(keys=["target"], minv=0))
+        transform_list.append(ScaleIntensityd(keys=["target"], factor=-0.99975)) 
+        print('scale1000 normalization')
+
+    elif normalize=='inputonlyzscore':
+        transform_list.append(NormalizeIntensityd(keys=["source"], nonzero=False, channel_wise=True))
+        print('only normalize input MRI images')
+
+    elif normalize=='inputonlyminmax':
+        transform_list.append(ScaleIntensityd(keys=["source"], minv=configs.dataset.normmin, maxv=configs.dataset.normmax))
+        print('only normalize input MRI images')
+    elif normalize=='none':
+        print('no normalization')
+    transform_list.append(Spacingd(keys=["source"], pixdim=(1.0, 1.0, 1.0), mode="bilinear")) # 
+    transform_list.append(Spacingd(keys=["target", "mask"], pixdim=(1.0, 1.0 , 2.5), mode="bilinear")) #
+    transform_list.append(ResizeWithPadOrCropd(keys=["source", "target", "mask"], spatial_size=resized_size,mode=pad))
+    # transform_list.append(ScaleIntensityRanged(keys=["target"],a_min=WINDOW_LEVEL-(WINDOW_WIDTH/2), a_max=WINDOW_LEVEL+(WINDOW_WIDTH/2),b_min=0, b_max=1, clip=True))
+
+    if mode == 'train':
+        from monai.transforms import (
+            # data augmentation
+            RandRotated,
+            RandZoomd,
+            RandBiasFieldd,
+            RandAffined,
+            RandGridDistortiond,
+            RandGridPatchd,
+            RandShiftIntensityd,
+            RandGibbsNoised,
+            RandAdjustContrastd,
+            RandGaussianSmoothd,
+            RandGaussianSharpend,
+            RandGaussianNoised,
+        )
+        Aug=True
+        if Aug:
+            transform_list.append(RandRotated(keys=["source", "target", "mask"], range_x = 0.1, range_y = 0.1, range_z = 0.1, prob=prob, padding_mode="border", keep_size=True))
+            transform_list.append(RandZoomd(keys=["source", "target", "mask"], prob=prob, min_zoom=0.9, max_zoom=1.3,padding_mode= "minimum" ,keep_size=True))
+            transform_list.append(RandAffined(keys=["source", "target", "mask"],padding_mode="border" , prob=prob))
+            #transform_list.append(Rand3DElasticd(keys=["source", "target"], prob=prob, sigma_range=(5, 8), magnitude_range=(100, 200), spatial_size=None, mode='bilinear'))
+        intensityAug=False
+        if intensityAug:
+            print('intensity data augmentation is used')
+            transform_list.append(RandBiasFieldd(keys=["source"], degree=3, coeff_range=(0.0, 0.1), prob=prob)) # only apply to MRI images
+            transform_list.append(RandGaussianNoised(keys=["source"], prob=prob, mean=0.0, std=0.01))
+            transform_list.append(RandAdjustContrastd(keys=["source"], prob=prob, gamma=(0.5, 1.5)))
+            transform_list.append(RandShiftIntensityd(keys=["source"], prob=prob, offsets=20))
+            transform_list.append(RandGaussianSharpend(keys=["source"], alpha=(0.2, 0.8), prob=prob))
+        
+    #transform_list.append(Rotate90d(keys=["source", "target"], k=3))
+    #transform_list.append(DivisiblePadd(keys=["source", "target"], k=div_size, mode="minimum"))
+    #transform_list.append(Identityd(keys=["source", "target"]))  # do nothing for the no norm case
+    train_transforms = Compose(transform_list)
+    return train_transforms
+
+def get_length(dataset, patch_batch_size):
+    loader=DataLoader(dataset, batch_size=1)
+    iterator = iter(loader)
+    sum_nslices=0
+    for idx in range(len(loader)):
+        check_data = next(iterator)
+        nslices=check_data['source'].shape[-1]
+        sum_nslices+=nslices
+    if sum_nslices%patch_batch_size==0:
+        return sum_nslices//patch_batch_size
+    else:
+        return sum_nslices//patch_batch_size+1
+
+

dataprocesser/archive/checkdata.py

@@ -0,0 +1,91 @@
+from torch.utils.data import DataLoader
+import numpy as np
+import os
+def test_volumes_pixdim(train_volume_ds):
+    train_loader = DataLoader(train_volume_ds, batch_size=1)
+    for step, data in enumerate(train_loader):
+        mr_data=data['source']
+        ct_data=data['target']
+        
+        print(f"source image shape: {mr_data.shape}")
+        print(f"source image affine:\n{mr_data.meta['affine']}")
+        print(f"source image pixdim:\n{mr_data.pixdim}")
+
+        # target image information
+        print(f"target image shape: {ct_data.shape}")
+        print(f"target image affine:\n{ct_data.meta['affine']}")
+        print(f"target image pixdim:\n{ct_data.pixdim}")
+
+
+def check_volumes(train_ds, train_volume_ds, val_volume_ds, val_ds):
+    # use batch_size=1 to check the volumes because the input volumes have different shapes
+    train_loader = DataLoader(train_volume_ds, batch_size=1)
+    val_loader = DataLoader(val_volume_ds, batch_size=1)
+    train_iterator = iter(train_loader)
+    val_iterator = iter(val_loader)
+    print('check training data:')
+    idx=0
+    for idx in range(len(train_loader)):
+        try:
+            train_check_data = next(train_iterator)
+            ds_idx = idx * 1
+            current_item = train_ds[ds_idx]
+            current_name = os.path.basename(os.path.dirname(current_item['image']))
+            print(idx, current_name, 'image:', train_check_data['image'].shape, 'label:', train_check_data['label'].shape)
+        except:
+            ds_idx = idx * 1
+            current_item = train_ds[ds_idx]
+            current_name = os.path.basename(os.path.dirname(current_item['image']))
+            print('check data error! Check the input data:',current_name)
+    print("checked all training data.")
+
+    print('check validation data:')
+    idx=0
+    for idx in range(len(val_loader)):
+        try:
+            val_check_data = next(val_iterator)
+            ds_idx = idx * 1
+            current_item = val_ds[ds_idx]
+            current_name = os.path.basename(os.path.dirname(current_item['image']))
+            print(idx, current_name, 'image:', val_check_data['image'].shape, 'label:', val_check_data['label'].shape)
+        except:
+            ds_idx = idx * 1
+            current_item = val_ds[ds_idx]
+            current_name = os.path.basename(os.path.dirname(current_item['image']))
+            print('check data error! Check the input data:',current_name)
+    print("checked all validation data.")
+
+def save_volumes(train_ds, val_ds, saved_name_train, saved_name_val):
+    shape_list_train=[]
+    shape_list_val=[]
+    # use the function of saving information before
+    for sample in train_ds:
+        name = os.path.basename(os.path.dirname(sample['image']))
+        shape_list_train.append({'patient': name})
+    for sample in val_ds:
+        name = os.path.basename(os.path.dirname(sample['image']))
+        shape_list_val.append({'patient': name})
+    np.savetxt(saved_name_train,shape_list_train,delimiter=',',fmt='%s',newline='\n') # f means format, r means raw string
+    np.savetxt(saved_name_val,shape_list_val,delimiter=',',fmt='%s',newline='\n') # f means format, r means raw string
+
+def check_batch_data(train_loader,val_loader,train_patch_ds,val_volume_ds,train_batch_size,val_batch_size):
+    for idx, train_check_data in enumerate(train_loader):
+        ds_idx = idx * train_batch_size
+        current_item = train_patch_ds[ds_idx]
+        print('check train data:')
+        print(current_item, 'image:', train_check_data['image'].shape, 'label:', train_check_data['label'].shape)
+    
+    for idx, val_check_data in enumerate(val_loader):
+        ds_idx = idx * val_batch_size
+        current_item = val_volume_ds[ds_idx]
+        print('check val data:')
+        print(current_item, 'image:', val_check_data['image'].shape, 'label:', val_check_data['label'].shape)
+
+def len_patchloader(train_volume_ds,train_batch_size):
+    slice_number=sum(train_volume_ds[i]['source'].shape[-1] for i in range(len(train_volume_ds)))
+    print('total slices in training set:',slice_number)
+
+    import math
+    batch_number=sum(math.ceil(train_volume_ds[i]['source'].shape[-1]/train_batch_size) for i in range(len(train_volume_ds)))
+    print('total batches in training set:',batch_number)
+    return slice_number,batch_number

dataprocesser/archive/createsegtransform.py

@@ -0,0 +1,276 @@
+
+from totalsegmentator.python_api import totalsegmentator
+class CreateMaskTransformd: 
+    def __init__(self, keys, tissue_min, tissue_max, bone_min, bone_max, mask_value_bones=2,
+                 if_use_total_seg=False, organ_label_id=52, mask_value_organ=2, fast=True):
+        self.keys = keys
+        self.tissue_min = tissue_min
+        self.tissue_max = tissue_max
+        self.bone_min = bone_min
+        self.bone_max = bone_max
+        self.mask_value_bones = mask_value_bones
+
+        self.if_use_total_seg = if_use_total_seg
+        self.organ_label_id = organ_label_id
+        self.mask_value_organ = mask_value_organ
+        self.fast = fast
+
+    def extract_organ_mask(self, input_img, organ_label_id, mask_value):
+        # aorta = 52
+        """
+        Extracts a binary mask for a specific organ from a labeled NIFTI image.
+        
+        img_in: NIFTI image with segmentation labels.
+        organ_name: Name of the organ to extract.
+        label_map: Dictionary mapping label IDs to organ names.
+        
+        returns: Binary mask as a NIFTI image.
+        """
+        img_in = totalsegmentator(input=input_img, task='total',fast=self.fast)
+        data = img_in.get_fdata()
+
+        # Create a binary mask for the specified organ
+        organ_mask_data = np.zeros_like(data)
+        organ_mask_data[data == organ_label_id] = mask_value
+
+        # Create a new NIFTI image for the binary mask
+        organ_mask_img = nib.Nifti1Image(organ_mask_data, img_in.affine, img_in.header)
+        return organ_mask_img
+    
+    def __call__(self, data):
+        for key in self.keys:
+            x = data[key]
+            
+            mask = torch.zeros_like(x)
+            # [B, H, W, D]
+            # create a mask for each slice in the batch
+            for i in range(x.shape[0]):
+                if self.if_use_total_seg:
+                    mask_batch_i = self.extract_organ_mask(x[i,:,:,:], organ_label_id=self.organ_label_id, mask_value=self.mask_value_organ)
+                    mask[i,:,:,:] = mask_batch_i
+                for j in range(x.shape[-1]):
+                    mask_slice = create_body_mask(x[i,:,:,j], body_threshold=self.tissue_min)
+                    mask[i,:,:, j] = mask_slice
+            #mask = torch.zeros_like(x)
+            #mask[(x > self.tissue_min) & (x <= self.tissue_max)] = 1
+            mask[(x >= self.bone_min) & (x <= self.bone_max)] = self.mask_value_bones
+            data[key] = mask
+            #print("input and mask shape: ",x.shape,data[key].shape)
+        return data
+
+class CreateSegTransformd:
+    # create a mask by segmenting the input image using totalsegmentator
+    def __init__(self, keys, organ_label_id=52, mask_value=2, fast=True):
+        self.keys = keys
+        self.organ_label_id = organ_label_id
+        self.mask_value = mask_value
+        self.fast = fast
+
+    def extract_organ_mask(self, input_img, organ_label_id, mask_value):
+        # aorta = 52
+        """
+        Extracts a binary mask for a specific organ from a labeled NIFTI image.
+        
+        img_in: NIFTI image with segmentation labels.
+        organ_name: Name of the organ to extract.
+        label_map: Dictionary mapping label IDs to organ names.
+        
+        returns: Binary mask as a NIFTI image.
+        """
+        img_in = totalsegmentator(input=input_img, task='total',fast=self.fast)
+        data = img_in.get_fdata()
+
+        if organ_label_id>0:
+            # Create a binary mask for the specified organ
+            organ_mask_data = np.zeros_like(data)
+            organ_mask_data[data == organ_label_id] = mask_value
+        else:
+            organ_mask_data=data
+
+        # Create a new NIFTI image for the binary mask
+        organ_mask_img = nib.Nifti1Image(organ_mask_data, img_in.affine, img_in.header)
+        return organ_mask_img
+    
+    def __call__(self, data):
+        for key in self.keys:
+            x = data[key]
+            mask = torch.zeros_like(x)
+            # [B, H, W, D]
+            for i in range(x.shape[0]):
+                mask_batch_i = self.extract_organ_mask(x[i,:,:,:], organ_label_id=self.organ_label_id, mask_value=self.mask_value)
+                mask[i,:,:,:] = mask_batch_i
+            data[key] = mask
+        return data
+
+
+class CreateTotalSegTransformd:
+    # create a mask by segmenting the input image using totalsegmentator
+    def __init__(self, keys, fast=True):
+        self.keys = keys
+        self.fast = fast
+
+    def extract_organ_mask(self, input_img):
+        # aorta = 52
+        """
+        Extracts a binary mask for a specific organ from a labeled NIFTI image.
+        
+        img_in: NIFTI image with segmentation labels.
+        organ_name: Name of the organ to extract.
+        label_map: Dictionary mapping label IDs to organ names.
+        
+        returns: Binary mask as a NIFTI image.
+        """
+        #print(input_img.meta)
+        input_affine = input_img.meta['affine']
+
+        input_img = torch_tensor_to_nifti(input_img, affine=input_affine)
+        img_in = totalsegmentator(input=input_img, task='total', fast=self.fast)
+        data = img_in.get_fdata()
+        organ_mask_data=data
+        # Create a new NIFTI image for the binary mask
+        organ_mask_img = nib.Nifti1Image(organ_mask_data, img_in.affine, img_in.header)
+        return organ_mask_img
+    
+    def __call__(self, data):
+        for key in self.keys:
+            x = data[key]
+            mask = torch.zeros_like(x)
+            # [B, H, W, D]
+            for i in range(x.shape[0]):
+                mask_batch_i = self.extract_organ_mask(x[i,:,:,:])
+                numpy_data = mask_batch_i.get_fdata()
+    
+                # Convert the NumPy array to a PyTorch tensor
+                tensor_data = torch.from_numpy(numpy_data).float()
+                mask[i,:,:,:] = tensor_data
+            data[key] = mask
+        return data
+
+    def get_transforms(self, transform_list):
+        normalize=configs.dataset.normalize
+        pad=configs.dataset.pad
+        resized_size=configs.dataset.resized_size
+        WINDOW_WIDTH=configs.dataset.WINDOW_WIDTH
+        WINDOW_LEVEL=configs.dataset.WINDOW_LEVEL
+        prob=configs.dataset.augmentationProb
+        background=configs.dataset.background
+        indicator_A=configs.dataset.indicator_A
+        indicator_B=configs.dataset.indicator_B
+        load_masks=configs.dataset.load_masks
+        transform_list=[]
+        input_is_mask=configs.dataset.input_is_mask
+        # normally we input CT images and here we create masks for CT images
+        if not input_is_mask:
+            if not configs.dataset.use_all_masks:
+                transform_list.append(CreateMaskTransformd(keys=[indicator_A],
+                                                        tissue_min=configs.dataset.tissue_min,
+                                                        tissue_max=configs.dataset.tissue_max,
+                                                        bone_min=configs.dataset.bone_min,
+                                                        bone_max=configs.dataset.bone_max,
+                                                        mask_value_bones=2,
+                                                        ))
+            else:  # use all masks from the totalsegmentator
+                transform_list.append(CreateTotalSegTransformd(keys=[indicator_A],
+                                                        fast=True))
+        min, max=WINDOW_LEVEL-(WINDOW_WIDTH/2), WINDOW_LEVEL+(WINDOW_WIDTH/2)
+        #transform_list.append(ThresholdIntensityd(keys=[indicator_B], threshold=min, above=True, cval=background))
+        #transform_list.append(ThresholdIntensityd(keys=[indicator_B], threshold=max, above=False, cval=-1000))
+        # filter the source images
+        # transform_list.append(ThresholdIntensityd(keys=[indicator_A], threshold=configs.dataset.MRImax, above=False, cval=0))
+        if normalize=='zscore':
+            transform_list.append(NormalizeIntensityd(keys=[indicator_B], nonzero=False, channel_wise=True))
+            print('zscore normalization')
+        elif normalize=='minmax':
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=-1.0, maxv=1.0))
+            print('minmax normalization')
+
+        elif normalize=='scale1000_wrongbutworks':
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=0))
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], factor=-0.999)) 
+            print('scale1000 normalization')
+
+        elif normalize=='scale1000':
+            transform_list.append(ShiftIntensityd(keys=[indicator_B], offset=1024))
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None, factor=-0.999)) 
+            print('scale1000 normalization')
+        
+        elif normalize=='scale4000':
+            transform_list.append(ShiftIntensityd(keys=[indicator_B], offset=1024))
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None, factor=-0.99975))
+            print('scale4000 normalization')
+            
+        elif normalize=='scale10':
+            #transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=0))
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None,factor=-0.9)) 
+            print('scale10 normalization')
+
+        elif normalize=='inputonlyzscore':
+            transform_list.append(NormalizeIntensityd(keys=[indicator_A], nonzero=False, channel_wise=True))
+            print('only normalize input MRI images')
+
+        elif normalize=='inputonlyminmax':
+            transform_list.append(ScaleIntensityd(keys=[indicator_A], minv=configs.dataset.normmin, maxv=configs.dataset.normmax))
+            print('only normalize input MRI images')
+        
+        elif normalize=='none' or normalize=='nonorm':
+            print('no normalization')
+
+        spaceXY=self.configs.dataset.spaceXY
+        if spaceXY>0:
+            transform_list.append(Spacingd(keys=[indicator_A], pixdim=(spaceXY, spaceXY, 2.5), mode="bilinear", ensure_same_shape=True)) # 
+            transform_list.append(Spacingd(keys=[indicator_B, "mask"] if load_masks else [indicator_B], 
+                                           pixdim=(spaceXY, spaceXY , 2.5), mode="bilinear", ensure_same_shape=True))
+        
+        transform_list.append(Zoomd(keys=[indicator_A, indicator_B,"mask"] if load_masks 
+                                                   else [indicator_A, indicator_B], 
+                                                  zoom=configs.dataset.zoom, keep_size=False, mode='area',padding_mode='minimum'))
+
+
+        transform_list.append(DivisiblePadd(keys=[indicator_A, indicator_B,"mask"] if load_masks else [indicator_A, indicator_B],
+                                            k=self.configs.dataset.div_size, mode="minimum"))
+        transform_list.append(ResizeWithPadOrCropd(keys=[indicator_A, indicator_B,"mask"] if load_masks else [indicator_A, indicator_B], 
+                                                  spatial_size=resized_size,mode=pad))
+
+        if configs.dataset.rotate:
+            transform_list.append(Rotate90d(keys=[indicator_A, indicator_B, "mask"] if load_masks else [indicator_A, indicator_B], k=3))
+
+        if mode == 'train':
+            from monai.transforms import (
+                # data augmentation
+                RandRotated,
+                RandZoomd,
+                RandBiasFieldd,
+                RandAffined,
+                RandGridDistortiond,
+                RandGridPatchd,
+                RandShiftIntensityd,
+                RandGibbsNoised,
+                RandAdjustContrastd,
+                RandGaussianSmoothd,
+                RandGaussianSharpend,
+                RandGaussianNoised,
+            )
+            shapeAug=configs.dataset.shapeAug
+            if shapeAug:
+                #transform_list.append(RandRotated(keys=[indicator_A, indicator_B, "mask"] if load_masks else [indicator_A, indicator_B],
+                #                                  range_x = 0.0, range_y = 1.0, range_z = 1.0, 
+                #                                  prob=prob, padding_mode="border", keep_size=False))
+                transform_list.append(RandZoomd(keys=[indicator_A, indicator_B, "mask"] if load_masks else [indicator_A, indicator_B], 
+                                                prob=prob, min_zoom=self.configs.dataset.rand_min_zoom, max_zoom=self.configs.dataset.rand_max_zoom,
+                                                padding_mode= "minimum" ,keep_size=False))
+                #transform_list.append(RandAffined(keys=[indicator_A, indicator_B], padding_mode="border" , prob=prob))
+                #transform_list.append(Rand3DElasticd(keys=[indicator_A, indicator_B], prob=prob, sigma_range=(5, 8), magnitude_range=(100, 200), spatial_size=None, mode='bilinear'))
+            intensityAug=configs.dataset.intensityAug
+            if intensityAug:
+                print('intensity data augmentation is used')
+                transform_list.append(RandBiasFieldd(keys=[indicator_A], degree=3, coeff_range=(0.0, 0.1), prob=prob)) # only apply to MRI images
+                transform_list.append(RandGaussianNoised(keys=[indicator_A], prob=prob, mean=0.0, std=0.01))
+                transform_list.append(RandAdjustContrastd(keys=[indicator_A], prob=prob, gamma=(0.5, 1.5)))
+                transform_list.append(RandShiftIntensityd(keys=[indicator_A], prob=prob, offsets=20))
+                transform_list.append(RandGaussianSharpend(keys=[indicator_A], alpha=(0.2, 0.8), prob=prob))
+            
+        #transform_list.append(Rotate90d(keys=[indicator_A, indicator_B], k=3))
+        #transform_list.append(DivisiblePadd(keys=[indicator_A, indicator_B], k=div_size, mode="minimum"))
+        #transform_list.append(Identityd(keys=[indicator_A, indicator_B]))  # do nothing for the no norm case
+        train_transforms = Compose(transform_list)
+        return train_transforms

dataprocesser/archive/csv_dataset.py

@@ -0,0 +1,121 @@
+def get_data_scaler(config):
+  """Data normalizer. Assume data are always in [0, 1]."""
+  if config.data.centered:
+    # Rescale to [-1, 1]
+    return lambda x: x * 2. - 1.
+  else:
+    return lambda x: x
+
+
+def get_data_inverse_scaler(config):
+  """Inverse data normalizer."""
+  if config.data.centered:
+    # Rescale [-1, 1] to [0, 1]
+    return lambda x: (x + 1.) / 2.
+  else:
+    return lambda x: x
+
+IMG_EXTENSIONS = [
+    #'.jpg', '.JPG', '.jpeg', '.JPEG',
+    #'.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP', 
+    '.nrrd', '.nii.gz'
+]
+
+
+def is_image_file(filename):
+    return any(filename.endswith(extension) for extension in IMG_EXTENSIONS)
+
+def volume_slicer(volume_tensor, transform, all_slices=None):
+    # Convert numpy array to PyTorch tensor
+    # Note: You might need to add channel dimension or perform other adjustments
+    volume_tensor = volume_tensor.permute(2, 1, 0) # [H, W, D] -> [D, H, W]
+    volume_tensor = volume_tensor.unsqueeze(1)  # Add channel dimension [D, H, W] -> [D, 1, H, W]
+    if transform is not None:
+        volume_tensor = transform(volume_tensor)
+
+    #print('stacking volume tensor:',volume_tensor.shape)
+    if all_slices is None:
+        all_slices = volume_tensor
+    else:
+        all_slices = torch.cat((all_slices, volume_tensor), 0)
+    return all_slices
+
+class csvDataset_3D(Dataset):
+    def __init__(self, csv_file, transform=None, load_patient_number=1):
+        """
+        Args:
+            csv_file (string): Path to the csv file with annotations.
+            transform (callable, optional): Optional transform to be applied on a sample.
+        """
+        self.data_frame = pd.read_csv(csv_file)
+        # control the length of the dataset
+        self.data_frame = self.data_frame[:load_patient_number]
+        self.transform = transform
+        
+    def __len__(self):
+        return len(self.data_frame)
+
+    def __getitem__(self, idx):
+        if torch.is_tensor(idx):
+            idx = idx.tolist()
+
+        img_path = self.data_frame.iloc[idx, -1]
+        image = nib.load(img_path).get_fdata()
+        image = torch.tensor(image, dtype=torch.float32)
+        
+        # Example: Using the 'Aorta_diss' column as a label
+        label = self.data_frame.iloc[idx, -3]
+        #label = torch.tensor(label, dtype=torch.float32)
+        
+        # If more processing is needed (e.g., normalization, adding channel dimension), do it here
+        image = image.unsqueeze(0)  # Add channel dimension if it's a single channel image
+        
+        sample = {'image': image, 'label': label}
+        
+        return sample
+    
+class csvDataset_2D(Dataset):
+    def __init__(self, csv_file, transform=None, load_patient_number=1):
+        self.csv_file = csv_file
+        self.transform = transform
+        self.load_patient_number = load_patient_number
+        self.data_frame = pd.read_csv(csv_file)
+        if len(self.data_frame) == 0:
+            raise RuntimeError(f"Found 0 images in: {csv_file}")
+        
+        # Initialize dataset
+        self.initialize_dataset()
+
+    def initialize_dataset(self):
+        print('Loading dataset...')
+        self.data_frame = self.data_frame[:self.load_patient_number]
+        all_slices = None
+        all_labels = []
+        
+        for idx in tqdm(range(len(self.data_frame))):
+            img_path = self.data_frame.iloc[idx, -1]
+            volume = nib.load(img_path)
+            volume_data = volume.get_fdata()  # Load as [H, W, D]
+            volume_tensor = torch.tensor(volume_data, dtype=torch.float32)
+            all_slices = volume_slicer(volume_tensor, self.transform, all_slices)  # -> [D, 1, H, W] and pile up all the slices
+            label = self.data_frame.iloc[idx, -3]
+            all_labels = all_labels + [label] * volume_tensor.shape[0]
+        
+        print('All stacked slices:', all_slices.shape)
+        self.all_slices = all_slices
+        self.all_labels = all_labels
+
+    def __len__(self):
+        return self.all_slices.shape[0]
+
+    def __getitem__(self, idx):
+        if torch.is_tensor(idx):
+            idx = idx.tolist()
+        image = self.all_slices[idx]
+        label = self.all_labels[idx]
+        sample = {'source': image, 'target': label}
+        return sample
+
+    def reset(self):
+        print('Resetting dataset...')
+        self.initialize_dataset()

dataprocesser/archive/csv_dataset_slices.py

@@ -0,0 +1,20 @@
+from torch.utils.data import DataLoader
+import os.path
+from PIL import Image
+import torch
+from PIL import ImageFile
+import os
+import pandas as pd
+import monai
+import json
+Image.MAX_IMAGE_PIXELS = None  # Disable DecompressionBombError
+ImageFile.LOAD_TRUNCATED_IMAGES = True  # Disable OSError: image file is truncated
+
+
+
+
+from dataprocesser.list_dataset_base import BaseDataLoader
+
+            
+
+

dataprocesser/archive/csv_dataset_slices_assigned.py

@@ -0,0 +1,11 @@
+from dataprocesser.csv_dataset_slices import csv_slices_DataLoader
+from dataprocesser.customized_transforms import MaskHUAssigmentd
+
+from monai.transforms import (
+    ScaleIntensityd,
+    ThresholdIntensityd,
+    NormalizeIntensityd,
+    ShiftIntensityd,
+)
+
+

dataprocesser/archive/data_create_seg.py

@@ -0,0 +1,28 @@
+from step1_init_data_list import (
+    list_img_ad_from_anish_csv, 
+    list_img_pID_from_synthrad_folder,
+    )
+def run():
+    number = 1
+    dataset='anish'
+    if dataset=='anish':
+        data_dir = 'D:\Projects\SynthRad\synthrad_conversion\healthy_dissec_home.csv'
+        target_file_list, _ =list_img_ad_from_anish_csv(data_dir) # a csv_file
+    elif dataset=='synthrad':
+        data_dir = 'D:\Projects\data\synthrad\train\Task1\pelvis'
+        target_file_list, _=list_img_pID_from_synthrad_folder(data_dir, accepted_modalities='ct', saved_name="target_filenames.txt")
+    create_segmentation(target_file_list[0: number])
+
+def create_segmentation(dataset_list):
+    import nibabel as nib
+    try:  
+        from totalsegmentator.python_api import totalsegmentator
+        for sample in dataset_list:
+            input_path=sample
+            print(f'create segmentation mask for {input_path}')
+            output_path=input_path.replace('.nii','_seg.nii')
+            input_img = nib.load(input_path)
+            totalsegmentator(input=input_img, output=output_path, task='total', fast=False, ml=True)
+            print(f'segmentation mask is saved as {output_path}')
+    except:
+        print("An exception occurred")

dataprocesser/archive/data_slicing.py

@@ -0,0 +1,13 @@
+from dataprocesser.step1_init_data_list import init_dataset
+import os
+
+loader, opt, my_paths = init_dataset()
+path=r'E:\Projects\yang_proj\data\seg2med\seg2med_nifti_2d_343'
+train_path=os.path.join(path, 'train')
+val_path=os.path.join(path,'val')
+os.makedirs(path,exist_ok=True)
+os.makedirs(train_path,exist_ok=True)
+os.makedirs(val_path,exist_ok=True)
+
+loader.save_slices_nifti_and_csv(train_path, loader.train_volume_ds)
+loader.save_slices_nifti_and_csv(val_path, loader.val_volume_ds)

dataprocesser/archive/dataset_med.py

@@ -0,0 +1,188 @@
+from torch.utils.data import DataLoader, Dataset
+import torch.utils.data as data
+import os.path
+import random
+from torchvision import transforms
+from PIL import Image
+import torch
+from PIL import ImageFile
+from utils.MattingLaplacian import compute_laplacian
+
+import nibabel as nib
+import numpy as np
+
+Image.MAX_IMAGE_PIXELS = None  # Disable DecompressionBombError
+ImageFile.LOAD_TRUNCATED_IMAGES = True  # Disable OSError: image file is truncated
+
+
+
+IMG_EXTENSIONS = [
+    #'.jpg', '.JPG', '.jpeg', '.JPEG',
+    #'.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP', 
+    '.nrrd', '.nii.gz'
+]
+
+
+def is_image_file(filename):
+    return any(filename.endswith(extension) for extension in IMG_EXTENSIONS)
+
+def make_dataset_modality(dir, modality='ct'):
+    images = []
+    assert os.path.isdir(dir), '%s is not a valid directory' % dir
+
+    # for image data in the following structure:
+    # root/
+    #     patient_folder/
+    #         ct_image.nii.gz
+    #         mr_image.nii.gz
+    #         ...
+    #     patient_folder2/
+    #         ct_image.nii.gz
+    #         mr_image.nii.gz
+    #         ...
+    for patient_folder, _, fnames in sorted(os.walk(dir)): # means that it will go through all the files in the directory
+        #print(patient_folder)
+        if patient_folder != dir:
+            #print('patient folder:',patient_folder)
+            for root2, _, fnames2 in sorted(os.walk(patient_folder)):
+                #print('files:',fnames2)
+                for fname2 in fnames2:
+                    if is_image_file(fname2) and modality in fname2:
+                        #print('passed file:',fname2)
+                        path = os.path.join(root2, fname2)
+                        images.append(path)
+    return images
+
+
+class CTImageDataset(Dataset):
+    def __init__(self, root, modality='ct', transform=None, 
+                 load_patient_number=1,
+                 use_lap=True, win_rad=1):
+        self.imgs_paths = sorted(make_dataset_modality(root, modality))
+        self.transform = transform
+        self.to_tensor = transforms.ToTensor()  # Might need adjustment for 3D
+
+        if len(self.imgs_paths) == 0:
+            raise RuntimeError(f"Found 0 images in: {root}")
+        # form the images to be in the form of [D, H, W]
+        all_slices = None
+        for img_path in self.imgs_paths[:load_patient_number]:
+            volume = nib.load(img_path)
+            volume_data = volume.get_fdata() # load as [H, W, D]
+            # 
+            # Convert numpy array to PyTorch tensor
+            # Note: You might need to add channel dimension or perform other adjustments
+            volume_tensor = torch.tensor(volume_data, dtype=torch.float32)
+            volume_tensor = volume_tensor.permute(2, 1, 0) # [N, H, W]
+            volume_tensor = volume_tensor.unsqueeze(3)  # Add channel dimension [N, H, W] -> [N, H, W, 1]
+            # pasting grayscale information to all three channels.
+            volume_tensor = volume_tensor.repeat(1, 1, 1, 3)
+            #print('Debug, volume tensor:',volume_tensor.shape)
+            if self.transform is not None:
+                volume_tensor = self.transform(volume_tensor)
+            if all_slices is None:
+                all_slices = volume_tensor
+            else:
+                all_slices = torch.cat((all_slices, volume_tensor), 0)
+        print(f'slices of {modality} dataset:',all_slices.shape)
+        self.all_slices = all_slices
+        self.use_lap = use_lap
+        self.win_rad = win_rad
+
+    def __getitem__(self, index):
+        img = self.all_slices[index]
+        #print('Debug 1, img shape:',img.shape)
+        if self.use_lap:    
+            laplacian_m = compute_laplacian(img, win_rad=self.win_rad)
+        else:
+            laplacian_m = None
+        #print('Debug 2, laplacian_m:',laplacian_m.shape)
+        # permute img from [H, W, C] to [C, H, W]
+        img = img.permute(2, 0, 1)
+        return {'img': img, 'laplacian_m': laplacian_m}
+
+    def __len__(self):
+        return self.all_slices.shape[0]
+
+from monai.transforms import (
+    ResizeWithPadOrCrop,
+    ScaleIntensity,
+    Compose,
+)
+
+def get_data_loader_folder(input_folder, modality, 
+                           batch_size, new_size=288, 
+                           height=256, width=256, 
+                           num_workers=None, load_patient_number=1):
+    transform_list = []
+    transform_list = [ResizeWithPadOrCrop(spatial_size=[height,width, -1],mode="minimum")] + transform_list
+    transform_list = [ScaleIntensity(minv=0, maxv=1.0)]+ transform_list
+    #transform_list = [ScaleIntensity(factor=-0.9)]+ transform_list
+    #transform_list = [transforms.Resize(new_size)] + transform_list
+    transform = Compose(transform_list)
+
+    dataset = CTImageDataset(input_folder, modality=modality, transform=transform, load_patient_number=load_patient_number)
+    
+    if num_workers is None:
+        num_workers = 0
+    loader = DataLoader(dataset=dataset, 
+                        batch_size=batch_size, 
+                        drop_last=True, 
+                        num_workers=num_workers, 
+                        sampler=InfiniteSamplerWrapper(dataset), 
+                        collate_fn=collate_fn
+                        )
+    return loader
+
+def main(root = r'C:\Users\56991\Projects\Datasets\Task1\pelvis',modality='ct'):
+    # Example usage
+    
+    batch_size = 8
+    new_size = 512
+    height = 512
+    width = 512
+    num_workers = None
+    load_patient_number = 1
+    loader = get_data_loader_folder(root,modality, batch_size, new_size, height, width, num_workers, load_patient_number)
+    #print length of loader
+    print('Length of loader:',len(loader))
+    for i, batch in enumerate(loader):
+        print(f'Batch {i}:',batch['img'].shape)
+    print('Done')
+
+if __name__=='__main__':
+    main()
+    
+
+def InfiniteSampler(n):
+    # i = 0
+    i = n - 1
+    order = np.random.permutation(n)
+    while True:
+        yield order[i]
+        i += 1
+        if i >= n:
+            np.random.seed()
+            order = np.random.permutation(n)
+            i = 0
+
+
+class InfiniteSamplerWrapper(data.sampler.Sampler):
+    def __init__(self, data_source):
+        self.num_samples = len(data_source)
+
+    def __iter__(self):
+        return iter(InfiniteSampler(self.num_samples))
+
+    def __len__(self):
+        return 2 ** 31
+
+
+def collate_fn(batch):
+    img = [b['img'] for b in batch]
+    img = torch.stack(img, dim=0)
+
+    laplacian_m = [b['laplacian_m'] for b in batch]
+
+    return {'img': img, 'laplacian_m': laplacian_m}
+

dataprocesser/archive/gan_loader.py

@@ -0,0 +1,310 @@
+import monai
+import os
+import numpy as np
+from monai.transforms import (
+    Compose,
+    LoadImaged,
+    EnsureChannelFirstd,
+    SqueezeDimd,
+    CenterSpatialCropd,
+    Rotate90d,
+    ScaleIntensityd,
+    ResizeWithPadOrCropd,
+    DivisiblePadd,
+    ThresholdIntensityd,
+    NormalizeIntensityd,
+    ShiftIntensityd,
+    Identityd,
+    ScaleIntensityRanged,
+    Spacingd,
+)
+
+from monai.data import Dataset
+from torch.utils.data import DataLoader
+import torch
+from .checkdata import check_volumes, save_volumes, check_batch_data, test_volumes_pixdim
+
+def get_transforms(configs, mode='train'):
+    normalize=configs.dataset.normalize
+    pad=configs.dataset.pad
+    resized_size=configs.dataset.resized_size
+    WINDOW_WIDTH=configs.dataset.WINDOW_WIDTH
+    WINDOW_LEVEL=configs.dataset.WINDOW_LEVEL
+    prob=configs.dataset.augmentationProb
+    background=configs.dataset.background
+
+    transform_list=[]
+    min, max=WINDOW_LEVEL-(WINDOW_WIDTH/2), WINDOW_LEVEL+(WINDOW_WIDTH/2)
+    #transform_list.append(ThresholdIntensityd(keys=["target"], threshold=min, above=True, cval=background))
+    #transform_list.append(ThresholdIntensityd(keys=["target"], threshold=max, above=False, cval=-1000))
+    # filter the source images
+    # transform_list.append(ThresholdIntensityd(keys=["source"], threshold=configs.dataset.MRImax, above=False, cval=0))
+    if normalize=='zscore':
+        transform_list.append(NormalizeIntensityd(keys=["source", "target"], nonzero=False, channel_wise=True))
+        print('zscore normalization')
+    elif normalize=='minmax':
+        transform_list.append(ScaleIntensityd(keys=["source", "target"], minv=-1.0, maxv=1.0))
+        print('minmax normalization')
+
+    elif normalize=='scale4000':
+        transform_list.append(ScaleIntensityd(keys=["source"], minv=0, maxv=1))
+        transform_list.append(ScaleIntensityd(keys=["target"], minv=0))
+        transform_list.append(ScaleIntensityd(keys=["target"], factor=-0.99975)) # x=x(1+factor)
+        print('scale1000 normalization')
+
+    elif normalize=='scale1000':
+        transform_list.append(ScaleIntensityd(keys=["source"], minv=0, maxv=1))
+        transform_list.append(ScaleIntensityd(keys=["target"], minv=0))
+        transform_list.append(ScaleIntensityd(keys=["target"], factor=-0.999)) 
+        print('scale1000 normalization')
+
+    elif normalize=='inputonlyzscore':
+        transform_list.append(NormalizeIntensityd(keys=["source"], nonzero=False, channel_wise=True))
+        print('only normalize input MRI images')
+
+    elif normalize=='inputonlyminmax':
+        transform_list.append(ScaleIntensityd(keys=["source"], minv=configs.dataset.normmin, maxv=configs.dataset.normmax))
+        print('only normalize input MRI images')
+    
+    elif normalize=='none' or normalize=='nonorm':
+        print('no normalization')
+
+    spaceXY=0
+    if spaceXY>0:
+        transform_list.append(Spacingd(keys=["source"], pixdim=(spaceXY, spaceXY, 2.5), mode="bilinear")) # 
+        transform_list.append(Spacingd(keys=["target", "mask"], pixdim=(spaceXY, spaceXY , 2.5), mode="bilinear")) #
+    transform_list.append(ResizeWithPadOrCropd(keys=["source", "target"], spatial_size=resized_size,mode=pad))
+    # transform_list.append(ScaleIntensityRanged(keys=["target"],a_min=WINDOW_LEVEL-(WINDOW_WIDTH/2), a_max=WINDOW_LEVEL+(WINDOW_WIDTH/2),b_min=0, b_max=1, clip=True))
+    
+    if configs.dataset.rotate:
+        transform_list.append(Rotate90d(keys=["source",  "target"], k=3))
+
+    if mode == 'train':
+        from monai.transforms import (
+            # data augmentation
+            RandRotated,
+            RandZoomd,
+            RandBiasFieldd,
+            RandAffined,
+            RandGridDistortiond,
+            RandGridPatchd,
+            RandShiftIntensityd,
+            RandGibbsNoised,
+            RandAdjustContrastd,
+            RandGaussianSmoothd,
+            RandGaussianSharpend,
+            RandGaussianNoised,
+        )
+        Aug=True
+        if Aug:
+            transform_list.append(RandRotated(keys=["source", "target", "mask"], range_x = 0.1, range_y = 0.1, range_z = 0.1, prob=prob, padding_mode="border", keep_size=True))
+            transform_list.append(RandZoomd(keys=["source", "target", "mask"], prob=prob, min_zoom=0.9, max_zoom=1.3,padding_mode= "minimum" ,keep_size=True))
+            transform_list.append(RandAffined(keys=["source", "target"],padding_mode="border" , prob=prob))
+            #transform_list.append(Rand3DElasticd(keys=["source", "target"], prob=prob, sigma_range=(5, 8), magnitude_range=(100, 200), spatial_size=None, mode='bilinear'))
+        intensityAug=False
+        if intensityAug:
+            print('intensity data augmentation is used')
+            transform_list.append(RandBiasFieldd(keys=["source"], degree=3, coeff_range=(0.0, 0.1), prob=prob)) # only apply to MRI images
+            transform_list.append(RandGaussianNoised(keys=["source"], prob=prob, mean=0.0, std=0.01))
+            transform_list.append(RandAdjustContrastd(keys=["source"], prob=prob, gamma=(0.5, 1.5)))
+            transform_list.append(RandShiftIntensityd(keys=["source"], prob=prob, offsets=20))
+            transform_list.append(RandGaussianSharpend(keys=["source"], alpha=(0.2, 0.8), prob=prob))
+        
+    #transform_list.append(Rotate90d(keys=["source", "target"], k=3))
+    #transform_list.append(DivisiblePadd(keys=["source", "target"], k=div_size, mode="minimum"))
+    #transform_list.append(Identityd(keys=["source", "target"]))  # do nothing for the no norm case
+    train_transforms = Compose(transform_list)
+    return train_transforms
+def myslicesloader(configs,paths):
+    data_path=configs.dataset.data_dir
+    train_number=configs.dataset.train_number
+    val_number=configs.dataset.val_number
+    train_batch_size=configs.dataset.batch_size
+    val_batch_size=configs.dataset.val_batch_size
+    saved_name_train=paths["saved_name_train"]
+    saved_name_val=paths["saved_name_val"]
+    center_crop=configs.dataset.center_crop
+    source=configs.dataset.source
+    target=configs.dataset.target
+   
+    # volume-level transforms for both image and label
+    train_transforms = get_transforms(configs,mode='train')
+    val_transforms = get_transforms(configs,mode='val')
+
+    #list all files in the folder
+    file_list=[i for i in os.listdir(data_path) if 'overview' not in i]
+    file_list_path=[os.path.join(data_path,i) for i in file_list]
+    #list all ct and mr files in folder
+    mask='mask'
+    source_file_list=[os.path.join(j,f'{source}.nii.gz') for j in file_list_path]
+    target_file_list=[os.path.join(j,f'{target}.nii.gz') for j in file_list_path]
+    mask_file_list=[os.path.join(j,f'{mask}.nii.gz') for j in file_list_path]
+    train_ds = [{'source': i, 'target': j, 'mask': k, 'A_paths': i, 'B_paths': j, 'mask_path': k} 
+                for i, j, k in zip(source_file_list[0:train_number], target_file_list[0:train_number], mask_file_list[0:train_number])]
+    val_ds = [{'source': i, 'target': j, 'mask': k, 'A_paths': i, 'B_paths': j, 'mask_path': k} 
+              for i, j, k in zip(source_file_list[-val_number:], target_file_list[-val_number:], mask_file_list[-val_number:])]
+    print('all files in dataset:',len(file_list))
+
+    # load volumes and center crop
+    if center_crop>0:
+        crop=Compose([LoadImaged(keys=["source", "target", "mask"]),
+                    EnsureChannelFirstd(keys=["source", "target", "mask"]),
+                    CenterSpatialCropd(keys=["source", "target", "mask"], roi_size=(-1,-1,center_crop)),
+                    
+                    ])
+        train_crop_ds = monai.data.Dataset(data=train_ds, transform=crop)
+        val_crop_ds = monai.data.Dataset(data=val_ds, transform=crop)
+        print('center crop:',center_crop)
+    else:
+        crop=Compose([LoadImaged(keys=["source", "target", "mask"]),
+            EnsureChannelFirstd(keys=["source", "target", "mask"]),
+            ])
+        train_crop_ds = monai.data.Dataset(data=train_ds, transform=crop)
+        val_crop_ds = monai.data.Dataset(data=val_ds, transform=crop)
+
+    # load volumes
+    train_volume_ds = monai.data.Dataset(data=train_crop_ds, transform=train_transforms) 
+    val_volume_ds = monai.data.Dataset(data=val_crop_ds, transform=val_transforms)
+    ifsave,ifcheck,iftest=False,False,False
+    if ifsave:
+        save_volumes(train_ds, val_ds, saved_name_train, saved_name_val)
+    if ifcheck:
+        check_volumes(train_ds, train_volume_ds, val_volume_ds, val_ds)
+    if iftest:
+        test_volumes_pixdim(train_volume_ds)
+
+    # batch-level slicer for both image and label
+    window_width=1
+    patch_func = monai.data.PatchIterd(
+        keys=["source", "target", "mask"],
+        patch_size=(None, None, window_width),  # dynamic first two dimensions
+        start_pos=(0, 0, 0)
+    )
+    if window_width==1:
+        patch_transform = Compose(
+            [
+                SqueezeDimd(keys=["source", "target", "mask"], dim=-1),  # squeeze the last dim
+            ]
+        )
+    else:
+        patch_transform = None
+        
+    # for training
+    train_patch_ds = monai.data.GridPatchDataset(
+        data=train_volume_ds, patch_iter=patch_func, transform=patch_transform, with_coordinates=False)
+    train_loader = DataLoader(
+        train_patch_ds,
+        batch_size=train_batch_size,
+        num_workers=2,
+        pin_memory=torch.cuda.is_available(),
+    )
+
+    # for validation
+    val_loader = DataLoader(
+        val_volume_ds, 
+        num_workers=1, 
+        batch_size=val_batch_size,
+        pin_memory=torch.cuda.is_available())
+    
+    if ifcheck:
+        check_batch_data(train_loader,val_loader,train_patch_ds,val_volume_ds,train_batch_size,val_batch_size)
+    
+    return train_crop_ds,val_crop_ds,train_loader,val_loader,train_transforms,val_transforms
+
+def ddpmloader(configs,paths):
+    data_path=configs.dataset.data_dir
+    train_number=configs.dataset.train_number
+    val_number=configs.dataset.val_number
+    train_batch_size=configs.dataset.batch_size
+    val_batch_size=configs.dataset.val_batch_size
+    saved_name_train=paths["saved_name_train"]
+    saved_name_val=paths["saved_name_val"]
+    center_crop=configs.dataset.center_crop
+    source=configs.dataset.source
+    target=configs.dataset.target
+   
+    # volume-level transforms for both image and label
+    train_transforms = get_transforms(configs,mode='train')
+    val_transforms = get_transforms(configs,mode='val')
+
+    #list all files in the folder
+    file_list=[i for i in os.listdir(data_path) if 'overview' not in i]
+    file_list_path=[os.path.join(data_path,i) for i in file_list]
+    #list all ct and mr files in folder
+    mask='mask'
+    source_file_list=[os.path.join(j,f'{source}.nii.gz') for j in file_list_path]
+    target_file_list=[os.path.join(j,f'{target}.nii.gz') for j in file_list_path]
+    mask_file_list=[os.path.join(j,f'{mask}.nii.gz') for j in file_list_path]
+    train_ds = [{'source': i, 'target': j, 'mask': k, 'A_paths': i, 'B_paths': j, 'mask_path': k} 
+                for i, j, k in zip(source_file_list[0:train_number], target_file_list[0:train_number], mask_file_list[0:train_number])]
+    val_ds = [{'source': i, 'target': j, 'mask': k, 'A_paths': i, 'B_paths': j, 'mask_path': k} 
+              for i, j, k in zip(source_file_list[-val_number:], target_file_list[-val_number:], mask_file_list[-val_number:])]
+    print('all files in dataset:',len(file_list))
+
+    # load volumes and center crop
+    if center_crop>0:
+        crop=Compose([LoadImaged(keys=["source", "target"]),
+                    EnsureChannelFirstd(keys=["source", "target"]),
+                    CenterSpatialCropd(keys=["source", "target"], roi_size=(-1,-1,center_crop)),
+                    
+                    ])
+        train_crop_ds = monai.data.Dataset(data=train_ds, transform=crop)
+        val_crop_ds = monai.data.Dataset(data=val_ds, transform=crop)
+        print('center crop:',center_crop)
+    else:
+        crop=Compose([LoadImaged(keys=["source", "target"]),
+            EnsureChannelFirstd(keys=["source", "target"]),
+            ])
+        train_crop_ds = monai.data.Dataset(data=train_ds, transform=crop)
+        val_crop_ds = monai.data.Dataset(data=val_ds, transform=crop)
+
+    # load volumes
+    train_volume_ds = monai.data.Dataset(data=train_crop_ds, transform=train_transforms) 
+    val_volume_ds = monai.data.Dataset(data=val_crop_ds, transform=val_transforms)
+    ifsave,ifcheck,iftest=False,False,False
+    if ifsave:
+        save_volumes(train_ds, val_ds, saved_name_train, saved_name_val)
+    if ifcheck:
+        check_volumes(train_ds, train_volume_ds, val_volume_ds, val_ds)
+    if iftest:
+        test_volumes_pixdim(train_volume_ds)
+
+    # batch-level slicer for both image and label
+    window_width=1
+    patch_func = monai.data.PatchIterd(
+        keys=["source", "target"],
+        patch_size=(None, None, window_width),  # dynamic first two dimensions
+        start_pos=(0, 0, 0)
+    )
+    if window_width==1:
+        patch_transform = Compose(
+            [
+                SqueezeDimd(keys=["source", "target"], dim=-1),  # squeeze the last dim
+            ]
+        )
+    else:
+        patch_transform = None
+        
+    # for training
+    train_patch_ds = monai.data.GridPatchDataset(
+        data=train_volume_ds, patch_iter=patch_func, transform=patch_transform, with_coordinates=False)
+    train_loader = DataLoader(
+        train_patch_ds,
+        batch_size=train_batch_size,
+        num_workers=0,
+        pin_memory=torch.cuda.is_available(),
+    )
+
+    # for validation
+    val_patch_ds = monai.data.GridPatchDataset(
+        data=val_volume_ds, patch_iter=patch_func, transform=patch_transform, with_coordinates=False)
+    val_loader = DataLoader(
+        val_patch_ds, #val_volume_ds, 
+        num_workers=0, 
+        batch_size=val_batch_size,
+        pin_memory=torch.cuda.is_available())
+    
+    if ifcheck:
+        check_batch_data(train_loader,val_loader,train_patch_ds,val_volume_ds,train_batch_size,val_batch_size)
+    
+    return train_crop_ds,val_crop_ds,train_loader,val_loader,train_transforms,val_transforms

dataprocesser/archive/json_dataset_slices.py

@@ -0,0 +1,28 @@
+from torch.utils.data import DataLoader
+import os.path
+from PIL import Image
+import torch
+from PIL import ImageFile
+import os
+import pandas as pd
+import monai
+
+Image.MAX_IMAGE_PIXELS = None  # Disable DecompressionBombError
+ImageFile.LOAD_TRUNCATED_IMAGES = True  # Disable OSError: image file is truncated
+
+IMG_EXTENSIONS = [
+    #'.jpg', '.JPG', '.jpeg', '.JPEG',
+    #'.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP', 
+    '.nrrd', '.nii.gz'
+]
+
+def is_image_file(filename):
+    return any(filename.endswith(extension) for extension in IMG_EXTENSIONS)
+
+
+from dataprocesser.list_dataset_base import BaseDataLoader
+
+                
+
+
+

dataprocesser/archive/list_dataset_Anika.py

@@ -0,0 +1,10 @@
+
+
+
+
+
+
+
+
+
+

dataprocesser/archive/list_dataset_Anish_seg.py

@@ -0,0 +1,42 @@
+from torch.utils.data import DataLoader, Dataset, random_split
+import torch.utils.data as data
+import os.path
+import random
+from torchvision import transforms
+from PIL import Image
+import torch
+from PIL import ImageFile
+#from utils.MattingLaplacian import compute_laplacian
+
+import nibabel as nib
+import numpy as np
+import os
+import csv
+import pandas as pd
+#from transformers import CLIPTokenizer
+from tqdm import tqdm
+Image.MAX_IMAGE_PIXELS = None  # Disable DecompressionBombError
+ImageFile.LOAD_TRUNCATED_IMAGES = True  # Disable OSError: image file is truncated
+
+import os
+import numpy as np
+
+import torch
+
+IMG_EXTENSIONS = [
+    #'.jpg', '.JPG', '.jpeg', '.JPEG',
+    #'.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP', 
+    '.nrrd', '.nii.gz'
+]
+
+def is_image_file(filename):
+    return any(filename.endswith(extension) for extension in IMG_EXTENSIONS)
+
+
+
+
+from dataprocesser.customized_transforms import CreateMaskTransformd, MergeMasksTransformd
+from dataprocesser.list_dataset_base import BaseDataLoader
+
+
+

dataprocesser/archive/list_dataset_base.py

@@ -0,0 +1,983 @@
+import monai
+import os
+import numpy as np
+from monai.transforms import (
+    Compose,
+    LoadImaged,
+    EnsureChannelFirstd,
+    SqueezeDimd,
+    CenterSpatialCropd,
+    Rotate90d,
+    ScaleIntensityd,
+    ResizeWithPadOrCropd,
+    DivisiblePadd,
+    Zoomd,
+    ThresholdIntensityd,
+    NormalizeIntensityd,
+    ShiftIntensityd,
+    Identityd,
+    ScaleIntensityRanged,
+    Spacingd,
+)
+from torch.utils.data import DataLoader
+from torch.utils.data import ConcatDataset
+import torch
+from abc import ABC, abstractmethod
+
+from datetime import datetime
+import json
+from tqdm import tqdm
+
+from step1_init_data_list import (
+    list_img_ad_from_anish_csv, 
+    list_img_ad_pIDs_from_anish_csv,
+    list_img_pID_from_synthrad_folder,
+    list_from_anika_dataset,
+    list_from_json,
+    list_from_slice_csv,
+    )
+from step5_data_check_and_log import finalcheck
+VERBOSE = False
+
+def make_dataset_modality():
+    images = []
+    return images
+
+class ABCLoader(ABC):
+    @abstractmethod
+    def __init__(self):
+        """Subclass must implement this method."""
+        pass
+
+    def get_loader(self):
+        """Subclass must implement this method."""
+        pass
+
+    def create_dataset(self):
+        """Subclass must implement this method."""
+        pass
+
+    def get_transforms(self):
+        """Subclass must implement this method."""
+        pass
+
+    def get_normlization(self):
+        """Subclass must implement this method."""
+        pass
+
+    def get_shape_transform(self):
+        """Subclass must implement this method."""
+        print("no shape transform here!!!!!!!!!!!!!!!!!!!!!!")
+        pass
+
+    def get_augmentation(self):
+        """Subclass must implement this method."""
+        pass
+
+class BaseDataLoader(ABCLoader):
+    def __init__(self,configs,paths=None,dimension=2, **kwargs): 
+        self.configs=configs
+        self.paths=paths
+        self.init_parameters_and_transforms()
+        self.get_loader()
+        #print('all files in dataset:',len(self.source_file_list))
+        
+
+        self.rotation_level = kwargs.get('rotation_level', 0) # Default to no rotation (0)
+        self.zoom_level = kwargs.get('zoom_level', 1.0)  # Default to no zoom (1.0)
+        self.flip = kwargs.get('flip', 0)  # Default to no flip
+
+        self.create_dataset(dimension=dimension)
+
+        ifsave = None if paths is None else True
+        finalcheck(self.train_ds, self.val_ds, 
+               self.train_volume_ds, self.val_volume_ds,
+               self.train_loader, self.val_loader,
+               self.train_patch_ds, 
+               self.train_batch_size, self.val_batch_size,
+               self.saved_name_train, self.saved_name_val, 
+               self.indicator_A, self.indicator_B,
+            ifsave=ifsave, ifcheck=False,iftest_volumes_pixdim=False)
+
+    def get_loader(self):
+        self.source_file_list = []
+        self.train_ds=[]
+        self.val_ds=[]
+        
+
+    def init_parameters_and_transforms(self):
+        self.indicator_A=self.configs.dataset.indicator_A	
+        self.indicator_B=self.configs.dataset.indicator_B
+        self.train_number=self.configs.dataset.train_number
+        self.val_number=self.configs.dataset.val_number
+        self.train_batch_size=self.configs.dataset.batch_size
+        self.val_batch_size=self.configs.dataset.val_batch_size
+        self.load_masks=self.configs.dataset.load_masks
+
+        self.keys = [self.indicator_A, self.indicator_B, "mask"] if self.load_masks else [self.indicator_A, self.indicator_B]
+
+        if self.configs.model_name=='augmentation':
+            # Fixed parameters for rotation and zooming
+            self.train_transforms = self.get_augmentation(transform_list=[], flip=self.flip, rotation_level=self.rotation_level, zoom_level=self.zoom_level)
+        else:
+            self.train_transforms = self.get_transforms(mode='train')
+        self.val_transforms = self.get_transforms(mode='val')
+        
+        if self.paths is not None:
+            self.saved_name_train=self.paths["saved_name_train"]
+            self.saved_name_val=self.paths["saved_name_val"]
+
+    def create_volume_dataset(self):
+        # load volumes and center crop
+        center_crop = self.configs.dataset.center_crop
+        transformations_crop = [
+            LoadImaged(keys=self.keys),
+            EnsureChannelFirstd(keys=self.keys),
+        ]
+        if center_crop>0:
+            transformations_crop.append(CenterSpatialCropd(keys=self.keys, roi_size=(-1,-1,center_crop)))
+        transformations_crop=Compose(transformations_crop)
+        train_crop_ds = monai.data.Dataset(data=self.train_ds, transform=transformations_crop)
+        val_crop_ds = monai.data.Dataset(data=self.val_ds, transform=transformations_crop)
+
+        # load volumes
+        self.train_volume_ds = monai.data.Dataset(data=train_crop_ds, transform=self.train_transforms) 
+        self.val_volume_ds = monai.data.Dataset(data=val_crop_ds, transform=self.val_transforms)
+    
+    def create_patch_dataset_and_dataloader(self, dimension=2):
+        train_batch_size=self.configs.dataset.batch_size
+        val_batch_size=self.configs.dataset.val_batch_size
+        if dimension==2:
+            # batch-level slicer for both image and label
+            window_width=1
+            patch_func = monai.data.PatchIterd(
+                keys=self.keys,
+                patch_size=(None, None, window_width),  # dynamic first two dimensions
+                start_pos=(0, 0, 0)
+            )
+            if window_width==1:
+                patch_transform = Compose(
+                    [
+                        SqueezeDimd(keys=self.keys, dim=-1),  # squeeze the last dim
+                    ]
+                )
+            else:
+                patch_transform = None
+                
+            # for training
+            train_patch_ds = monai.data.GridPatchDataset(
+                data=self.train_volume_ds, patch_iter=patch_func, transform=patch_transform, with_coordinates=False)
+            train_loader = DataLoader(
+                train_patch_ds,
+                batch_size=train_batch_size,
+                num_workers=self.configs.dataset.num_workers,
+                pin_memory=torch.cuda.is_available(),
+            )
+
+            # for validation
+            if self.configs.model_name=='ddpm' or 'ddpm2d_seg2med' or 'ddpm2d':
+                val_patch_ds = monai.data.GridPatchDataset(
+                data=self.val_volume_ds, patch_iter=patch_func, transform=patch_transform, with_coordinates=False)
+                val_loader = DataLoader(
+                    val_patch_ds, #val_volume_ds, 
+                    num_workers=self.configs.dataset.num_workers, 
+                    batch_size=val_batch_size,
+                    pin_memory=torch.cuda.is_available())
+            else:
+                val_loader = DataLoader(
+                    self.val_volume_ds, 
+                    num_workers=self.configs.dataset.num_workers, 
+                    batch_size=val_batch_size,
+                    pin_memory=torch.cuda.is_available())
+            self.train_patch_ds=train_patch_ds
+
+        elif dimension==2.5:
+            # batch-level slicer for both image and label
+            # 2.5 means stack slices together as a small volume patch
+            # if window_width>1, means we train a 2.5D network
+            patch_size=self.configs.dataset.patch_size # (None, None, window_width)
+            window_width=patch_size[-1]
+            patch_func = monai.data.PatchIterd(
+                keys=self.keys,
+                patch_size=patch_size,  # dynamic first two dimensions: (None, None, window_width)
+                start_pos=(0, 0, 0)
+            )
+            if window_width==1:
+                print(f"slice patch is 1, we use 2D-training")
+                patch_transform = Compose(
+                    [
+                        SqueezeDimd(keys=self.keys, dim=-1),  # squeeze the last dim
+                    ]
+                )
+            else:
+                print(f"use consecutive {window_width} slices for 2.5D-training")
+                # there would be an error if original size < patch_size during training, so we should pad it in this case
+                patch_transform = ResizeWithPadOrCropd(keys=self.keys, 
+                                                  spatial_size=patch_size, mode='minimum')
+                
+            # for training
+            train_patch_ds = monai.data.GridPatchDataset(
+                data=self.train_volume_ds, patch_iter=patch_func, transform=patch_transform, with_coordinates=False)
+            train_loader = DataLoader(
+                train_patch_ds,
+                batch_size=train_batch_size,
+                num_workers=2,
+                pin_memory=torch.cuda.is_available(),
+            )
+
+            # for validation
+            if self.configs.model_name=='ddpm':
+                val_patch_ds = monai.data.GridPatchDataset(
+                data=self.val_volume_ds, patch_iter=patch_func, transform=patch_transform, with_coordinates=False)
+                val_loader = DataLoader(
+                    val_patch_ds, #val_volume_ds, 
+                    num_workers=0, 
+                    batch_size=val_batch_size,
+                    pin_memory=torch.cuda.is_available())
+            else:
+                val_loader = DataLoader(
+                    self.val_volume_ds, 
+                    num_workers=1, 
+                    batch_size=val_batch_size,
+                    pin_memory=torch.cuda.is_available())
+            self.train_patch_ds=train_patch_ds
+            
+        elif dimension==3:
+            # 3 means use the whole input volume for training 
+            train_loader = DataLoader(
+                self.train_volume_ds, 
+                num_workers=self.configs.dataset.num_workers, 
+                batch_size=train_batch_size,
+                pin_memory=torch.cuda.is_available())
+            val_loader = DataLoader(
+                self.val_volume_ds, 
+                num_workers=self.configs.dataset.num_workers, 
+                batch_size=val_batch_size,
+                pin_memory=torch.cuda.is_available())
+                
+        elif dimension==3.5: 
+            # 3.5 means create patch from the original volume
+            patch_func = monai.data.PatchIterd(
+                keys=[self.indicator_A, self.indicator_B],
+                patch_size=self.configs.dataset.patch_size,  # dynamic first two dimensions
+                start_pos=(0, 0, 0),
+                mode="replicate",
+            )
+            patch_transform = None
+                
+            # for training
+            train_patch_ds = monai.data.GridPatchDataset(
+                data=self.train_volume_ds, patch_iter=patch_func, transform=patch_transform, with_coordinates=False)
+            train_loader = DataLoader(
+                train_patch_ds,
+                batch_size=train_batch_size,
+                num_workers=self.configs.dataset.num_workers,
+                pin_memory=torch.cuda.is_available(),
+            )
+
+            val_patch_ds = monai.data.GridPatchDataset(
+                data=self.val_volume_ds, patch_iter=patch_func, transform=patch_transform, with_coordinates=False)
+            val_loader = DataLoader(
+                val_patch_ds, #val_volume_ds, 
+                num_workers=self.configs.dataset.num_workers, 
+                batch_size=val_batch_size,
+                pin_memory=torch.cuda.is_available())
+        else:
+            print('dimension of input data must be 2 or 2.5 or 3 or 3.5!')
+
+        
+
+        self.train_batch_size=train_batch_size
+        self.val_batch_size=val_batch_size
+        self.train_loader=train_loader
+        self.val_loader=val_loader
+
+    def create_dataset(self,dimension=2):
+        self.create_volume_dataset()
+        self.create_patch_dataset_and_dataloader(dimension=dimension)
+
+    def get_transforms(self, mode='train'):
+        transform_list=[]
+        transform_list = self.get_pretransforms(transform_list)
+        transform_list = self.get_intensity_transforms(transform_list)
+        transform_list = self.get_normlization(transform_list)
+        transform_list = self.get_shape_transform(transform_list)
+        train_transforms = Compose(transform_list)
+        return train_transforms
+    
+    def get_pretransforms(self, transform_list):
+        #print("customized transforms")
+        return transform_list
+    
+    def get_intensity_transforms(self, transform_list):
+        threshold_low=self.configs.dataset.WINDOW_LEVEL - self.configs.dataset.WINDOW_WIDTH / 2
+        threshold_high=self.configs.dataset.WINDOW_LEVEL + self.configs.dataset.WINDOW_WIDTH / 2
+        offset=(-1)*threshold_low
+        # if filter out the pixel with values below threshold1, set above=True, and the cval1>=threshold1, otherwise there will be problem
+        # mask = img > self.threshold if self.above else img < self.threshold
+        # res = where(mask, img, self.cval)
+        transform_list.append(ThresholdIntensityd(keys=[self.indicator_B], threshold=threshold_low, above=True, cval=threshold_low))
+        transform_list.append(ThresholdIntensityd(keys=[self.indicator_B], threshold=threshold_high, above=False, cval=threshold_high))
+        transform_list.append(ShiftIntensityd(keys=[self.indicator_B], offset=offset))
+        return transform_list
+    
+    def get_normlization(self, transform_list):
+        normalize=self.configs.dataset.normalize
+        indicator_A=self.configs.dataset.indicator_A
+        indicator_B=self.configs.dataset.indicator_B
+        # offset = self.configs.dataset.offset
+        # we don't need normalization for segmentation mask
+        if normalize=='zscore':
+            transform_list.append(NormalizeIntensityd(keys=[indicator_B], nonzero=False, channel_wise=True))
+            print('zscore normalization')
+        elif normalize=='minmax':
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=-1.0, maxv=1.0))
+            print('minmax normalization')
+
+        elif normalize=='scale1000_wrongbutworks':
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=0))
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], factor=-0.999)) 
+            print('scale1000 normalization')
+
+        elif normalize=='scale4000':
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None, factor=-0.99975))
+            print('scale4000 normalization')
+
+        elif normalize=='scale2000':
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None, factor=-0.9995))
+            print('scale2000 normalization')
+
+        elif normalize=='scale1000':
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None, factor=-0.999)) 
+            print('scale1000 normalization')
+        
+        elif normalize=='scale100':
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None,factor=-0.99)) 
+            print('scale10 normalization')
+
+        elif normalize=='scale10':
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None,factor=-0.9)) 
+            print('scale10 normalization')
+
+        elif normalize=='inputonlyzscore':
+            transform_list.append(NormalizeIntensityd(keys=[indicator_A], nonzero=False, channel_wise=True))
+            print('only normalize input MRI images')
+
+        elif normalize=='inputonlyminmax':
+            transform_list.append(ScaleIntensityd(keys=[indicator_A], minv=self.configs.dataset.normmin, maxv=self.configs.dataset.normmax))
+            print('only normalize input MRI images')
+
+        elif normalize == 'nonegative':
+            transform_list.append(ShiftIntensityd(keys=[indicator_B], offset=self.configs.dataset.offset))
+            print('none negative normalization')
+
+        elif normalize=='none' or normalize=='nonorm':
+            print('no normalization')
+
+        return transform_list
+    
+    def get_shape_transform(self, transform_list):
+        spaceXY=self.configs.dataset.spaceXY
+        load_masks=self.configs.dataset.load_masks
+
+        indicator_A=self.configs.dataset.indicator_A
+        indicator_B=self.configs.dataset.indicator_B
+        pad_value=0 #offset*(-1)
+        keys = self.keys #[indicator_A, indicator_B, "mask"] if load_masks else [indicator_A, indicator_B]
+        if spaceXY>0:
+            transform_list.append(Spacingd(keys=[indicator_A], pixdim=(spaceXY, spaceXY, 2.5), mode="bilinear", ensure_same_shape=True)) # 
+            transform_list.append(Spacingd(keys=[indicator_B, "mask"] if load_masks else [indicator_B], 
+                                           pixdim=(spaceXY, spaceXY , 2.5), mode="bilinear", ensure_same_shape=True))
+        
+        transform_list.append(Zoomd(keys=keys, 
+                                    zoom=self.configs.dataset.zoom, keep_size=False, mode='area', padding_mode="constant", value=pad_value))
+        transform_list.append(DivisiblePadd(keys=keys,
+                                            k=self.configs.dataset.div_size, mode="constant", value=pad_value))
+        transform_list.append(ResizeWithPadOrCropd(keys=keys, 
+                                                  spatial_size=self.configs.dataset.resized_size,mode="constant", value=pad_value))
+
+        if self.configs.dataset.rotate:
+            transform_list.append(Rotate90d(keys=keys, k=3))
+        return transform_list
+
+class anish_loader(BaseDataLoader):
+    def __init__(self,configs,paths,dimension=2): 
+        self.configs=configs
+        self.paths=paths
+        self.get_loader()
+        super().create_dataset(dimension=dimension)
+        self.finalcheck(ifsave=True,ifcheck=False,iftest_volumes_pixdim=False)
+
+    def get_loader(self):
+        indicator_A=self.configs.dataset.indicator_A	
+        indicator_B=self.configs.dataset.indicator_B
+        self.indicator_A=indicator_A
+        self.indicator_B=indicator_B
+        train_number=self.configs.dataset.train_number
+        val_number=self.configs.dataset.val_number
+        train_batch_size=self.configs.dataset.batch_size
+        val_batch_size=self.configs.dataset.val_batch_size
+        load_masks=self.configs.dataset.load_masks
+
+        
+        #source_file_list=[os.path.join(j,f'{self.configs.dataset.source_name}.nii.gz') for j in file_list_path] # "ct" for example
+        #target_file_list=[os.path.join(j,f'{self.configs.dataset.target_name}.nii.gz') for j in file_list_path] # "mr" for example
+        #mask_file_list=[os.path.join(j,f'{self.configs.dataset.mask_name}.nii.gz') for j in file_list_path]
+        if self.configs.dataset.data_dir is not None and os.path.exists(self.configs.dataset.data_dir):
+            # check if import data is csv file
+            if self.configs.dataset.data_dir.endswith('.csv'):
+                csv_file = self.configs.dataset.data_dir
+            else:
+                raise ValueError('The data directory in this case must be a csv file!')
+        else:
+            if self.configs.server == 'helix' or self.configs.server == 'helixSingle' or self.configs.server=='helixMultiple':
+                csv_file = './healthy_dissec_helix.csv'
+            else:
+                csv_file = './healthy_dissec.csv'
+
+        if self.configs.dataset.input_is_mask:
+            load_seg=True
+        else:
+            load_seg=False
+        source_file_list, source_Aorta_diss_list=list_img_ad_from_anish_csv(csv_file, load_seg)
+        target_file_list, target_Aorta_diss_list=list_img_ad_from_anish_csv(csv_file)
+        mask_file_list, mask_Aorta_diss_list=list_img_ad_from_anish_csv(csv_file)
+        if load_masks:  
+            train_ds = [{indicator_A: i, indicator_B: j, 'mask': k, 'A_paths': i, 'B_paths': j, 'mask_path': k} 
+                        for i, j, k in zip(source_file_list[0:train_number], target_file_list[0:train_number], mask_file_list[0:train_number])]
+            val_ds = [{indicator_A: i, indicator_B: j, 'mask': k, 'A_paths': i, 'B_paths': j, 'mask_path': k} 
+                    for i, j, k in zip(source_file_list[-val_number:], target_file_list[-val_number:], mask_file_list[-val_number:])]
+        else:
+            train_ds = [{indicator_A: i, indicator_B: j, 'A_paths': i, 'B_paths': j, 'Aorta_diss':ad} 
+                        for i, j, ad in zip(source_file_list[0:train_number], target_file_list[0:train_number], source_Aorta_diss_list[0:train_number])]
+            val_ds = [{indicator_A: i, indicator_B: j, 'A_paths': i, 'B_paths': j, 'Aorta_diss':ad} 
+                    for i, j, ad in zip(source_file_list[-val_number:], target_file_list[-val_number:], source_Aorta_diss_list[-val_number:])]
+        self.train_ds=train_ds
+        self.val_ds=val_ds
+        self.source_file_list=source_file_list
+        self.target_file_list=target_file_list
+        self.mask_file_list=mask_file_list  
+
+    def get_pretransforms(self, transform_list):
+        normalize=self.configs.dataset.normalize
+        indicator_A=self.configs.dataset.indicator_A
+        indicator_B=self.configs.dataset.indicator_B
+        load_masks=self.configs.dataset.load_masks
+        input_is_mask=self.configs.dataset.input_is_mask
+        if not input_is_mask:
+            transform_list.append(CreateMaskTransformd(keys=[indicator_A],
+                                                    tissue_min=self.configs.dataset.tissue_min,
+                                                    tissue_max=self.configs.dataset.tissue_max,
+                                                    bone_min=self.configs.dataset.bone_min,
+                                                    bone_max=self.configs.dataset.bone_max))
+
+from dataprocesser.customized_transforms import CreateMaskTransformd, MergeMasksTransformd
+
+class synthrad_seg_loader(BaseDataLoader):
+    def __init__(self,configs,paths,dimension=2,**kwargs): 
+        super().__init__(configs,paths,dimension,**kwargs)
+        
+    def get_loader(self):
+        # volume-level transforms for both image and label
+        indicator_A=self.configs.dataset.indicator_A	
+        indicator_B=self.configs.dataset.indicator_B
+        train_number=self.configs.dataset.train_number
+        val_number=self.configs.dataset.val_number
+        self.indicator_A=indicator_A
+        self.indicator_B=indicator_B
+        load_masks=self.configs.dataset.load_masks
+        # Conditional dictionary keys based on whether masks are loaded
+        
+        #list all files in the folder
+        file_list=[i for i in os.listdir(self.configs.dataset.data_dir) if 'overview' not in i]
+        file_list_path=[os.path.join(self.configs.dataset.data_dir,i) for i in file_list]
+        #list all ct and mr files in folder
+        
+        
+        # mask file means the images are used for extracting body contour, see get_pretransforms() below
+        source_file_list, patient_IDs=list_img_pID_from_synthrad_folder(self.configs.dataset.data_dir, accepted_modalities=self.configs.dataset.source_name, saved_name=os.path.join(self.paths["saved_logs_folder"],"source_filenames.txt"))
+        target_file_list, _=list_img_pID_from_synthrad_folder(self.configs.dataset.data_dir, accepted_modalities=self.configs.dataset.target_name, saved_name=os.path.join(self.paths["saved_logs_folder"],"target_filenames.txt"))
+        mask_file_list, _=list_img_pID_from_synthrad_folder(self.configs.dataset.data_dir, accepted_modalities=self.configs.dataset.target_name, saved_name=os.path.join(self.paths["saved_logs_folder"],"mask_filenames.txt"))
+
+        self.source_file_list=source_file_list
+        self.target_file_list=target_file_list
+        self.mask_file_list=mask_file_list
+        
+        Manual_Set_Aorta_Diss = 0
+        ad = Manual_Set_Aorta_Diss
+        train_ds = [{indicator_A: i, indicator_B: j, 'mask': k, 'A_paths': i, 'B_paths': j, 'mask_path': k, 'Aorta_diss':ad, 'patient_ID': pID} 
+                    for i, j, k, pID in zip(source_file_list[0:train_number], target_file_list[0:train_number], mask_file_list[0:train_number], patient_IDs[0:train_number])]
+        val_ds = [{indicator_A: i, indicator_B: j, 'mask': k, 'A_paths': i, 'B_paths': j, 'mask_path': k, 'Aorta_diss':ad, 'patient_ID': pID} 
+                for i, j, k, pID in zip(source_file_list[-val_number:], target_file_list[-val_number:], mask_file_list[-val_number:], patient_IDs[-val_number:])]
+        self.train_ds=train_ds
+        self.val_ds=val_ds
+
+    def get_pretransforms(self, transform_list):
+        indicator_A=self.configs.dataset.indicator_A
+        indicator_B=self.configs.dataset.indicator_B
+
+        transform_list.append(CreateMaskTransformd(keys=['mask'],
+                                                        body_threshold=-500,
+                                                        body_mask_value=1,
+                                                        ))
+        transform_list.append(MergeMasksTransformd(keys=[indicator_A, 'mask']))
+        return transform_list
+    
+
+from dataprocesser.customized_transforms import CreateMaskTransformd, MergeMasksTransformd, MaskHUAssigmentd
+
+from monai.transforms import (
+    ScaleIntensityd,
+    ThresholdIntensityd,
+    NormalizeIntensityd,
+    ShiftIntensityd,
+)
+
+class anish_seg_loader(BaseDataLoader):
+    def __init__(self,configs,paths=None,dimension=2, **kwargs): 
+        super().__init__(configs,paths,dimension, **kwargs)
+
+    def get_loader(self):
+        indicator_A=self.configs.dataset.indicator_A	
+        indicator_B=self.configs.dataset.indicator_B
+        self.indicator_A=indicator_A
+        self.indicator_B=indicator_B
+        train_number=self.configs.dataset.train_number
+        val_number=self.configs.dataset.val_number
+        train_batch_size=self.configs.dataset.batch_size
+        val_batch_size=self.configs.dataset.val_batch_size
+        load_masks=self.configs.dataset.load_masks
+
+        
+        #source_file_list=[os.path.join(j,f'{self.configs.dataset.source_name}.nii.gz') for j in file_list_path] # "ct" for example
+        #target_file_list=[os.path.join(j,f'{self.configs.dataset.target_name}.nii.gz') for j in file_list_path] # "mr" for example
+        #mask_file_list=[os.path.join(j,f'{self.configs.dataset.mask_name}.nii.gz') for j in file_list_path]
+        print('use csv dataset:',self.configs.dataset.data_dir)
+        if self.configs.dataset.data_dir is not None and os.path.exists(self.configs.dataset.data_dir):
+            # check if import data is csv file
+            if self.configs.dataset.data_dir.endswith('.csv'):
+                csv_file = self.configs.dataset.data_dir
+            else:
+                raise ValueError('The data directory in this case must be a csv file!')
+        else:
+            if self.configs.server == 'helix' or self.configs.server == 'helixSingle' or self.configs.server=='helixMultiple':
+                csv_file = './healthy_dissec_helix.csv'
+            else:
+                csv_file = './healthy_dissec.csv'
+
+        if self.configs.dataset.input_is_mask:
+            load_seg=True
+        else:
+            load_seg=False
+        source_file_list, source_Aorta_diss_list, patient_IDs=list_img_ad_pIDs_from_anish_csv(csv_file, load_seg)
+        target_file_list, _, _ =list_img_ad_pIDs_from_anish_csv(csv_file)
+        mask_file_list, _, _=list_img_ad_pIDs_from_anish_csv(csv_file)
+
+        # here the original CT images are loaded as mask because they will be further processed as body contour and merged into mask.
+
+        if load_masks:  
+            train_ds = [{indicator_A: i, indicator_B: j, 'mask': k, 'A_paths': i, 'B_paths': j, 'mask_path': k, 'Aorta_diss':ad, 'patient_ID': pID} 
+                        for i, j, k, ad, pID in zip(source_file_list[0:train_number], target_file_list[0:train_number], mask_file_list[0:train_number], source_Aorta_diss_list[0:train_number], patient_IDs[0:train_number])]
+            
+            val_ds = [{indicator_A: i, indicator_B: j, 'mask': k, 'A_paths': i, 'B_paths': j, 'mask_path': k, 'Aorta_diss':ad, 'patient_ID': pID} 
+                    for i, j, k, ad, pID in zip(source_file_list[-val_number:], target_file_list[-val_number:], mask_file_list[-val_number:], source_Aorta_diss_list[-val_number:], patient_IDs[-val_number:])]
+        else:
+            train_ds = [{indicator_A: i, indicator_B: j, 'A_paths': i, 'B_paths': j, 'Aorta_diss':ad} 
+                        for i, j, ad in zip(source_file_list[0:train_number], target_file_list[0:train_number], source_Aorta_diss_list[0:train_number])]
+            val_ds = [{indicator_A: i, indicator_B: j, 'A_paths': i, 'B_paths': j, 'Aorta_diss':ad} 
+                    for i, j, ad in zip(source_file_list[-val_number:], target_file_list[-val_number:], source_Aorta_diss_list[-val_number:])]
+        print('train_ds: \n')
+        for i in train_ds:
+            print(i)
+            print('\n')
+        self.train_ds=train_ds
+        self.val_ds=val_ds
+        self.source_file_list=source_file_list
+        self.target_file_list=target_file_list
+        self.mask_file_list=mask_file_list
+        
+    def get_pretransforms(self, transform_list):
+        indicator_A=self.configs.dataset.indicator_A
+        indicator_B=self.configs.dataset.indicator_B
+
+        transform_list.append(CreateMaskTransformd(keys=['mask'],
+                                                        body_threshold=-500,
+                                                        body_mask_value=1,
+                                                        ))
+        transform_list.append(MergeMasksTransformd(keys=[indicator_A, 'mask']))
+        return transform_list
+    
+
+class combined_seg_loader(BaseDataLoader):
+    def __init__(self,configs,paths,dimension=2,**kwargs): 
+        self.dimension = dimension
+        self.train_number_1 = kwargs.get('train_number_1', 170) 
+        self.train_number_2 = kwargs.get('train_number_2', 152)  
+        self.val_number_1 = kwargs.get('val_number_1', 10) 
+        self.val_number_2 = kwargs.get('val_number_2', 10)  
+        self.data_dir_1 = kwargs.get('data_dir_1', 'E:\Projects\yang_proj\data\synthrad\Task1\pelvis')
+        self.data_dir_2 = kwargs.get('data_dir_2', 'E:\Projects\yang_proj\SynthRad_GAN\synthrad_conversion\healthy_dissec.csv')
+        super().__init__(configs,paths,dimension,**kwargs)
+        
+
+    def get_loader(self):
+        # define the dataset sizes for the dataset 1
+        self.configs.dataset.data_dir = self.data_dir_1
+        self.configs.dataset.train_number = self.train_number_1
+        self.configs.dataset.val_number = self.val_number_1
+        self.configs.dataset.source_name = ["ct_seg"]
+        self.configs.dataset.target_name = ["ct"]
+        self.configs.dataset.offset = 1024
+        loader1 = synthrad_seg_loader(self.configs,self.paths,self.dimension)
+        source_file_list1 = loader1.source_file_list
+
+        # define the dataset sizes for the dataset 2
+        self.configs.dataset.data_dir = self.data_dir_2
+        self.configs.dataset.train_number = self.train_number_2
+        self.configs.dataset.val_number = self.val_number_2
+        self.configs.dataset.offset = 1000
+        loader2 = anish_seg_loader(self.configs,self.paths,self.dimension)
+        source_file_list2 = loader2.source_file_list
+
+        train_ds1 = loader1.train_ds
+        train_ds2 = loader2.train_ds
+
+        val_ds1 = loader1.val_ds
+        val_ds2 = loader2.val_ds
+
+        self.train_ds = ConcatDataset([train_ds1, train_ds2])
+        self.val_ds = ConcatDataset([val_ds1, val_ds2])
+        self.source_file_list = source_file_list1+source_file_list2
+    def get_pretransforms(self, transform_list):
+        indicator_A=self.configs.dataset.indicator_A
+        indicator_B=self.configs.dataset.indicator_B
+
+        transform_list.append(CreateMaskTransformd(keys=['mask'],
+                                                        body_threshold=-500,
+                                                        body_mask_value=1,
+                                                        ))
+        transform_list.append(MergeMasksTransformd(keys=[indicator_A, 'mask']))
+        return transform_list
+
+    def save_nifti(self, save_output_path, case=0):
+        from monai.transforms import SaveImage
+        step = 0
+        with torch.no_grad():
+            for data in self.train_loader:
+                si_input = SaveImage(output_dir=f'{save_output_path}',
+                    separate_folder=False,
+                    output_postfix=f'', # aug_{step}
+                    resample=False)
+                si_seg = SaveImage(output_dir=f'{save_output_path}',
+                    separate_folder=False,
+                    output_postfix=f'', # aug_{step}
+                    resample=False)
+                
+                image_batch = data['img'].squeeze()
+                seg_batch = data['seg'].squeeze()
+                file_path_batch = data['B_paths']
+                Aorta_diss = data['Aorta_diss']
+
+                batch_size = len(file_path_batch)
+
+                for i in range(batch_size):
+                    step += 1
+
+                    file_path = file_path_batch[i]
+                    image = image_batch[i]
+                    seg = seg_batch[i]
+
+                    patient_ID = os.path.splitext(os.path.basename(file_path))[0]
+                    save_name_img = patient_ID + str(case) + '_' + str(step)
+                    save_name_img = os.path.join(save_output_path, save_name_img)
+
+                    save_name_seg = patient_ID + str(case) + '_' + str(step) + '_seg'
+                    save_name_seg = os.path.join(save_output_path, save_name_seg)
+                    
+                    si_input(image.unsqueeze(0), data['img'].meta, filename=save_name_img)
+                    si_seg(seg.unsqueeze(0), data['seg'].meta, filename=save_name_seg)
+
+class combined_seg_assigned_loader(combined_seg_loader):
+    def __init__(self,configs,paths=None,dimension=2, **kwargs): 
+        self.anatomy_list = kwargs.get('anatomy_list', 'synthrad_conversion/TA2_anatomy.csv')
+        super().__init__(configs, paths, dimension, **kwargs)
+        
+    def get_pretransforms(self, transform_list):
+        indicator_A=self.configs.dataset.indicator_A
+        indicator_B=self.configs.dataset.indicator_B
+
+        transform_list.append(CreateMaskTransformd(keys=['mask'],
+                                                        body_threshold=-500,
+                                                        body_mask_value=1,
+                                                        ))
+        transform_list.append(MergeMasksTransformd(keys=[indicator_A, 'mask']))
+        transform_list.append(MaskHUAssigmentd(keys=[self.indicator_A], csv_file=self.anatomy_list))
+        return transform_list
+    
+    def get_intensity_transforms(self, transform_list):
+        threshold_low=self.configs.dataset.WINDOW_LEVEL - self.configs.dataset.WINDOW_WIDTH / 2
+        threshold_high=self.configs.dataset.WINDOW_LEVEL + self.configs.dataset.WINDOW_WIDTH / 2
+        offset=(-1)*threshold_low
+        # if filter out the pixel with values below threshold1, set above=True, and the cval1>=threshold1, otherwise there will be problem
+        # mask = img > self.threshold if self.above else img < self.threshold
+        # res = where(mask, img, self.cval)
+        transform_list.append(ThresholdIntensityd(keys=[self.indicator_A,self.indicator_B], threshold=threshold_low, above=True, cval=threshold_low))
+        transform_list.append(ThresholdIntensityd(keys=[self.indicator_A,self.indicator_B], threshold=threshold_high, above=False, cval=threshold_high))
+        transform_list.append(ShiftIntensityd(keys=[self.indicator_A,self.indicator_B], offset=offset))
+        return transform_list
+    
+    def get_normlization(self, transform_list):
+        normalize=self.configs.dataset.normalize
+        # offset = self.configs.dataset.offset
+        # we don't need normalization for segmentation mask
+        if normalize=='zscore':
+            transform_list.append(NormalizeIntensityd(keys=[self.indicator_A,self.indicator_B], nonzero=False, channel_wise=True))
+            print('zscore normalization')
+
+        elif normalize=='scale2000':
+            transform_list.append(ScaleIntensityd(keys=[self.indicator_A,self.indicator_B], minv=None, maxv=None, factor=-0.9995))
+            print('scale2000 normalization')
+
+        elif normalize=='none' or normalize=='nonorm':
+            print('no normalization')
+
+        return transform_list
+    
+class slices_nifti_DataLoader(BaseDataLoader):
+    def __init__(self,configs,paths=None,dimension=2, **kwargs): 
+        super().__init__(configs, paths, dimension, **kwargs)
+
+    def get_loader(self):
+        print('use json dataset:',self.configs.dataset.data_dir)
+        if self.configs.dataset.data_dir is not None and os.path.exists(self.configs.dataset.data_dir):
+            json_file_root = self.configs.dataset.data_dir
+        else:
+            raise ValueError('please check the data dir in config file!')
+        json_file_train = os.path.join(json_file_root, 'train', 'dataset.json')
+        json_file_val = os.path.join(json_file_root, 'val', 'dataset.json')
+
+        self.train_ds = list_from_json(json_file_train, self.indicator_A, self.indicator_B)
+        self.val_ds = list_from_json(json_file_val, self.indicator_A, self.indicator_B)
+        
+    def create_patch_dataset_and_dataloader(self, dimension=2):
+        train_batch_size=self.configs.dataset.batch_size
+        val_batch_size=self.configs.dataset.val_batch_size
+        self.train_loader = DataLoader(
+            self.train_volume_ds, 
+            num_workers=self.configs.dataset.num_workers, 
+            batch_size=train_batch_size,
+            shuffle=True,
+            pin_memory=torch.cuda.is_available())
+        
+        self.val_loader = DataLoader(
+            self.val_volume_ds, 
+            num_workers=self.configs.dataset.num_workers, 
+            batch_size=val_batch_size,
+            shuffle=False,
+            pin_memory=torch.cuda.is_available())
+        
+class csv_slices_DataLoader(BaseDataLoader):
+    def __init__(self,configs,paths=None,dimension=2, **kwargs): 
+        super().__init__(configs, paths, dimension, **kwargs)
+
+    def get_loader(self):
+        print('use csv dataset:',self.configs.dataset.data_dir)
+        if self.configs.dataset.data_dir is not None and os.path.exists(self.configs.dataset.data_dir):
+            csv_file_root = self.configs.dataset.data_dir
+        else:
+            raise ValueError('please check the data dir in config file!')
+        folder_train = os.path.join(csv_file_root, 'train')
+        folder__val = os.path.join(csv_file_root, 'val')
+
+        self.train_ds = list_from_slice_csv(folder_train, self.indicator_A, self.indicator_B)
+        self.val_ds = list_from_slice_csv(folder__val, self.indicator_A, self.indicator_B)
+        
+    def create_patch_dataset_and_dataloader(self, dimension=2):
+        train_batch_size=self.configs.dataset.batch_size
+        val_batch_size=self.configs.dataset.val_batch_size
+        self.train_loader = DataLoader(
+            self.train_volume_ds, 
+            num_workers=self.configs.dataset.num_workers, 
+            batch_size=train_batch_size,
+            shuffle=True,
+            pin_memory=torch.cuda.is_available())
+        
+        self.val_loader = DataLoader(
+            self.val_volume_ds, 
+            num_workers=self.configs.dataset.num_workers, 
+            batch_size=val_batch_size,
+            shuffle=False,
+            pin_memory=torch.cuda.is_available())
+        
+class csv_slices_assigned_DataLoader(csv_slices_DataLoader):
+    def __init__(self,configs,paths=None,dimension=2, **kwargs): 
+        super().__init__(configs, paths, dimension, **kwargs)
+
+    def get_pretransforms(self, transform_list):
+        transform_list.append(MaskHUAssigmentd(keys=[self.indicator_A], csv_file=r'synthrad_conversion\TA2_anatomy.csv'))
+        return transform_list
+    
+    def get_intensity_transforms(self, transform_list):
+        threshold_low=self.configs.dataset.WINDOW_LEVEL - self.configs.dataset.WINDOW_WIDTH / 2
+        threshold_high=self.configs.dataset.WINDOW_LEVEL + self.configs.dataset.WINDOW_WIDTH / 2
+        offset=(-1)*threshold_low
+        # if filter out the pixel with values below threshold1, set above=True, and the cval1>=threshold1, otherwise there will be problem
+        # mask = img > self.threshold if self.above else img < self.threshold
+        # res = where(mask, img, self.cval)
+        transform_list.append(ThresholdIntensityd(keys=[self.indicator_A,self.indicator_B], threshold=threshold_low, above=True, cval=threshold_low))
+        transform_list.append(ThresholdIntensityd(keys=[self.indicator_A,self.indicator_B], threshold=threshold_high, above=False, cval=threshold_high))
+        transform_list.append(ShiftIntensityd(keys=[self.indicator_A,self.indicator_B], offset=offset))
+        return transform_list
+    
+    def get_normlization(self, transform_list):
+        normalize=self.configs.dataset.normalize
+        # offset = self.configs.dataset.offset
+        # we don't need normalization for segmentation mask
+        if normalize=='zscore':
+            transform_list.append(NormalizeIntensityd(keys=[self.indicator_A,self.indicator_B], nonzero=False, channel_wise=True))
+            print('zscore normalization')
+
+        elif normalize=='scale2000':
+            transform_list.append(ScaleIntensityd(keys=[self.indicator_A,self.indicator_B], minv=None, maxv=None, factor=-0.9995))
+            print('scale2000 normalization')
+
+        elif normalize=='none' or normalize=='nonorm':
+            print('no normalization')
+
+        return transform_list
+
+# for MRI -> CT task
+
+class synthrad_mr2ct_loader(BaseDataLoader):
+    def __init__(self,configs,paths=None,dimension=2): 
+        super().__init__(configs,paths,dimension)
+        
+    def get_loader(self):
+        # volume-level transforms for both image and label
+        indicator_A=self.configs.dataset.indicator_A	
+        indicator_B=self.configs.dataset.indicator_B
+        train_number=self.configs.dataset.train_number
+        val_number=self.configs.dataset.val_number
+        self.indicator_A=indicator_A
+        self.indicator_B=indicator_B
+        load_masks=self.configs.dataset.load_masks
+        # Conditional dictionary keys based on whether masks are loaded
+        
+        #list all files in the folder
+        file_list=[i for i in os.listdir(self.configs.dataset.data_dir) if 'overview' not in i]
+        file_list_path=[os.path.join(self.configs.dataset.data_dir,i) for i in file_list]
+        #list all ct and mr files in folder
+        
+        
+        #source_file_list=[os.path.join(j,f'{self.configs.dataset.source_name}.nii.gz') for j in file_list_path] # "ct" for example
+        #target_file_list=[os.path.join(j,f'{self.configs.dataset.target_name}.nii.gz') for j in file_list_path] # "mr" for example
+        #mask_file_list=[os.path.join(j,f'{self.configs.dataset.mask_name}.nii.gz') for j in file_list_path]
+        source_file_list,_=list_img_pID_from_synthrad_folder(self.configs.dataset.data_dir, accepted_modalities=self.configs.dataset.source_name,saved_name=None)
+        target_file_list,_=list_img_pID_from_synthrad_folder(self.configs.dataset.data_dir, accepted_modalities=self.configs.dataset.target_name,saved_name=None)
+        mask_file_list,_=list_img_pID_from_synthrad_folder(self.configs.dataset.data_dir, accepted_modalities=self.configs.dataset.mask_name,saved_name=None)
+
+        def write_write_file(images, file):
+            with open(file,"w") as file:
+                for image in images:
+                    file.write(f'{image} \n')
+                    
+        if self.paths is not None:
+            write_write_file(source_file_list, os.path.join(self.paths["saved_logs_folder"],"source_filenames.txt"))
+            write_write_file(target_file_list, os.path.join(self.paths["saved_logs_folder"],"target_filenames.txt"))
+            write_write_file(mask_file_list, os.path.join(self.paths["saved_logs_folder"],"mask_filenames.txt"))
+
+        self.source_file_list=source_file_list
+        self.target_file_list=target_file_list
+        self.mask_file_list=mask_file_list
+        
+        if load_masks:  
+            train_ds = [{indicator_A: i, indicator_B: j, 'mask': k, 'A_paths': i, 'B_paths': j, 'mask_path': k} 
+                        for i, j, k in zip(source_file_list[0:train_number], target_file_list[0:train_number], mask_file_list[0:train_number])]
+            val_ds = [{indicator_A: i, indicator_B: j, 'mask': k, 'A_paths': i, 'B_paths': j, 'mask_path': k} 
+                    for i, j, k in zip(source_file_list[-val_number:], target_file_list[-val_number:], mask_file_list[-val_number:])]
+        else:
+            train_ds = [{indicator_A: i, indicator_B: j, 'A_paths': i, 'B_paths': j} 
+                        for i, j in zip(source_file_list[0:train_number], target_file_list[0:train_number])]
+            val_ds = [{indicator_A: i, indicator_B: j, 'A_paths': i, 'B_paths': j} 
+                    for i, j in zip(source_file_list[-val_number:], target_file_list[-val_number:])]
+        self.train_ds=train_ds
+        self.val_ds=val_ds
+
+    def get_normlization(self, transform_list):
+        normalize=self.configs.dataset.normalize
+        indicator_A=self.configs.dataset.indicator_A
+        indicator_B=self.configs.dataset.indicator_B
+        load_masks=self.configs.dataset.load_masks
+        if normalize=='zscore':
+            transform_list.append(NormalizeIntensityd(keys=[indicator_A, indicator_B], nonzero=False, channel_wise=True))
+            print('zscore normalization')
+        elif normalize=='minmax':
+            transform_list.append(ScaleIntensityd(keys=[indicator_A, indicator_B], minv=-1.0, maxv=1.0))
+            print('minmax normalization')
+
+        elif normalize=='scale4000':
+            transform_list.append(ScaleIntensityd(keys=[indicator_A], minv=0, maxv=1))
+            transform_list.append(ShiftIntensityd(keys=[indicator_B], offset=1024))
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None, factor=-0.99975)) # x=x(1+factor)
+            print('scale4000 normalization')
+
+        elif normalize=='scale1000_wrongbutworks':
+            transform_list.append(ScaleIntensityd(keys=[indicator_A], minv=0, maxv=1))
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=0))
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], factor=-0.999)) 
+            print('scale1000 normalization')
+
+        elif normalize=='scale1000':
+            transform_list.append(ScaleIntensityd(keys=[indicator_A], minv=0, maxv=1))
+            transform_list.append(ShiftIntensityd(keys=[indicator_B], offset=1024))
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None, factor=-0.999)) 
+            print('scale1000 normalization')
+        
+        elif normalize=='scale10':
+            transform_list.append(ScaleIntensityd(keys=[indicator_A], minv=0, maxv=1))
+            #transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=0))
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None,factor=-0.9)) 
+            print('scale10 normalization')
+
+        elif normalize=='inputonlyzscore':
+            transform_list.append(NormalizeIntensityd(keys=[indicator_A], nonzero=False, channel_wise=True))
+            print('only normalize input MRI images')
+
+        elif normalize=='inputonlyminmax':
+            transform_list.append(ScaleIntensityd(keys=[indicator_A], minv=self.configs.dataset.normmin, maxv=self.configs.dataset.normmax))
+            print('only normalize input MRI images')
+        
+        elif normalize=='none' or normalize=='nonorm':
+            print('no normalization')
+        return transform_list
+
+class anika_registrated_mr2ct_loader(synthrad_mr2ct_loader):
+    def __init__(self,configs,paths,dimension): 
+        super().__init__(configs,paths,dimension)
+
+    def get_loader(self):
+        indicator_A=self.configs.dataset.indicator_A	
+        indicator_B=self.configs.dataset.indicator_B
+        self.indicator_A=indicator_A
+        self.indicator_B=indicator_B
+        train_number=self.configs.dataset.train_number
+        val_number=self.configs.dataset.val_number
+        train_batch_size=self.configs.dataset.batch_size
+        val_batch_size=self.configs.dataset.val_batch_size
+        load_masks=self.configs.dataset.load_masks
+
+        # Conditional dictionary keys based on whether masks are loaded
+        keys = [indicator_A, indicator_B, "mask"] if load_masks else [indicator_A, indicator_B]
+
+        ct_dir = r'E:\Datasets\M2olie_Patientdata\CT'
+        mri_dir = r'E:\Results\MultistepReg\M2olie_Patientdata\Multistep_network_A\predict'
+        
+        ct_dir = self.configs.dataset.ct_dir #'E:\Datasets\M2olie_Patientdata\CT'
+        mri_dir = self.configs.dataset.mri_dir #'E:\Results\MultistepReg\M2olie_Patientdata\Multistep_network_A\predict'
+        matched_pairs = list_from_anika_dataset(ct_dir, mri_dir, self.configs.dataset.mri_mode)
+        for patient_id, paths in matched_pairs.items():
+            print(f"Patient ID: {patient_id}, CT: {paths['CT']}, MRI: {paths['MRI']}")
+
+        # use the matched pairs to form the dataset
+        train_ds = [{indicator_A: paths['MRI'], indicator_B: paths['CT']} for patient_id, paths in list(matched_pairs.items())[:train_number]]
+        val_ds = [{indicator_A: paths['MRI'], indicator_B: paths['CT']} for patient_id, paths in list(matched_pairs.items())[-val_number:]]

dataprocesser/archive/list_dataset_combined_seg.py

@@ -0,0 +1,15 @@
+import os
+from dataprocesser.customized_transforms import CreateMaskTransformd, MergeMasksTransformd
+IMG_EXTENSIONS = [
+    #'.jpg', '.JPG', '.jpeg', '.JPEG',
+    #'.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP', 
+    '.nrrd', '.nii.gz'
+]
+
+def is_image_file(filename):
+    return any(filename.endswith(extension) for extension in IMG_EXTENSIONS)
+import torch
+from dataprocesser.list_dataset_synthrad_seg import synthrad_seg_loader
+from dataprocesser.list_dataset_Anish_seg import anish_seg_loader
+from dataprocesser.list_dataset_base import BaseDataLoader
+

dataprocesser/archive/list_dataset_combined_seg_assigned.py

@@ -0,0 +1 @@
+

dataprocesser/archive/list_dataset_synthrad_seg.py

@@ -0,0 +1,3 @@
+
+
+    

dataprocesser/archive/monai_loader_3D.py

@@ -0,0 +1,367 @@
+import monai
+import os
+import numpy as np
+from monai.transforms import (
+    Compose,
+    LoadImaged,
+    EnsureChannelFirstd,
+    SqueezeDimd,
+    CenterSpatialCropd,
+    Rotate90d,
+    ScaleIntensityd,
+    ResizeWithPadOrCropd,
+    DivisiblePadd,
+    
+    ThresholdIntensityd,
+    NormalizeIntensityd,
+    ShiftIntensityd,
+    Identityd,
+    ScaleIntensityRanged,
+    Spacingd,
+)
+from torch.utils.data import DataLoader
+import torch
+
+IMG_EXTENSIONS = [
+    #'.jpg', '.JPG', '.jpeg', '.JPEG',
+    #'.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP', 
+    '.nrrd', '.nii.gz'
+]
+
+def is_image_file(filename):
+    return any(filename.endswith(extension) for extension in IMG_EXTENSIONS)
+
+def make_dataset_modality(dir, accepted_modalities = ["ct"], saved_name="source_filenames.txt"):
+    # it works for root path of any layer:
+    # data_path/Task1 or Task2/pelvis or brain
+            # |-patient1
+            #   |-ct.nii.gz
+            #   |-mr.nii.gz
+            # |-patient2
+            #   |-ct.nii.gz
+            #   |-mr.nii.gz
+    images = []
+    assert os.path.isdir(dir), '%s is not a valid directory' % dir
+    for roots, _, files in sorted(os.walk(dir)): # os.walk digs all folders and subfolders in all layers of dir
+        for file in files:
+            if is_image_file(file) and file.split('.')[0] in accepted_modalities:
+                path = os.path.join(roots, file)
+                images.append(path)
+    print(f'Found {len(images)} {accepted_modalities} files in {dir} \n')
+    with open(saved_name,"w") as file:
+        for image in images:
+            file.write(f'{image} \n')
+    return images
+
+class monai_loader_3D:
+    def __init__(self,configs,paths): 
+        self.configs=configs
+        self.paths=paths
+        self.get_loader()
+        self.finalcheck(ifsave=True,ifcheck=False,iftest_volumes_pixdim=False)
+
+    def get_loader(self):
+        # volume-level transforms for both image and label
+        train_transforms = self.get_transforms(self.configs,mode='train')
+        val_transforms = self.get_transforms(self.configs,mode='val')
+        indicator_A=self.configs.dataset.indicator_A	
+        indicator_B=self.configs.dataset.indicator_B
+        self.indicator_A=indicator_A
+        self.indicator_B=indicator_B
+        train_number=self.configs.dataset.train_number
+        val_number=self.configs.dataset.val_number
+        train_batch_size=self.configs.dataset.batch_size
+        val_batch_size=self.configs.dataset.val_batch_size
+        load_masks=self.configs.dataset.load_masks
+
+        # Conditional dictionary keys based on whether masks are loaded
+        keys = [indicator_A, indicator_B, "mask"] if load_masks else [indicator_A, indicator_B]
+
+
+        #list all files in the folder
+        file_list=[i for i in os.listdir(self.configs.dataset.data_dir) if 'overview' not in i]
+        file_list_path=[os.path.join(self.configs.dataset.data_dir,i) for i in file_list]
+        #list all ct and mr files in folder
+        
+        
+        #source_file_list=[os.path.join(j,f'{self.configs.dataset.source_name}.nii.gz') for j in file_list_path] # "ct" for example
+        #target_file_list=[os.path.join(j,f'{self.configs.dataset.target_name}.nii.gz') for j in file_list_path] # "mr" for example
+        #mask_file_list=[os.path.join(j,f'{self.configs.dataset.mask_name}.nii.gz') for j in file_list_path]
+        source_file_list=make_dataset_modality(self.configs.dataset.data_dir, accepted_modalities=self.configs.dataset.source_name, saved_name=os.path.join(self.paths["saved_logs_folder"],"source_filenames.txt"))
+        target_file_list=make_dataset_modality(self.configs.dataset.data_dir, accepted_modalities=self.configs.dataset.target_name, saved_name=os.path.join(self.paths["saved_logs_folder"],"target_filenames.txt"))
+        mask_file_list=make_dataset_modality(self.configs.dataset.data_dir, accepted_modalities=self.configs.dataset.mask_name, saved_name=os.path.join(self.paths["saved_logs_folder"],"mask_filenames.txt"))
+
+        if load_masks:  
+            train_ds = [{indicator_A: i, indicator_B: j, 'mask': k, 'A_paths': i, 'B_paths': j, 'mask_path': k} 
+                        for i, j, k in zip(source_file_list[0:train_number], target_file_list[0:train_number], mask_file_list[0:train_number])]
+            val_ds = [{indicator_A: i, indicator_B: j, 'mask': k, 'A_paths': i, 'B_paths': j, 'mask_path': k} 
+                    for i, j, k in zip(source_file_list[-val_number:], target_file_list[-val_number:], mask_file_list[-val_number:])]
+        else:
+            train_ds = [{indicator_A: i, indicator_B: j, 'A_paths': i, 'B_paths': j} 
+                        for i, j in zip(source_file_list[0:train_number], target_file_list[0:train_number])]
+            val_ds = [{indicator_A: i, indicator_B: j, 'A_paths': i, 'B_paths': j} 
+                    for i, j in zip(source_file_list[-val_number:], target_file_list[-val_number:])]
+
+        print('all files in dataset:',len(source_file_list))
+
+        # load volumes and center crop
+        center_crop = self.configs.dataset.center_crop
+        transformations_crop = [
+            LoadImaged(keys=keys),
+            EnsureChannelFirstd(keys=keys),
+        ]
+        if center_crop>0:
+            transformations_crop.append(CenterSpatialCropd(keys=keys, roi_size=(-1,-1,center_crop)))
+        transformations_crop=Compose(transformations_crop)
+        train_crop_ds = monai.data.Dataset(data=train_ds, transform=transformations_crop)
+        val_crop_ds = monai.data.Dataset(data=val_ds, transform=transformations_crop)
+
+        # load volumes
+        train_volume_ds = monai.data.Dataset(data=train_crop_ds, transform=train_transforms) 
+        val_volume_ds = monai.data.Dataset(data=val_crop_ds, transform=val_transforms)
+        
+        train_loader = DataLoader(train_volume_ds, batch_size=train_batch_size, shuffle=True, num_workers=self.configs.dataset.num_workers)
+        val_loader = DataLoader(val_volume_ds, batch_size=val_batch_size, shuffle=False, num_workers=self.configs.dataset.num_workers)
+
+        self.saved_name_train=self.paths["saved_name_train"]
+        self.saved_name_val=self.paths["saved_name_val"]
+
+        self.train_ds=train_ds
+        self.val_ds=val_ds
+        self.train_volume_ds=train_volume_ds
+        self.val_volume_ds=val_volume_ds
+        
+        self.train_batch_size=train_batch_size
+        self.val_batch_size=val_batch_size
+
+        self.train_crop_ds=train_crop_ds
+        self.val_crop_ds=val_crop_ds
+        self.train_transforms=train_transforms
+        self.val_transforms=val_transforms
+
+        self.train_loader=train_loader
+        self.val_loader=val_loader
+        
+    def get_transforms(self, configs, mode='train'):
+        normalize=configs.dataset.normalize
+        pad=configs.dataset.pad
+        resized_size=configs.dataset.resized_size
+        WINDOW_WIDTH=configs.dataset.WINDOW_WIDTH
+        WINDOW_LEVEL=configs.dataset.WINDOW_LEVEL
+        prob=configs.dataset.augmentationProb
+        background=configs.dataset.background
+        indicator_A=configs.dataset.indicator_A
+        indicator_B=configs.dataset.indicator_B
+        load_masks=self.configs.dataset.load_masks
+        transform_list=[]
+        min, max=WINDOW_LEVEL-(WINDOW_WIDTH/2), WINDOW_LEVEL+(WINDOW_WIDTH/2)
+        #transform_list.append(ThresholdIntensityd(keys=[indicator_B], threshold=min, above=True, cval=background))
+        #transform_list.append(ThresholdIntensityd(keys=[indicator_B], threshold=max, above=False, cval=-1000))
+        # filter the source images
+        # transform_list.append(ThresholdIntensityd(keys=[indicator_A], threshold=configs.dataset.MRImax, above=False, cval=0))
+        if normalize=='zscore':
+            transform_list.append(NormalizeIntensityd(keys=[indicator_A, indicator_B], nonzero=False, channel_wise=True))
+            print('zscore normalization')
+        elif normalize=='minmax':
+            transform_list.append(ScaleIntensityd(keys=[indicator_A, indicator_B], minv=-1.0, maxv=1.0))
+            print('minmax normalization')
+
+        elif normalize=='scale4000':
+            transform_list.append(ScaleIntensityd(keys=[indicator_A], minv=0, maxv=1))
+            transform_list.append(ShiftIntensityd(keys=[indicator_B], offset=1024))
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None, factor=-0.99975)) # x=x(1+factor)
+            print('scale4000 normalization')
+
+        elif normalize=='scale1000_wrongbutworks':
+            transform_list.append(ScaleIntensityd(keys=[indicator_A], minv=0, maxv=1))
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=0))
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], factor=-0.999)) 
+            print('scale1000 normalization')
+
+        elif normalize=='scale1000':
+            transform_list.append(ScaleIntensityd(keys=[indicator_A], minv=0, maxv=1))
+            transform_list.append(ShiftIntensityd(keys=[indicator_B], offset=1024))
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None, factor=-0.999)) 
+            print('scale1000 normalization')
+        
+        elif normalize=='scale10':
+            transform_list.append(ScaleIntensityd(keys=[indicator_A], minv=0, maxv=1))
+            #transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=0))
+            transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None,factor=-0.9)) 
+            print('scale10 normalization')
+
+        elif normalize=='inputonlyzscore':
+            transform_list.append(NormalizeIntensityd(keys=[indicator_A], nonzero=False, channel_wise=True))
+            print('only normalize input MRI images')
+
+        elif normalize=='inputonlyminmax':
+            transform_list.append(ScaleIntensityd(keys=[indicator_A], minv=configs.dataset.normmin, maxv=configs.dataset.normmax))
+            print('only normalize input MRI images')
+        
+        elif normalize=='none' or normalize=='nonorm':
+            print('no normalization')
+
+        spaceXY=0
+        if spaceXY>0:
+            transform_list.append(Spacingd(keys=[indicator_A], pixdim=(spaceXY, spaceXY, 2.5), mode="bilinear")) # 
+            transform_list.append(Spacingd(keys=[indicator_B, "mask"] if load_masks else [indicator_B], 
+                                           pixdim=(spaceXY, spaceXY , 2.5), mode="bilinear"))
+        
+        transform_list.append(ResizeWithPadOrCropd(keys=[indicator_A, indicator_B,"mask"] if load_masks else [indicator_A, indicator_B], 
+                                                  spatial_size=resized_size,mode=pad))
+                
+        if configs.dataset.rotate:
+            transform_list.append(Rotate90d(keys=[indicator_A, indicator_B, "mask"] if load_masks else [indicator_A, indicator_B], k=3))
+
+        if mode == 'train':
+            from monai.transforms import (
+                # data augmentation
+                RandRotated,
+                RandZoomd,
+                RandBiasFieldd,
+                RandAffined,
+                RandGridDistortiond,
+                RandGridPatchd,
+                RandShiftIntensityd,
+                RandGibbsNoised,
+                RandAdjustContrastd,
+                RandGaussianSmoothd,
+                RandGaussianSharpend,
+                RandGaussianNoised,
+            )
+            shapeAug=configs.dataset.shapeAug
+            if shapeAug:
+                transform_list.append(RandRotated(keys=[indicator_A, indicator_B, "mask"] if load_masks else [indicator_A, indicator_B],
+                                                  range_x = 0.1, range_y = 0.1, range_z = 0.1, 
+                                                  prob=prob, padding_mode="border", keep_size=True))
+                transform_list.append(RandZoomd(keys=[indicator_A, indicator_B, "mask"] if load_masks else [indicator_A, indicator_B], 
+                                                prob=prob, min_zoom=0.9, max_zoom=1.3,padding_mode= "minimum" ,keep_size=True))
+                transform_list.append(RandAffined(keys=[indicator_A, indicator_B], padding_mode="border" , prob=prob))
+                #transform_list.append(Rand3DElasticd(keys=[indicator_A, indicator_B], prob=prob, sigma_range=(5, 8), magnitude_range=(100, 200), spatial_size=None, mode='bilinear'))
+            intensityAug=configs.dataset.intensityAug
+            if intensityAug:
+                print('intensity data augmentation is used')
+                transform_list.append(RandBiasFieldd(keys=[indicator_A], degree=3, coeff_range=(0.0, 0.1), prob=prob)) # only apply to MRI images
+                transform_list.append(RandGaussianNoised(keys=[indicator_A], prob=prob, mean=0.0, std=0.01))
+                transform_list.append(RandAdjustContrastd(keys=[indicator_A], prob=prob, gamma=(0.5, 1.5)))
+                transform_list.append(RandShiftIntensityd(keys=[indicator_A], prob=prob, offsets=20))
+                transform_list.append(RandGaussianSharpend(keys=[indicator_A], alpha=(0.2, 0.8), prob=prob))
+            
+        #transform_list.append(Rotate90d(keys=[indicator_A, indicator_B], k=3))
+        #transform_list.append(DivisiblePadd(keys=[indicator_A, indicator_B], k=div_size, mode="minimum"))
+        #transform_list.append(Identityd(keys=[indicator_A, indicator_B]))  # do nothing for the no norm case
+        train_transforms = Compose(transform_list)
+        return train_transforms
+
+    def finalcheck(self,ifsave=False,ifcheck=False,iftest_volumes_pixdim=False):
+        if ifsave:
+            self.save_volumes(self.train_ds, self.val_ds, self.saved_name_train, self.saved_name_val)
+        if iftest_volumes_pixdim:
+            self.test_volumes_pixdim(self.train_volume_ds)
+        if ifcheck:
+            self.check_volumes(self.train_ds, self.train_volume_ds, self.val_volume_ds, self.val_ds)
+            self.check_batch_data(self.train_loader,self.val_loader,
+                                  self.train_patch_ds,self.val_volume_ds,
+                                  self.train_batch_size,self.val_batch_size)
+    
+    def test_volumes_pixdim(self, train_volume_ds):
+        train_loader = DataLoader(train_volume_ds, batch_size=1)
+        for step, data in enumerate(train_loader):
+            mr_data=data[self.indicator_A]
+            ct_data=data[self.indicator_B]
+            
+            print(f"source image shape: {mr_data.shape}")
+            print(f"source image affine:\n{mr_data.meta['affine']}")
+            print(f"source image pixdim:\n{mr_data.pixdim}")
+
+            # target image information
+            print(f"target image shape: {ct_data.shape}")
+            print(f"target image affine:\n{ct_data.meta['affine']}")
+            print(f"target image pixdim:\n{ct_data.pixdim}")
+
+    def check_volumes(self, train_ds, train_volume_ds, val_volume_ds, val_ds):
+        # use batch_size=1 to check the volumes because the input volumes have different shapes
+        train_loader = DataLoader(train_volume_ds, batch_size=1)
+        val_loader = DataLoader(val_volume_ds, batch_size=1)
+        train_iterator = iter(train_loader)
+        val_iterator = iter(val_loader)
+        print('check training data:')
+        idx=0
+        for idx in range(len(train_loader)):
+            try:
+                train_check_data = next(train_iterator)
+                ds_idx = idx * 1
+                current_item = train_ds[ds_idx]
+                current_name = os.path.basename(os.path.dirname(current_item['image']))
+                print(idx, current_name, 'image:', train_check_data['image'].shape, 'label:', train_check_data['label'].shape)
+            except:
+                ds_idx = idx * 1
+                current_item = train_ds[ds_idx]
+                current_name = os.path.basename(os.path.dirname(current_item['image']))
+                print('check data error! Check the input data:',current_name)
+        print("checked all training data.")
+
+        print('check validation data:')
+        idx=0
+        for idx in range(len(val_loader)):
+            try:
+                val_check_data = next(val_iterator)
+                ds_idx = idx * 1
+                current_item = val_ds[ds_idx]
+                current_name = os.path.basename(os.path.dirname(current_item['image']))
+                print(idx, current_name, 'image:', val_check_data['image'].shape, 'label:', val_check_data['label'].shape)
+            except:
+                ds_idx = idx * 1
+                current_item = val_ds[ds_idx]
+                current_name = os.path.basename(os.path.dirname(current_item['image']))
+                print('check data error! Check the input data:',current_name)
+        print("checked all validation data.")
+
+    def save_volumes(self, train_ds, val_ds, saved_name_train, saved_name_val):
+        shape_list_train=[]
+        shape_list_val=[]
+        # use the function of saving information before
+        for sample in train_ds:
+            name = os.path.basename(os.path.dirname(sample[self.indicator_A]))
+            shape_list_train.append({'patient': name})
+        for sample in val_ds:
+            name = os.path.basename(os.path.dirname(sample[self.indicator_A]))
+            shape_list_val.append({'patient': name})
+        np.savetxt(saved_name_train,shape_list_train,delimiter=',',fmt='%s',newline='\n') # f means format, r means raw string
+        np.savetxt(saved_name_val,shape_list_val,delimiter=',',fmt='%s',newline='\n') # f means format, r means raw string
+
+    def check_batch_data(self, train_loader,val_loader,train_patch_ds,val_volume_ds,train_batch_size,val_batch_size):
+        for idx, train_check_data in enumerate(train_loader):
+            ds_idx = idx * train_batch_size
+            current_item = train_patch_ds[ds_idx]
+            print('check train data:')
+            print(current_item, 'image:', train_check_data['image'].shape, 'label:', train_check_data['label'].shape)
+        
+        for idx, val_check_data in enumerate(val_loader):
+            ds_idx = idx * val_batch_size
+            current_item = val_volume_ds[ds_idx]
+            print('check val data:')
+            print(current_item, 'image:', val_check_data['image'].shape, 'label:', val_check_data['label'].shape)
+
+    def len_patchloader(self, train_volume_ds,train_batch_size):
+        slice_number=sum(train_volume_ds[i][self.indicator_A].shape[-1] for i in range(len(train_volume_ds)))
+        print('total slices in training set:',slice_number)
+
+        import math
+        batch_number=sum(math.ceil(train_volume_ds[i][self.indicator_A].shape[-1]/train_batch_size) for i in range(len(train_volume_ds)))
+        print('total batches in training set:',batch_number)
+        return slice_number,batch_number
+    
+    def get_length(self, dataset, patch_batch_size):
+        loader=DataLoader(dataset, batch_size=1)
+        iterator = iter(loader)
+        sum_nslices=0
+        for idx in range(len(loader)):
+            check_data = next(iterator)
+            nslices=check_data[self.indicator_A].shape[-1]
+            sum_nslices+=nslices
+        if sum_nslices%patch_batch_size==0:
+            return sum_nslices//patch_batch_size
+        else:
+            return sum_nslices//patch_batch_size+1
+

dataprocesser/archive/slice_loader.py

@@ -0,0 +1,124 @@
+import monai
+from monai.transforms import (
+    Compose,
+    LoadImaged,
+    Rotate90d,
+    ScaleIntensityd,
+    EnsureChannelFirstd,
+    ResizeWithPadOrCropd,
+    DivisiblePadd,
+    ThresholdIntensityd,
+    NormalizeIntensityd,
+    SqueezeDimd,
+    Identityd,
+    CenterSpatialCropd,
+)
+from monai.data import Dataset
+from torch.utils.data import DataLoader
+import torch
+
+from .basics import get_file_list, get_transforms, load_volumes, crop_volumes
+from .checkdata import check_batch_data, check_volumes, save_volumes
+##### slices #####
+def load_batch_slices(train_volume_ds,val_volume_ds, train_batch_size=8,val_batch_size=1,window_width=1,ifcheck=True):
+    patch_func = monai.data.PatchIterd(
+        keys=["source", "target"],
+        patch_size=(None, None, window_width),  # dynamic first two dimensions
+        start_pos=(0, 0, 0)
+    )
+    if window_width==1:
+        patch_transform = Compose(
+            [
+                SqueezeDimd(keys=["source", "target"], dim=-1),  # squeeze the last dim
+            ]
+        )
+    else:
+        patch_transform = None
+    # for training
+    train_patch_ds = monai.data.GridPatchDataset(
+        data=train_volume_ds, patch_iter=patch_func, transform=patch_transform, with_coordinates=False)
+    train_loader = DataLoader(
+        train_patch_ds,
+        batch_size=train_batch_size,
+        num_workers=0,
+        pin_memory=torch.cuda.is_available(),
+    )
+
+    # for validation
+    val_patch_ds = monai.data.GridPatchDataset(
+        data=val_volume_ds, patch_iter=patch_func, transform=patch_transform, with_coordinates=False)
+    val_loader = DataLoader(
+        val_patch_ds, #val_volume_ds, 
+        num_workers=0, 
+        batch_size=val_batch_size,
+        pin_memory=torch.cuda.is_available())
+    
+    if ifcheck:
+        check_batch_data(train_loader,val_loader,train_patch_ds,val_volume_ds,train_batch_size,val_batch_size)
+    return train_loader,val_loader
+def myslicesloader(data_pelvis_path,
+                   normalize='minmax',
+                   pad='minimum',
+                   train_number=1,
+                   val_number=1,
+                   train_batch_size=8,
+                   val_batch_size=1,
+                   saved_name_train='./train_ds_2d.csv',
+                   saved_name_val='./val_ds_2d.csv',
+                   resized_size=(512,512,None),
+                   div_size=(16,16,None),
+                   center_crop=20,
+                   ifcheck_volume=True,
+                   ifcheck_sclices=False,):
+    
+    # volume-level transforms for both image and label
+    train_transforms = get_transforms(normalize,pad,resized_size,div_size,mode='train',prob=0.8)
+    val_transforms = get_transforms(normalize,pad,resized_size,div_size,mode='val')
+    train_ds, val_ds = get_file_list(data_pelvis_path, 
+                                     train_number, 
+                                     val_number)
+    train_crop_ds, val_crop_ds = crop_volumes(train_ds, val_ds,center_crop)
+    train_ds, val_ds = load_volumes(train_transforms, val_transforms,
+                                                train_crop_ds, val_crop_ds, 
+                                                train_ds, val_ds, 
+                                                saved_name_train, saved_name_val,
+                                                ifsave=True,
+                                                ifcheck=ifcheck_volume)
+    train_loader,val_loader = load_batch_slices(train_ds, 
+                                                val_ds, 
+                                                train_batch_size,
+                                                val_batch_size=val_batch_size,
+                                                window_width=1,
+                                                ifcheck=ifcheck_sclices)
+    return train_ds, val_ds, train_loader,val_loader,train_transforms,val_transforms
+
+def len_patchloader(train_volume_ds,train_batch_size):
+    slice_number=sum(train_volume_ds[i]['source'].shape[-1] for i in range(len(train_volume_ds)))
+    print('total slices in training set:',slice_number)
+
+    import math
+    batch_number=sum(math.ceil(train_volume_ds[i]['source'].shape[-1]/train_batch_size) for i in range(len(train_volume_ds)))
+    print('total batches in training set:',batch_number)
+    return slice_number,batch_number
+
+if __name__ == '__main__':
+    dataset_path=r"F:\yang_Projects\Datasets\Task1\pelvis"
+    train_volume_ds,_,train_loader,_,_,_ = myslicesloader(dataset_path,
+                    normalize='none',
+                    train_number=2,
+                    val_number=1,
+                    train_batch_size=4,
+                    val_batch_size=1,
+                    saved_name_train='./train_ds_2d.csv',
+                    saved_name_val='./val_ds_2d.csv',
+                    resized_size=(512, 512, None),
+                    div_size=(16,16,None),
+                    ifcheck_volume=False,
+                    ifcheck_sclices=False,)
+    from tqdm import tqdm
+    parameter_file=r'.\test.txt'
+    for data in tqdm(train_loader):
+         with open(parameter_file, 'a') as f:
+            f.write('image batch:' + str(data["image"].shape)+'\n')
+            f.write('label batch:' + str(data["label"].shape)+'\n')
+            f.write('\n')

dataprocesser/build_dataset.py

@@ -0,0 +1,22 @@
+class BaseDataLoader:
+    def __init__(self, configs, paths=None, dimension=2, **kwargs): 
+        self.configs=configs
+        self.paths=paths
+        self.init_parameters_and_transforms()
+        self.get_loader()
+        #print('all files in dataset:',len(self.source_file_list))
+        
+
+        self.rotation_level = kwargs.get('rotation_level', 0) # Default to no rotation (0)
+        self.zoom_level = kwargs.get('zoom_level', 1.0)  # Default to no zoom (1.0)
+        self.flip = kwargs.get('flip', 0)  # Default to no flip
+
+        self.create_dataset(dimension=dimension)
+
+        ifsave = None if paths is None else True
+        self.finalcheck(ifsave=ifsave,ifcheck=False,iftest_volumes_pixdim=False)
+
+    def get_loader(self):
+        self.source_file_list = []
+        self.train_ds=[]
+        self.val_ds=[]

dataprocesser/config_example.yaml

@@ -0,0 +1,43 @@
+model_name: 'ddpm2d_seg2med'
+GPU_ID: [3]
+ckpt_path: 'logs\241118ddpm_512.pt'
+mode: 'test'
+dataset:
+  train_csv: 'synthrad_conversion\datacsv\ct_synthrad_testrest_newserver.csv'
+  test_csv: 'synthrad_conversion\datacsv\ct_synthrad_testrest_newserver.csv'
+  batch_size: 1
+  val_batch_size: 8
+  normalize: 'scale2000'
+  zoom: (1.0,1.0,1.0)
+  resized_size: (512,512,None)
+  div_size: (None,None,None)
+  WINDOW_WIDTH: 2000
+  WINDOW_LEVEL: 0
+  
+train:
+  val_epoch_interval: 1 
+  save_ckpt_interval: 1
+  num_epochs: 100
+  learning_rate: 0.0002
+  writeTensorboard: True
+  sample_range_lower: 0
+  sample_range_upper: 100000000
+  earlystopping_patience: 10
+  earlystopping_delta: 0.001
+
+validation:
+  evaluate_restore_transforms: True
+  x_lower_limit: -1000
+  x_upper_limit: 3000
+  manual_aorta_diss: -1
+ddpm:
+  num_train_timesteps: 500
+  num_inference_steps: 500
+  num_channels: (64, 128, 256, 256)
+  attention_levels: (False, False, False, True)
+  num_res_units: 2
+  norm_num_groups: 32
+  num_head_channels: 32
+  noise_type: 'normal'
+
+  

dataprocesser/create_csv.py

@@ -0,0 +1,87 @@
+import csv
+from dataprocesser.dataset_anika import (
+    all_list_single_modality_from_anika_dataset_include_duplicate, 
+    extract_patientID_from_Anika_dataset, 
+    all_list_from_anika_dataset_include_duplicate)
+from dataprocesser.dataset_synthrad import list_img_pID_from_synthrad_folder
+from dataprocesser.dataset_anish import list_img_seg_ad_pIDs_from_anish_csv
+from dataprocesser.dataset_dominik import all_list_from_dominik_dataset
+from dataprocesser.step1_init_data_list import appart_img_and_seg, appart_merged_seg
+from dataprocesser.step1_init_data_list import extract_patient_id
+
+def create_csv_combine_lists_synthrad_anika_mr(synthrad_dir, anika_dir_mr, output_mr_csv_file, ifwrtiecsv=True):
+    #synthrad_seg_list, synthrad_pIDs = list_img_pID_from_synthrad_folder(synthrad_dir, ["mr_seg"], None)
+    seg_name_pattern = "mr_merged_seg" #r"^mr_merged_seg_\d{1}[A-Z]{2}\d{3}$"
+    synthrad_seg_list, synthrad_pIDs = list_img_pID_from_synthrad_folder(synthrad_dir, [seg_name_pattern], None)
+    synthrad_mr_list, _ = list_img_pID_from_synthrad_folder(synthrad_dir, ["mr"], None)
+    synthrad_Aorta_diss = [0] * len(synthrad_seg_list)
+    datalist_synthrad = [[id,Aorta_diss,seg,image] for id,Aorta_diss,seg,image in zip(synthrad_pIDs, synthrad_Aorta_diss, synthrad_seg_list, synthrad_mr_list)]
+
+    mr_list = all_list_single_modality_from_anika_dataset_include_duplicate(anika_dir_mr)
+    mr_files, mr_seg_files = appart_img_and_seg(mr_list)
+    mr_seg_files = appart_merged_seg(mr_seg_files)
+    mr_pIDs = extract_patientID_from_Anika_dataset(mr_files)
+
+    mr_Aorta_diss = [0] * len(mr_files)
+    datalist_mr = [[id,Aorta_diss,seg,image] for id,Aorta_diss,seg,image in zip(mr_pIDs, mr_Aorta_diss, mr_seg_files, mr_files)]
+
+    print('length dataset 1: ', len(datalist_synthrad))
+    print('length dataset 2: ', len(datalist_mr))
+    dataset_list=datalist_synthrad+datalist_mr
+    if ifwrtiecsv:
+        create_csv_info_file(dataset_list, output_mr_csv_file) 
+    return dataset_list
+
+def create_csv_info_file(dataset_list, output_mr_csv_file):
+    with open(output_mr_csv_file, 'w', newline='') as f:
+        csvwriter = csv.writer(f)
+        csvwriter.writerow(['id', 'Aorta_diss', 'seg', 'img'])
+        csvwriter.writerows(dataset_list) 
+    
+def create_csv_synthrad_mr(synthrad_dir, output_csv_file):
+    synthrad_seg_list, synthrad_pIDs = list_img_pID_from_synthrad_folder(synthrad_dir, ["mr_merged_seg"], None)
+    synthrad_ct_list, _ = list_img_pID_from_synthrad_folder(synthrad_dir, ["mr"], None)
+    synthrad_Aorta_diss = [0] * len(synthrad_seg_list)
+    datalist_synthrad = [[id,Aorta_diss,seg,image] for id,Aorta_diss,seg,image in zip(synthrad_pIDs, synthrad_Aorta_diss, synthrad_seg_list, synthrad_ct_list)]
+
+    print('length dataset 2: ', len(datalist_synthrad))
+    dataset_list=datalist_synthrad
+    create_csv_info_file(dataset_list, output_csv_file)
+
+def create_csv_combine_lists_synthrad_anish(synthrad_dir, anish_csv, output_csv_file):
+    synthrad_seg_list, synthrad_pIDs = list_img_pID_from_synthrad_folder(synthrad_dir, ["ct_seg"], None)
+    synthrad_ct_list, _ = list_img_pID_from_synthrad_folder(synthrad_dir, ["ct"], None)
+    synthrad_Aorta_diss = [0] * len(synthrad_seg_list)
+    
+    #anish_pIDs, anish_Aorta_diss, anish_seg_list, anish_ct_list = list_img_seg_ad_pIDs_from_new_simplified_csv(anish_csv)
+    anish_pIDs, anish_Aorta_diss, anish_seg_list, anish_ct_list = list_img_seg_ad_pIDs_from_anish_csv(anish_csv)
+    datalist_synthrad = [[id,Aorta_diss,seg,image] for id,Aorta_diss,seg,image in zip(synthrad_pIDs, synthrad_Aorta_diss, synthrad_seg_list, synthrad_ct_list)]
+    datalist_anish = [[id,Aorta_diss,seg,image] for id,Aorta_diss,seg,image in zip(anish_pIDs, anish_Aorta_diss, anish_seg_list, anish_ct_list)]
+
+    print('length dataset 1: ', len(synthrad_ct_list))
+    print('length dataset 2: ', len(datalist_synthrad))
+    dataset_list=datalist_synthrad+datalist_anish
+    create_csv_info_file(dataset_list, output_csv_file)
+
+def create_csv_Anika(ct_dir, mri_dir, output_ct_csv_file, output_mr_csv_file):
+    ct_list, mr_list = all_list_from_anika_dataset_include_duplicate(ct_dir, mri_dir)
+    ct_files, ct_seg_files = appart_img_and_seg(ct_list)
+    ct_pIDs = extract_patientID_from_Anika_dataset(ct_files)
+    ct_Aorta_diss = [0] * len(ct_list)
+    datalist_ct = [[id,Aorta_diss,seg,image] for id,Aorta_diss,seg,image in zip(ct_pIDs, ct_Aorta_diss, ct_seg_files, ct_files)]
+    create_csv_info_file(datalist_ct, output_ct_csv_file)
+
+    mr_files, mr_seg_files = appart_img_and_seg(mr_list)
+    mr_pIDs = extract_patientID_from_Anika_dataset(mr_files)
+    mr_Aorta_diss = [0] * len(mr_files)
+    datalist_mr = [[id,Aorta_diss,seg,image] for id,Aorta_diss,seg,image in zip(mr_pIDs, mr_Aorta_diss, mr_seg_files, mr_files)]
+    create_csv_info_file(datalist_mr, output_mr_csv_file)
+
+def create_csv_Dominik(mri_dir, output_mr_csv_file):
+    mr_list = all_list_from_dominik_dataset(mri_dir)
+    mr_files, mr_seg_files = appart_img_and_seg(mr_list)
+    mr_seg_files = appart_merged_seg(mr_seg_files)
+    mr_pIDs = [extract_patient_id(mr_file) for mr_file in mr_files]
+    mr_Aorta_diss = [0] * len(mr_files)
+    datalist_mr = [[id,Aorta_diss,seg,image] for id,Aorta_diss,seg,image in zip(mr_pIDs, mr_Aorta_diss, mr_seg_files, mr_files)]
+    create_csv_info_file(datalist_mr, output_mr_csv_file)

dataprocesser/create_csv_xcat.py

@@ -0,0 +1,25 @@
+import os
+import csv
+
+def extract_prefixes_from_directory(directory):
+    prefixes = set()
+    for filename in os.listdir(directory):
+        if filename.endswith('.nrrd'):
+            prefix = filename.split('_')[0]
+            prefixes.add(prefix)
+    return sorted(prefixes)
+
+def save_prefixes_to_csv(prefixes, output_csv_path):
+    with open(output_csv_path, mode='w', newline='') as file:
+        writer = csv.writer(file)
+        for prefix in prefixes:
+            writer.writerow([os.path.join(directory, prefix)])
+
+if __name__ == "__main__":
+    directory = r"F:\yang_Projects\ICTUNET_torch\datasets\train"
+    output_csv_path = r"F:\yang_Projects\ICTUNET_torch\data_table\train_all.csv"
+
+    prefixes = extract_prefixes_from_directory(directory)
+    save_prefixes_to_csv(prefixes, output_csv_path)
+
+    print(f"CSV file with prefixes saved to: {output_csv_path}")

dataprocesser/create_json_lodopab.py

@@ -0,0 +1,59 @@
+from tqdm import tqdm
+import json
+
+from configs import config as cfg
+import os
+
+VERBOSE = cfg.verbose
+from collections import defaultdict
+
+IMG_EXTENSIONS = [
+    #'.jpg', '.JPG', '.jpeg', '.JPEG',
+    #'.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP', 
+    '.nrrd', '.nii.gz',
+    '.hdf5',
+]
+
+def is_image_file(filename):
+    return any(filename.endswith(extension) for extension in IMG_EXTENSIONS)
+
+def create_metadata_jsonl_lodopab(base_path, mode='train', output_json_file= 'lodopab_dataset.json'):
+    ground_truth_path = os.path.join(base_path, 'ground_truth_'+mode)
+    observation_path = os.path.join(base_path, 'observation_'+mode)
+
+    # Initialize dataset list
+    dataset_list = []
+
+    # Iterate through the ground truth files
+    for gt_file in os.listdir(ground_truth_path):
+        if is_image_file(gt_file):
+            # Get the corresponding observation file
+            obs_file = gt_file.replace('ground_truth', 'observation')
+            
+            # Create the entry
+            entry = {
+                'ground_truth': os.path.join(ground_truth_path, gt_file),
+                'observation': os.path.join(observation_path, obs_file)
+            }
+            
+            # Append to the dataset list
+            dataset_list.append(entry)
+
+    # Save the dataset list as a JSON file
+    with open(output_json_file, 'w') as json_file:
+        json.dump(dataset_list, json_file, indent=4)
+
+    print(f'Dataset list saved to lodopab_dataset.json with {len(dataset_list)} entries.')
+   
+def read_metadata_jsonl(file_path):
+    with open(file_path, 'r') as f:
+        dataset = json.load(f)
+    return dataset
+
+def print_json_info(data_info):
+        for entry in tqdm(data_info, desc="Calculating slice info"):
+            print(entry['patient_name'])
+            
+if __name__ == '__main__':
+    base_path = r"F:\yang_Projects\Datasets\LoDoPaB"
+    create_metadata_jsonl_lodopab(base_path, mode='train', output_json_file= './data_table/lodopab_dataset.json')

dataprocesser/create_json_xcat.py

@@ -0,0 +1,70 @@
+from tqdm import tqdm
+import json
+
+from configs import config as cfg
+import os
+
+VERBOSE = cfg.verbose
+from collections import defaultdict
+
+IMG_EXTENSIONS = [
+    #'.jpg', '.JPG', '.jpeg', '.JPEG',
+    #'.png', '.PNG', '.ppm', '.PPM', '.bmp', '.BMP', 
+    '.nrrd', '.nii.gz',
+    '.hdf5',
+]
+
+def is_image_file(filename):
+    return any(filename.endswith(extension) for extension in IMG_EXTENSIONS)
+
+def create_metadata_jsonl_xcat(base_path, 
+                               mode='train', 
+                               sino_entry = "_sino_Metal.nrrd",
+                               img_entry = "_img_GT_noNoise.nrrd", 
+                               output_json_file= 'xcat_dataset.json'):
+    
+    train_set_path = os.path.join(base_path, mode)
+    # Initialize dataset list
+    dataset_list = []
+    prefixes = set()
+    for filename in os.listdir(train_set_path):
+        if is_image_file(filename):
+            prefix = filename.split('_')[0]
+            prefixes.add(prefix)
+    prefixes = sorted(prefixes)
+    
+    for prefix in prefixes:
+        sino_path = os.path.join(train_set_path, prefix + sino_entry)
+        img_path = os.path.join(train_set_path, prefix + img_entry)
+
+        # Create the entry
+        entry = {
+            'ground_truth': img_path,
+            'observation': sino_path
+        }
+        
+        # Append to the dataset list
+        dataset_list.append(entry)
+
+    # Save the dataset list as a JSON file
+    with open(output_json_file, 'w') as json_file:
+        json.dump(dataset_list, json_file, indent=4)
+
+    print(f'Dataset list saved to xcat_dataset.json with {len(dataset_list)} entries.')
+   
+def read_metadata_jsonl(file_path):
+    with open(file_path, 'r') as f:
+        dataset = json.load(f)
+    return dataset
+
+def print_json_info(data_info):
+        for entry in tqdm(data_info, desc="Calculating slice info"):
+            print(entry['patient_name'])
+            
+if __name__ == '__main__':
+    base_path = r"F:\yang_Projects\ICTUNET_torch\datasets"
+    create_metadata_jsonl_xcat(base_path, 
+                               mode='train', 
+                               sino_entry = "_sino_Metal.nrrd",
+                               img_entry = "_img_GT_noNoise.nrrd", 
+                               output_json_file= './data_table/xcat_dataset.json')

dataprocesser/customized_datasets.py

@@ -0,0 +1,115 @@
+import torch.utils.data as data
+import nibabel as nib
+import torch
+import os
+import numpy as np
+import torch
+from torch.utils.data import Dataset, DataLoader
+import pandas as pd
+from tqdm import tqdm
+
+VERBOSE = False
+
+
+def volume_slicer(volume_tensor, transform, all_slices=None):
+    # Convert numpy array to PyTorch tensor
+    # Note: You might need to add channel dimension or perform other adjustments
+    volume_tensor = volume_tensor.permute(2, 1, 0) # [H, W, D] -> [D, H, W]
+    volume_tensor = volume_tensor.unsqueeze(1)  # Add channel dimension [D, H, W] -> [D, 1, H, W]
+    if transform is not None:
+        volume_tensor = transform(volume_tensor)
+
+    #print('stacking volume tensor:',volume_tensor.shape)
+    if all_slices is None:
+        all_slices = volume_tensor
+    else:
+        all_slices = torch.cat((all_slices, volume_tensor), 0)
+    return all_slices
+
+def infinite_loader(loader):
+    """Yield batches indefinitely from a DataLoader."""
+    while True:
+        for batch in loader:
+            yield batch
+        # This explicitly resets the iterator
+        loader.dataset.reset()
+        
+class csvDataset_3D(Dataset):
+    def __init__(self, csv_file, transform=None, load_patient_number=1):
+        """
+        Args:
+            csv_file (string): Path to the csv file with annotations.
+            transform (callable, optional): Optional transform to be applied on a sample.
+        """
+        self.data_frame = pd.read_csv(csv_file)
+        # control the length of the dataset
+        self.data_frame = self.data_frame[:load_patient_number]
+        self.transform = transform
+        
+    def __len__(self):
+        return len(self.data_frame)
+
+    def __getitem__(self, idx):
+        if torch.is_tensor(idx):
+            idx = idx.tolist()
+
+        img_path = self.data_frame.iloc[idx, -1]
+        image = nib.load(img_path).get_fdata()
+        image = torch.tensor(image, dtype=torch.float32)
+        
+        # Example: Using the 'Aorta_diss' column as a label
+        label = self.data_frame.iloc[idx, -3]
+        #label = torch.tensor(label, dtype=torch.float32)
+        
+        # If more processing is needed (e.g., normalization, adding channel dimension), do it here
+        image = image.unsqueeze(0)  # Add channel dimension if it's a single channel image
+        
+        sample = {'image': image, 'label': label}
+        
+        return sample
+    
+class csvDataset_2D(Dataset):
+    def __init__(self, csv_file, transform=None, load_patient_number=1):
+        self.csv_file = csv_file
+        self.transform = transform
+        self.load_patient_number = load_patient_number
+        self.data_frame = pd.read_csv(csv_file)
+        if len(self.data_frame) == 0:
+            raise RuntimeError(f"Found 0 images in: {csv_file}")
+        
+        # Initialize dataset
+        self.initialize_dataset()
+
+    def initialize_dataset(self):
+        print('Loading dataset...')
+        self.data_frame = self.data_frame[:self.load_patient_number]
+        all_slices = None
+        all_labels = []
+        
+        for idx in tqdm(range(len(self.data_frame))):
+            img_path = self.data_frame.iloc[idx, -1]
+            volume = nib.load(img_path)
+            volume_data = volume.get_fdata()  # Load as [H, W, D]
+            volume_tensor = torch.tensor(volume_data, dtype=torch.float32)
+            all_slices = volume_slicer(volume_tensor, self.transform, all_slices)  # -> [D, 1, H, W] and pile up all the slices
+            label = self.data_frame.iloc[idx, -3]
+            all_labels = all_labels + [label] * volume_tensor.shape[0]
+        
+        print('All stacked slices:', all_slices.shape)
+        self.all_slices = all_slices
+        self.all_labels = all_labels
+
+    def __len__(self):
+        return self.all_slices.shape[0]
+
+    def __getitem__(self, idx):
+        if torch.is_tensor(idx):
+            idx = idx.tolist()
+        image = self.all_slices[idx]
+        label = self.all_labels[idx]
+        sample = {'source': image, 'target': label}
+        return sample
+
+    def reset(self):
+        print('Resetting dataset...')
+        self.initialize_dataset()

dataprocesser/customized_normalization.py

@@ -0,0 +1,149 @@
+import nibabel as nib
+import numpy as np
+from scipy.interpolate import interp1d
+
+def nyul_apply_standard_scale(input_image,
+                              standard_hist,
+                              input_mask=None,
+                              interp_type='linear'):
+    """
+
+    Based on J.Reinhold code:
+    https://github.com/jcreinhold/intensity-normalization
+
+    Use Nyul and Udupa method ([1,2]) to normalize the intensities
+    of a MRI image passed as input.
+
+    Args:
+        input_image (np.ndarray): input image to normalize
+        standard_hist (str): path to output or use standard histogram landmarks
+        input_mask (nii): optional brain mask
+
+    Returns:
+        normalized (np.ndarray): normalized input image
+
+    References:
+        [1] N. Laszlo G and J. K. Udupa, “On Standardizing the MR Image
+            Intensity Scale,” Magn. Reson. Med., vol. 42, pp. 1072–1081,
+            1999.
+        [2] M. Shah, Y. Xiao, N. Subbanna, S. Francis, D. L. Arnold,
+            D. L. Collins, and T. Arbel, “Evaluating intensity
+            normalization on MRIs of human brain with multiple sclerosis,”
+            Med. Image Anal., vol. 15, no. 2, pp. 267–282, 2011.
+    """
+
+    # load learned standard scale and the percentiles
+    standard_scale, percs = np.load(standard_hist)
+
+    # apply transformation to image
+    return do_hist_normalization(input_image,
+                                 percs,
+                                 standard_scale,
+                                 input_mask,
+                                 interp_type=interp_type)
+
+def do_hist_normalization(input_image,
+                          landmark_percs,
+                          standard_scale,
+                          mask=None,
+                          interp_type='linear'):
+    """
+    do the Nyul and Udupa histogram normalization routine with a given set of
+    learned landmarks
+
+    Based on J.Reinhold code:
+    https://github.com/jcreinhold/intensity-normalization
+
+    Args:
+        input_image (np.ndarray): image on which to find landmarks
+        landmark_percs (np.ndarray): corresponding landmark points of standard scale
+        standard_scale (np.ndarray): landmarks on the standard scale
+        mask (np.ndarray): foreground mask for img
+        interp_type (str): type of interpolation
+
+    Returns:
+        normalized (np.ndarray): normalized image
+    """
+
+    mask_data = input_image > input_image.mean() if mask is None else mask
+    masked = input_image[mask_data > 0]  # extract only part of image where mask is non-emtpy
+    landmarks = get_landmarks(masked, landmark_percs)
+    
+    f = interp1d(landmarks, standard_scale, kind=interp_type, fill_value='extrapolate')  # define interpolating function
+
+    # apply transformation to input image
+    return f(input_image)
+
+def get_landmarks(img, percs):
+    """
+    get the landmarks for the Nyul and Udupa norm method for a specific image
+
+    Based on J.Reinhold code:
+    https://github.com/jcreinhold/intensity-normalization
+
+    Args:
+        img (nibabel.nifti1.Nifti1Image): image on which to find landmarks
+        percs (np.ndarray): corresponding landmark percentiles to extract
+
+    Returns:
+        landmarks (np.ndarray): intensity values corresponding to percs in img
+    """
+    landmarks = np.percentile(img, percs)
+    return landmarks
+
+def nyul_train_standard_scale(img_fns,
+                              mask_fns=None,
+                              i_min=1,
+                              i_max=99,
+                              i_s_min=1,
+                              i_s_max=100,
+                              l_percentile=10,
+                              u_percentile=90,
+                              step=10):
+    """
+    determine the standard scale for the set of images
+
+    Based on J.Reinhold code:
+    https://github.com/jcreinhold/intensity-normalization
+
+
+    Args:
+        img_fns (list): set of NifTI MR image paths which are to be normalized
+        mask_fns (list): set of corresponding masks (if not provided, estimated)
+        i_min (float): minimum percentile to consider in the images
+        i_max (float): maximum percentile to consider in the images
+        i_s_min (float): minimum percentile on the standard scale
+        i_s_max (float): maximum percentile on the standard scale
+        l_percentile (int): middle percentile lower bound (e.g., for deciles 10)
+        u_percentile (int): middle percentile upper bound (e.g., for deciles 90)
+        step (int): step for middle percentiles (e.g., for deciles 10)
+
+    Returns:
+        standard_scale (np.ndarray): average landmark intensity for images
+        percs (np.ndarray): array of all percentiles used
+    """
+
+    # compute masks is those are not entered as a parameters
+    mask_fns = [None] * len(img_fns) if mask_fns is None else mask_fns
+
+    percs = np.concatenate(([i_min],
+                            np.arange(l_percentile, u_percentile+1, step),
+                            [i_max]))
+    standard_scale = np.zeros(len(percs))
+
+    # process each image in order to build the standard scale
+    for i, (img_fn, mask_fn) in enumerate(zip(img_fns, mask_fns)):
+        print('processing scan ', img_fn)
+        img_data = nib.load(img_fn).get_data()  # extract image as numpy array
+        mask = nib.load(mask_fn) if mask_fn is not None else None  # load mask as nibabel object
+        mask_data = img_data > img_data.mean() \
+            if mask is None else mask.get_data()  # extract mask as numpy array
+        masked = img_data[mask_data > 0]  # extract only part of image where mask is non-emtpy
+        landmarks = get_landmarks(masked, percs)
+        min_p = np.percentile(masked, i_min)
+        max_p = np.percentile(masked, i_max)
+        f = interp1d([min_p, max_p], [i_s_min, i_s_max])  # create interpolating function
+        landmarks = np.array(f(landmarks))  # interpolate landmarks
+        standard_scale += landmarks  # add landmark values of this volume to standard_scale
+    standard_scale = standard_scale / len(img_fns)  # get mean values
+    return standard_scale, percs

dataprocesser/customized_transform_list.py

@@ -0,0 +1,149 @@
+from dataprocesser.customized_transforms import (
+    CreateBodyContourTransformd, 
+    MergeMasksTransformd, 
+    UseContourToFilterImaged, 
+    MaskHUAssigmentd, 
+    MergeSegTissueTransformd,
+    NormalizationMultimodal,
+    CreateMaskWithBonesTransformd)
+from monai.transforms import (
+    Compose,
+    LoadImaged,
+    EnsureChannelFirstd,
+    SqueezeDimd,
+    CenterSpatialCropd,
+    Rotate90d,
+    ScaleIntensityd,
+    ResizeWithPadOrCropd,
+    DivisiblePadd,
+    Zoomd,
+    ThresholdIntensityd,
+    NormalizeIntensityd,
+    ShiftIntensityd,
+    Identityd,
+    ScaleIntensityRanged,
+    Spacingd,
+    SaveImage,
+)
+## intensity transforms
+def add_normalization_transform_single_B(transform_list, indicator_B, normalize):
+    if normalize=='zscore':
+        transform_list.append(NormalizeIntensityd(keys=[indicator_B], nonzero=False, channel_wise=True))
+        print('zscore normalization')
+    elif normalize=='minmax':
+        transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=-1.0, maxv=1.0))
+        print('minmax normalization')
+
+    elif normalize=='scale1000_wrongbutworks':
+        transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=0))
+        transform_list.append(ScaleIntensityd(keys=[indicator_B], factor=-0.999)) 
+        print('scale1000 normalization')
+
+    elif normalize=='scale4000':
+        transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None, factor=-0.99975))
+        print('scale4000 normalization')
+
+    elif normalize=='scale2000':
+        transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None, factor=-0.9995))
+        print('scale2000 normalization')
+
+    elif normalize=='scale1000':
+        transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None, factor=-0.999)) 
+        print('scale1000 normalization')
+    
+    elif normalize=='scale100':
+        transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None,factor=-0.99)) 
+        print('scale10 normalization')
+
+    elif normalize=='scale10':
+        transform_list.append(ScaleIntensityd(keys=[indicator_B], minv=None, maxv=None,factor=-0.9)) 
+        print('scale10 normalization')
+
+    elif normalize == 'nonegative':
+        offset=1000
+        transform_list.append(ShiftIntensityd(keys=[indicator_B], offset=offset))
+        print('none negative normalization')
+
+    elif normalize=='none' or normalize=='nonorm':
+        print('no normalization')
+    
+    return transform_list
+
+def add_normalization_multimodal(transform_list, indicator_A, indicator_B):
+    transform_list.append(NormalizationMultimodal(keys=[indicator_A,indicator_B]))
+    return transform_list
+
+def add_normalization_transform_A_B(transform_list, normalize, indicator_A, indicator_B):
+    if normalize=='zscore':
+            transform_list.append(NormalizeIntensityd(keys=[indicator_A,indicator_B], nonzero=False, channel_wise=True))
+            print('zscore normalization')
+    elif normalize=='scale2000':
+        transform_list.append(ScaleIntensityd(keys=[indicator_A,indicator_B], minv=None, maxv=None, factor=-0.9995))
+        print('scale2000 normalization')
+    elif normalize=='none' or normalize=='nonorm':
+        print('no normalization')
+    return transform_list
+
+def add_normalization_transform_input_only(transform_list, indicator_A, normalize):
+    if normalize=='inputonlyzscore':
+        transform_list.append(NormalizeIntensityd(keys=[indicator_A], nonzero=False, channel_wise=True))
+        print('only normalize input MRI images')
+
+    elif normalize=='inputonlyminmax':
+        normmin=0
+        normmax=1
+        transform_list.append(ScaleIntensityd(keys=[indicator_A], minv=normmin, maxv=normmax))
+        print('only normalize input MRI images')
+
+def add_CreateContour_MergeMask_transforms(transform_list, indicator_A):
+    transform_list.append(CreateBodyContourTransformd(keys=['mask'],
+                                                        body_threshold=-500,
+                                                        body_mask_value=1,
+                                                        ))
+    transform_list.append(MergeMasksTransformd(keys=[indicator_A, 'mask']))
+    return transform_list
+
+def add_CreateContour_MergeMask_MaskHUAssign_transforms(transform_list, indicator_A, anatomy_list_csv):
+    transform_list.append(CreateBodyContourTransformd(keys=['mask'],
+                                                        body_threshold=-500,
+                                                        body_mask_value=1,
+                                                        )) # image -> contour
+    transform_list.append(MergeMasksTransformd(keys=[indicator_A, 'mask'])) # seg+contour -> seg
+    transform_list.append(MaskHUAssigmentd(keys=[indicator_A], csv_file=anatomy_list_csv))
+    return transform_list
+
+def add_CreateContour_MergeSegTissue_MergeMask_MaskHUAssign_transforms(transform_list, indicator_A, anatomy_list_csv, anatomy_list_csv_mr):
+    transform_list.append(CreateBodyContourTransformd(keys=['mask'],
+                                                    body_threshold=-500,
+                                                    body_mask_value=1,
+                                                    )) # image -> contour
+    transform_list.append(MergeSegTissueTransformd(keys=[indicator_A, 'seg_tissue'])) # seg+seg_tissue -> seg
+    transform_list.append(MergeMasksTransformd(keys=[indicator_A, 'mask'])) # seg+contour -> seg
+    transform_list.append(MaskHUAssigmentd(keys=[indicator_A], csv_file=anatomy_list_csv))
+    return transform_list
+
+def add_Windowing_ZeroShift_ContourFilter_A_B_transforms(transform_list, WINDOW_LEVEL, WINDOW_WIDTH, indicator_A, indicator_B):
+    threshold_low=WINDOW_LEVEL - WINDOW_WIDTH / 2
+    threshold_high=WINDOW_LEVEL + WINDOW_WIDTH / 2
+    offset=(-1)*threshold_low
+    # if filter out the pixel with values below threshold1, set above=True, and the cval1>=threshold1, otherwise there will be problem
+    # mask = img > self.threshold if self.above else img < self.threshold
+    # res = where(mask, img, self.cval)
+    transform_list.append(ThresholdIntensityd(keys=[indicator_A,indicator_B], threshold=threshold_low, above=True, cval=threshold_low))
+    transform_list.append(ThresholdIntensityd(keys=[indicator_A,indicator_B], threshold=threshold_high, above=False, cval=threshold_high))
+    transform_list.append(ShiftIntensityd(keys=[indicator_A,indicator_B], offset=offset))
+    transform_list.append(UseContourToFilterImaged(keys=[indicator_B, 'mask'])) # image*contour -> image
+    return transform_list
+
+def add_Windowing_ZeroShift_ContourFilter_single_B_transforms(transform_list, WINDOW_LEVEL, WINDOW_WIDTH, indicator_B):
+    threshold_low=WINDOW_LEVEL - WINDOW_WIDTH / 2
+    threshold_high=WINDOW_LEVEL + WINDOW_WIDTH / 2
+    offset=(-1)*threshold_low
+    # if filter out the pixel with values below threshold1, set above=True, and the cval1>=threshold1, otherwise there will be problem
+    # mask = img > self.threshold if self.above else img < self.threshold
+    # res = where(mask, img``, self.cval)
+    transform_list.append(ThresholdIntensityd(keys=[indicator_B], threshold=threshold_low, above=True, cval=threshold_low))
+    transform_list.append(ThresholdIntensityd(keys=[indicator_B], threshold=threshold_high, above=False, cval=threshold_high))
+    transform_list.append(ShiftIntensityd(keys=[indicator_B], offset=offset))
+    transform_list.append(UseContourToFilterImaged(keys=[indicator_B, 'mask'])) # image*contour -> image
+    return transform_list

dataprocesser/customized_transforms.py

@@ -0,0 +1,507 @@
+import cv2
+import numpy as np
+import torch
+from typing import List
+
+VERBOSE = False
+
+def get_data_scaler(config):
+  """Data normalizer. Assume data are always in [0, 1]."""
+  if config.data.centered:
+    # Rescale to [-1, 1]
+    return lambda x: x * 2. - 1.
+  else:
+    return lambda x: x
+
+def get_data_inverse_scaler(config):
+  """Inverse data normalizer."""
+  if config.data.centered:
+    # Rescale [-1, 1] to [0, 1]
+    return lambda x: (x + 1.) / 2.
+  else:
+    return lambda x: x
+  
+def separate_maps(real_images,
+                    tissue_min, tissue_max,
+                    bone_min, bone_max):
+    mask = torch.zeros_like(real_images)
+    # Assign label 1 to tissue regions
+    mask[(real_images > tissue_min) & (real_images <= tissue_max)] = 1
+    # Assign label 2 to bone regions
+    mask[(real_images >= bone_min) & (real_images <= bone_max)] = 2
+    return mask
+
+def create_body_contour_old(tensor_img, body_threshold=-500):
+    """
+    Create a binary body mask from a CT image tensor, using a specific threshold for the body parts.
+    There would be problem if more body parts are presented (like two arms)
+    Args:
+    tensor_img (torch.Tensor): A tensor representation of a grayscale CT image, with intensity values from -1024 to 1500.
+
+    Returns:
+    torch.Tensor: A binary mask tensor where the entire body region is 1 and the background is 0.
+    """
+    # Convert tensor to numpy array
+    numpy_img = tensor_img.numpy().astype(np.int16)  # Ensure we can handle negative values correctly
+
+    # Threshold the image at -500 to separate potential body from the background
+    binary_img = np.where(numpy_img > body_threshold, 1, 0).astype(np.uint8)
+    #print(binary_img.shape)
+    #print(binary_img)
+    # Find contours from the binary image
+    contours, _ = cv2.findContours(binary_img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    # Create an empty mask and fill the largest contour
+    mask = np.zeros_like(binary_img)
+    if contours:
+        # Assume the largest contour is the body contour
+        largest_contour = max(contours, key=cv2.contourArea)
+        cv2.drawContours(mask, [largest_contour], -1, 1, thickness=cv2.FILLED)
+
+    # Convert the mask back to a tensor
+    mask_tensor = torch.tensor(mask, dtype=torch.int32)
+
+    return mask_tensor
+
+def create_body_contour(tensor_img, body_threshold=-500, min_contour_area=10000):
+    """
+    Create a binary body mask from a CT image tensor, using a specific threshold for the body parts.
+    Solve problem that more body parts are presented (like two arms)
+
+    Args:
+    tensor_img (torch.Tensor): A tensor representation of a grayscale CT image, with intensity values from -1024 to 1500.
+
+    Returns:
+    torch.Tensor: A binary mask tensor where the entire body region is 1 and the background is 0.
+    """
+    # Convert tensor to numpy array
+    if isinstance(tensor_img, torch.Tensor):
+        numpy_img = tensor_img.numpy().astype(np.int16)  # Ensure we can handle negative values correctly
+    elif isinstance(tensor_img, np.ndarray):
+        numpy_img = np.ascontiguousarray(tensor_img.astype(np.int16)) 
+    else:
+        print("This is not a PyTorch tensor or a NumPy array. Please Check!")
+    # Threshold the image at -500 to separate potential body from the background
+    binary_img = np.where(numpy_img > body_threshold, 1, 0).astype(np.uint8)
+
+    # Find contours from the binary image
+    contours, _ = cv2.findContours(binary_img, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
+
+    # Create an empty mask
+    mask = np.zeros_like(binary_img)
+
+    # Fill all detected body contours
+    if contours:
+        for contour in contours:
+            if cv2.contourArea(contour) >= min_contour_area:
+                if VERBOSE:
+                    print('current contour area: ', cv2.contourArea(contour), 'threshold: ', min_contour_area)
+                cv2.drawContours(mask, [contour], -1, 1, thickness=cv2.FILLED)
+
+    # Convert the mask back to a tensor
+    mask_tensor = torch.tensor(mask, dtype=torch.int32)
+
+    return mask_tensor
+
+import numpy as np
+import cv2
+
+def create_body_contour_by_seg_tissue(binary_mask: np.ndarray, area_threshold=1000) -> np.ndarray:
+    """
+    提取组织分割图中的身体轮廓（保留最大连通域/多个大区域），输出二值 mask。
+    
+    参数:
+        binary_mask: np.ndarray, 2D 输入图，非 0 为组织区域
+        area_threshold: int, 保留的最小连通域面积
+    返回:
+        contour_mask: np.uint8, 2D binary mask (0 or 1)
+    """
+    mask_uint8 = (binary_mask > 0).astype(np.uint8).copy() * 255
+
+    # 找轮廓（忽略空洞）
+    contours, _ = cv2.findContours(mask_uint8, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+    contour_mask = np.zeros_like(mask_uint8, dtype=np.uint8)
+
+    for cnt in contours:
+        area = cv2.contourArea(cnt)
+        if area > area_threshold:
+            cv2.drawContours(contour_mask, [cnt], -1, 1, thickness=-1)  # 填充轮廓
+
+    return (contour_mask > 0).astype(np.uint8)
+
+import pandas as pd
+
+def HU_assignment(mask, csv_file):
+    if isinstance(mask, torch.Tensor):
+        hu_mask = torch.zeros_like(mask)
+    elif isinstance(mask, np.ndarray):
+        hu_mask = np.zeros_like(mask)
+    
+    df = pd.read_csv(csv_file)
+    hu_values = dict(zip(df['Order Number'], df['HU Value']))
+    order_begin_from_0 = True if df['Order Number'].min()==0 else False
+    # Value Assigment
+    hu_mask[mask == 0] = -1000 # background
+    for organ_index, hu_value in hu_values.items():
+        assert isinstance(hu_value, int), f"Expected mask value an integer, but got {hu_value}. Ensure the mask is created by fine mode of totalsegmentator"
+        assert isinstance(organ_index, int), f"Expected organ_index an integer, but got {organ_index}. Ensure the mask is created by fine mode of totalsegmentator"
+        if order_begin_from_0:
+            hu_mask[mask == (organ_index+1)] = hu_value # mask value begin from 1 as body value, other than 0 in TA2 table, so organ_index+1
+        else:
+            hu_mask[mask == (organ_index)] = hu_value
+    return hu_mask
+
+class MaskHUAssigmentd: 
+    def __init__(self, keys, csv_file):
+        self.keys = keys
+        # Read the CSV into a DataFrame
+        self.df = pd.read_csv(csv_file)
+        #print(self.hu_values)
+
+    def __call__(self, data):
+        # Create a dictionary to map organ index to HU values
+        for key in self.keys:
+            mask = data[key]
+
+            self.hu_values = dict(zip(self.df['Order Number'], self.df['HU Value']))
+            self.order_begin_from_0 = True if self.df['Order Number'].min()==0 else False
+            hu_mask = torch.zeros_like(mask)
+            # Value Assigment
+            hu_mask[mask == 0] = -1000 # background
+            for organ_index, hu_value in self.hu_values.items():
+                assert isinstance(hu_value, int), f"Expected mask value an integer, but got {hu_value}. Ensure the mask is created by fine mode of totalsegmentator"
+                assert isinstance(organ_index, int), f"Expected organ_index an integer, but got {organ_index}. Ensure the mask is created by fine mode of totalsegmentator"
+                if self.order_begin_from_0:
+                    hu_mask[mask == (organ_index+1)] = hu_value # mask value begin from 1 as body value, other than 0 in TA2 table, so organ_index+1
+                else:
+                    hu_mask[mask == (organ_index)] = hu_value
+            data[key] = hu_mask
+        return data
+import pandas as pd
+import numpy as np
+
+def convert_segmentation_mask(source_mask, source_csv, target_csv, body_contour_value=1):
+    """
+    Converts segmentation mask values from source modality to target modality based on organ name mapping.
+
+    Parameters:
+    - source_mask (ndarray): The source segmentation mask array.
+    - source_csv (str): Path to the CSV file of the source modality (CT or MR).
+    - target_csv (str): Path to the CSV file of the target modality (MR or CT).
+    - body_contour_value (int): The class value for "body contour" in the target modality.
+
+    Returns:
+    - target_mask (ndarray): The converted segmentation mask.
+    """
+    # Load the source and target anatomy lists
+    source_df = pd.read_csv(source_csv)
+    target_df = pd.read_csv(target_csv)
+
+    # Create dictionaries mapping class values to organ names and vice versa
+    source_mapping = {row['Organ Name']: row.iloc[0] for _, row in source_df.iterrows()}
+    target_mapping = {row['Organ Name']: row.iloc[0] for _, row in target_df.iterrows()}
+
+    # Initialize the target mask
+    target_mask = np.full_like(source_mask, body_contour_value, dtype=source_mask.dtype)
+
+    # Convert each unique class in the source mask
+    for class_value in np.unique(source_mask):
+        # Find the corresponding organ name in the source modality
+        organ_name = {v: k for k, v in source_mapping.items()}.get(class_value, None)
+
+        # If organ name exists, find the target class value
+        if organ_name and organ_name in target_mapping:
+            target_value = target_mapping[organ_name]
+        else:
+            # Use body contour class value for unmapped organs
+            target_value = body_contour_value
+
+        # Replace class values in the target mask
+        target_mask[source_mask == class_value] = target_value
+
+    return target_mask
+
+class CreateBodyContourTransformd: 
+    def __init__(self, keys, body_threshold,body_mask_value):
+        self.keys = keys
+        self.body_threshold = body_threshold
+        self.body_mask_value = body_mask_value
+
+    def __call__(self, data):
+        # input medical image (CT) and create body contour, then replace the image by contour
+        for key in self.keys:
+            x = data[key]
+            #print(x)
+            mask = torch.zeros_like(x)
+            # [B, H, W, D]
+            # create a mask for each slice in the batch
+            for i in range(x.shape[0]):
+                for j in range(x.shape[-1]):
+                    mask_slice = create_body_contour(x[i,:,:,j], body_threshold=self.body_threshold)
+                    mask[i,:,:, j] = mask_slice
+            mask[mask == 1] = self.body_mask_value
+            if VERBOSE:
+                print("created mask shape:", mask.shape)
+            data[key] = mask
+        return data
+    
+class CreateBodyContourMultiModalTransformd: 
+    def __init__(self, keys, body_threshold,body_mask_value):
+        self.keys = keys
+        self.body_threshold = body_threshold
+        self.body_mask_value = body_mask_value
+
+    def __call__(self, data):
+        # input medical image (CT) and create body contour, then replace the image by contour
+        for key in self.keys:
+            x = data[key]
+            #print(x)
+            mask = torch.zeros_like(x)
+            # [B, H, W, D]
+            # create a mask for each slice in the batch
+            for i in range(x.shape[0]):
+                for j in range(x.shape[-1]):
+                    mask_slice = create_body_contour(x[i,:,:,j], body_threshold=self.body_threshold)
+                    mask[i,:,:, j] = mask_slice
+            mask[mask == 1] = self.body_mask_value
+            if VERBOSE:
+                print("created mask shape:", mask.shape)
+            data[key] = mask
+        return data
+    
+def convert_xcat_to_ct_mask(xcat_image, mapping_csv, tolerance=0.5):
+    """
+    Converts XCAT CT digital phantom images to simulated CT masks.
+
+    Parameters:
+    - xcat_image (torch.Tensor): The XCAT CT image tensor (in HU values).
+    - mapping_csv (str): Path to the CSV file containing organ, HU value, and mask value mappings.
+    - tolerance (float): Tolerance for HU value matching (default is ±0.5).
+
+    Returns:
+    - ct_mask (torch.Tensor): The converted CT mask tensor.
+    """
+    # Load the mapping CSV
+    mapping_df = pd.read_csv(mapping_csv)
+
+    # Initialize the CT mask as a tensor filled with zeros (or another default background value)
+    if isinstance(xcat_image, np.ndarray):
+        ct_mask = np.zeros_like(xcat_image, dtype=np.int32)
+    elif isinstance(xcat_image, torch.Tensor):
+        ct_mask = torch.zeros_like(xcat_image, dtype=torch.int32)
+    else:
+        raise TypeError("xcat_image must be a NumPy ndarray or a PyTorch tensor.")
+
+    # Iterate over the mapping and replace pixel values
+    for _, row in mapping_df.iterrows():
+        organ = row['Organ']
+        hu_value = row['HU_Value']
+        mask_value = row['Mask_Value']
+
+        # Apply the tolerance range for matching
+        lower_bound = hu_value - tolerance
+        upper_bound = hu_value + tolerance
+
+        # Replace matching pixels with the mask value
+        match_condition = (xcat_image >= lower_bound) & (xcat_image <= upper_bound)
+        ct_mask[match_condition] = mask_value
+
+        print(f"Processed {organ} with HU range [{lower_bound}, {upper_bound}] to mask value {mask_value}")
+    return ct_mask
+
+class UseContourToFilterImaged:
+    def __init__(self, 
+                 keys: List[str]
+                 ):
+        if len(keys) != 2:
+            raise ValueError("Keys must be a list with exactly two string elements.")
+        self.image_key = keys[0]
+        self.contour_key = keys[1]
+    def __call__(self, data):
+        image = data[self.image_key]
+        contour = data[self.contour_key]
+        data[self.image_key] = image*contour
+        return data
+    
+class MergeMasksTransformd: 
+    def __init__(self, 
+                 keys: List[str]):
+        if len(keys) != 2:
+            raise ValueError("Keys must be a list with exactly two string elements.")
+        self.seg_key = keys[0]
+        self.contour_key = keys[1]
+
+    def __call__(self, data):
+        seg = data[self.seg_key]
+        contour = data[self.contour_key]
+        merged_mask = seg + contour
+
+        data[self.seg_key] = merged_mask
+        return data
+    
+class MergeSegTissueTransformd: 
+    def __init__(self, 
+                 keys: List[str]):
+        if len(keys) != 2:
+            raise ValueError("Keys must be a list with exactly two string elements.")
+        self.seg_key = keys[0]
+        self.tissue_key = keys[1]
+
+    def __call__(self, data):
+        seg = data[self.seg_key]
+        tissue = data[self.tissue_key]
+        tissue += 100 # keep the tissue value always higher as segmentation organs
+        # Create a mask for overlapping areas
+        overlap_mask = (seg > 0) & (tissue > 0)
+        
+        # For overlapping areas, keep the lower value (organ values in seg)
+        merged_mask = tissue.copy()
+        merged_mask[overlap_mask] = seg[overlap_mask]
+        
+        # Keep all non-overlapping areas
+        merged_mask[seg > 0] = seg[seg > 0]
+
+
+        data[self.seg_key] = merged_mask
+        return data
+    
+class DivideTransformd: 
+    def __init__(self, 
+                 keys: List[str],
+                 divide_factor):
+        self.keys=keys
+        self.divide_factor=divide_factor
+    def __call__(self, data):
+        for key in self.keys:
+            data[key] = data[key]/self.divide_factor
+        return data
+
+class MergeMasksTransformOldd: 
+    def __init__(self, keys):
+        self.keys = keys
+
+    def __call__(self, data):
+        #print('check MergeMasksTransformd:', data)
+        merged_mask = torch.zeros_like(data[self.keys[0]], dtype=torch.int32)
+
+        for key in self.keys:
+            merged_mask += data[key].to(torch.int32)
+        for key in self.keys:
+            data[key] = merged_mask
+        return data
+    
+# convert the integer segemented labels to one-hot codes for training
+class ConvertToOneHotd:
+    def __init__(self, keys, number_classes):
+        self.keys = keys
+        self.nc = number_classes
+
+    def __call__(self, data):
+        for key in self.keys:
+            x = data[key]
+            # Ensure the tensor is of the correct type
+            if x.dtype != torch.long:
+                x = x.long()
+            # Create the one-hot encoded tensor
+            one_hot = torch.zeros(x.size(0), self.nc, x.size(1), x.size(2), device=x.device)
+            one_hot.scatter_(1, x.unsqueeze(1), 1)
+            data[key] = one_hot
+        return data
+
+# Example usage
+# Assuming `ct_image_tensor` is a PyTorch tensor of a CT image
+# ct_image_tensor = torch.tensor(img_array, dtype=torch.float32)
+# mask_tensor = create_body_contour(ct_image_tensor)
+
+class CreateMaskWithBonesTransform:
+    def __init__(self,tissue_min,tissue_max,bone_min,bone_max):
+        # You can add initialization parameters if needed
+        self.tissue_min = tissue_min
+        self.tissue_max = tissue_max
+        self.bone_min = bone_min
+        self.bone_max = bone_max
+
+    def __call__(self, x):
+        # x is the input tensor
+        # Initialize mask with zeros (background)
+        mask = torch.zeros_like(x)
+
+        # Assign label 1 to tissue regions (-500 to 200)
+        mask[(x > self.tissue_min) & (x <= self.tissue_max)] = 1
+
+        # Assign label 2 to bone regions (200 to 1500)
+        mask[(x >= self.bone_min) & (x <= self.bone_max)] = 2
+
+        return mask
+
+class CreateMaskWithBonesTransformd: 
+    def __init__(self, keys, tissue_min, tissue_max, bone_min, bone_max):
+        self.keys = keys
+        self.tissue_min = tissue_min
+        self.tissue_max = tissue_max
+        self.bone_min = bone_min
+        self.bone_max = bone_max
+    def __call__(self, data):
+        for key in self.keys:
+            x = data[key]
+            
+            mask = torch.zeros_like(x)
+            # [B, H, W, D]
+            for i in range(x.shape[0]):
+                for j in range(x.shape[-1]):
+                    mask_slice = create_body_contour(x[i,:,:,j], body_threshold=self.tissue_min)
+                    mask[i,:,:, j] = mask_slice
+            #mask = torch.zeros_like(x)
+            #mask[(x > self.tissue_min) & (x <= self.tissue_max)] = 1
+            mask[(x >= self.bone_min) & (x <= self.bone_max)] = 2
+            data[key] = mask
+            #print("input and mask shape: ",x.shape,data[key].shape)
+        return data
+
+class NormalizationMultimodal:
+    def __init__(self, keys):
+        if len(keys) != 2:
+            raise ValueError("Keys must be a list with exactly two string elements.")
+        self.prior_key = keys[0]
+        self.target_key = keys[1]
+
+        self.prior_modality_norm_dict = {
+            0: {'min': -300, 'max': 700},   # CT WW=1000, WL=200
+            1: {'min': 0, 'max': 9},       # T1
+            2: {'min': 0, 'max': 28},       # T2
+            3: {'min': 0, 'max': 9},       # VIBE-IN
+            4: {'min': 0, 'max': 10},       # VIBE-OPP
+            5: {'min': 0, 'max': 6},       # DIXON
+        }
+
+        self.target_modality_norm_dict = {
+            0: {'min': -300, 'max': 700},   # CT
+            1: {'min': 0, 'max': 800},       # T1
+            2: {'min': 0, 'max': 160},       # T2
+            3: {'min': 0, 'max': 500},       # VIBE-IN
+            4: {'min': 0, 'max': 520},       # VIBE-OPP
+            5: {'min': 0, 'max': 560},       # DIXON
+        }
+    
+    def __call__(self, data):
+        modality = int(data['modality'])  
+        
+        if modality not in self.target_modality_norm_dict:
+            raise ValueError(f"Unsupported modality id: {modality}")
+        
+        # Normalize target
+        x_target = data[self.target_key]
+        target_params = self.target_modality_norm_dict[modality]
+        x_target = torch.clamp(x_target, target_params['min'], target_params['max'])
+        x_target = (x_target - target_params['min']) / (target_params['max'] - target_params['min'])
+        data[self.target_key] = x_target
+
+        # Normalize prior
+        x_prior = data[self.prior_key]
+        prior_params = self.prior_modality_norm_dict[modality]
+        x_prior = torch.clamp(x_prior, prior_params['min'], prior_params['max'])
+        x_prior = (x_prior - prior_params['min']) / (prior_params['max'] - prior_params['min'])
+        data[self.prior_key] = x_prior
+
+        return data
+

dataprocesser/data_processing/.gitignore

@@ -0,0 +1,4 @@
+saved_png
+data
+__pycache__
+*/*.asv

dataprocesser/data_processing/README.md

@@ -0,0 +1,20 @@
+# mydataloader
+dataloader for all projects
+
+--0709 add center_crop in slicer_loader
+
+--0709 test recursively push submodule
+
+--0710 add center_crop in slicer_loader
+
+--0729 add make_cond.py
+
+--0730 add conditional_loader.py
+
+--1103 add input_only normalization in basics.py
+
+--1106 change the place of ResizeWithPadOrCropd into crop_volumes of basics.py to directly get 512*512 reversed output
+
+--1108 merge gan_loader.py together. Change get_file_list in basics.py, replace ct and mr as "source" and "target" 
+
+--1109 delete rotate in basics.py

dataprocesser/data_processing/__init__.py

@@ -0,0 +1,2 @@
+# python module for loading data from monai
+__all__ = ['3d_loader', 'slice_loader', 'slice_loader2', 'basics','manual_slice_loader']

dataprocesser/data_processing/data_process/.gitignore

@@ -0,0 +1 @@
+**pycache**/

dataprocesser/data_processing/data_process/CTbatchevaluate.py

@@ -0,0 +1,49 @@
+import glob
+import os
+from CTevaluate import *
+
+def singleevaluate(file_path,  window_width = 150, window_level = 30):
+    # Load the NIfTI CT image data using nibabel.
+    if file_path.endswith('.nii.gz'):
+        ct_image_nifti = nib.load(file_path)
+        ct_image_data = ct_image_nifti.get_fdata()
+        ct_image_nifti = nib.load(file_path)
+        ct_image_data = ct_image_nifti.get_fdata()
+    elif file_path.endswith('.nrrd'):
+        ct_image_data, header = nrrd.read(file_path)
+    
+    ct_data_shape=ct_image_data.shape
+
+    #plot_ct_value_distribution(ct_image_data)
+    ct_image_data = ct_windowing(ct_image_data, window_width, window_level)
+    #plot_ct_value_distribution(ct_image_data)
+    
+    # cut roi
+    center_x = ct_image_data.shape[0] // 2
+    center_y = ct_image_data.shape[1] // 2
+    ct_image_roi=extract_roi(ct_image_data, center_x=center_x, center_y=center_y, length=300, width=300)
+    ct_image_roi_mean=np.mean(ct_image_roi)
+
+    # Calculate contrast and standard deviation of CT values.
+    contrast = calculate_contrast(ct_image_data)
+    std_deviation = calculate_standard_deviation(ct_image_data)
+    return ct_image_roi_mean, contrast, std_deviation, ct_data_shape
+
+def batchevaluate(dataset_path, format='.nii.gz', save_path='', nii_name='test'):
+    for patient_data in glob.glob(dataset_path + "/*"):
+        if patient_data.endswith(format):
+            patient_name=os.path.basename(os.path.normpath(patient_data))
+            print('-------------', patient_name, '-------------')
+            ct_image_roi_mean, contrast, std_deviation, ct_data_shape = singleevaluate(patient_data)
+            with open(os.path.join(save_path, f'{nii_name}.txt'), 'a') as f:
+                f.write('-------------'+patient_name+'-------------\n')
+                f.write('Mean of CT values in ROI: '+str(ct_image_roi_mean)+'\n')
+                f.write('Contrast of CT image: '+str(contrast)+'\n')
+                f.write('Standard Deviation of CT values: '+str(std_deviation)+'\n')
+                f.write('Size of CT image: '+str(ct_data_shape)+'\n')
+def main():
+    dataset_path=r'D:\Data\dataNeaotomAlpha\NIFTI23072115'
+    batchevaluate(dataset_path=dataset_path, format='.nii.gz', save_path=dataset_path, nii_name='evaluate')
+
+if __name__=="__main__":
+    main()

dataprocesser/data_processing/data_process/CTevaluate.py

@@ -0,0 +1,137 @@
+import numpy as np
+import nibabel as nib
+import nrrd
+
+def extract_roi(ct_image, center_x=256, center_y=256, length=300, width=300):
+    """
+    Extract a Region of Interest (ROI) from the CT image.
+
+    Parameters:
+        ct_image (numpy.ndarray): The CT image data as a 3D NumPy array.
+        center_x (int): X-coordinate of the center of the ROI.
+        center_y (int): Y-coordinate of the center of the ROI.
+        length (int): Length of the square ROI.
+        width (int): Width of the square ROI.
+
+    Returns:
+        numpy.ndarray: The ROI extracted from the CT image.
+    """
+    half_length = length // 2
+    half_width = width // 2
+
+    start_x = max(0, center_x - half_length)
+    end_x = min(ct_image.shape[0], center_x + half_length)
+    start_y = max(0, center_y - half_width)
+    end_y = min(ct_image.shape[1], center_y + half_width)
+
+    return ct_image[:, start_x:end_x, start_y:end_y]
+
+def calculate_contrast(ct_image):
+    """
+    Calculate the contrast of a CT image.
+
+    Parameters:
+        ct_image (numpy.ndarray): The CT image data as a 3D NumPy array.
+
+    Returns:
+        float: The contrast of the CT image.
+    """
+    # Assuming the CT image data ranges from -1024 to 3071 (typical Hounsfield Units range for CT scans)
+    min_value = -1024.0
+    max_value = 3071.0
+    #ct_image_roi=extract_roi(ct_image, center_x=0, center_y=0, length=300, width=300)
+    #ct_image_roi_mean=np.mean(ct_image_roi)
+    contrast = np.abs((np.max(ct_image) - np.min(ct_image))) / (max_value - min_value)
+    return contrast
+
+def calculate_standard_deviation(ct_image):
+    """
+    Calculate the standard deviation of CT values in the image.
+
+    Parameters:
+        ct_image (numpy.ndarray): The CT image data as a 3D NumPy array.
+
+    Returns:
+        float: The standard deviation of CT values.
+    """
+    return np.std(ct_image)
+
+import matplotlib.pyplot as plt
+
+def plot_ct_value_distribution(ct_image):
+    """
+    Plot the distribution of CT values in the image.
+
+    Parameters:
+        ct_image (numpy.ndarray): The CT image data as a 3D NumPy array.
+    """
+    # Flatten the 3D array to a 1D array to get all CT values.
+    ct_values = ct_image.flatten()
+
+    # Create the histogram of CT values.
+    plt.hist(ct_values, bins=100, range=(-1024, 3071), color='blue', alpha=0.7)
+    plt.xlabel('CT Value')
+    plt.ylabel('Frequency')
+    plt.title('Distribution of CT Values')
+    plt.grid(True)
+    plt.show()
+
+def ct_windowing(ct_image, window_width, window_level):
+    """
+    Apply CT windowing to the CT image.
+
+    Parameters:
+        ct_image (numpy.ndarray): The CT image data as a 3D NumPy array.
+        window_width (float): The window width.
+        window_level (float): The window level.
+
+    Returns:
+        numpy.ndarray: The CT image data after applying windowing.
+    """
+    # Calculate the lower and upper bounds of the window.
+    lower_bound = window_level - window_width / 2.0
+    upper_bound = window_level + window_width / 2.0
+
+    # Clip the CT values within the window bounds.
+    ct_image_windowed = np.clip(ct_image, lower_bound, upper_bound)
+
+    return ct_image_windowed
+
+def main():
+    # Replace 'your_ct_image.nii' with the path to your NIfTI CT image file.
+    pcct_path = r'D:\Data\dataNeaotomAlpha\Nifti\2511\2511_2.nii.gz'
+    cbct_path = r'D:\Data\M2OLIE_Phantom\pre_cbct.nrrd'
+    nifti_file_path = pcct_path
+    nrrd_file_path = cbct_path
+
+    # Load the NIfTI CT image data using nibabel.
+    ct_image_nifti = nib.load(nifti_file_path)
+    ct_image_data = ct_image_nifti.get_fdata()
+
+    #ct_image_data, header = nrrd.read(nrrd_file_path)
+
+    window_width = 150
+    window_level = 30
+    #plot_ct_value_distribution(ct_image_data)
+    ct_image_data = ct_windowing(ct_image_data, window_width, window_level)
+    #plot_ct_value_distribution(ct_image_data)
+    
+    # cut roi
+    center_x = ct_image_data.shape[0] // 2
+    center_y = ct_image_data.shape[1] // 2
+    ct_image_roi=extract_roi(ct_image_data, center_x=center_x, center_y=center_y, length=300, width=300)
+    ct_image_roi_mean=np.mean(ct_image_roi)
+
+    # Calculate contrast and standard deviation of CT values.
+    contrast = calculate_contrast(ct_image_data)
+    std_deviation = calculate_standard_deviation(ct_image_data)
+    print(ct_image_data.shape)
+
+    print("size of ROI:", ct_image_roi.shape)
+    print("Mean of CT values in ROI:", ct_image_roi_mean)
+    print("Contrast of CT image:", contrast)
+    print("Standard Deviation of CT values:", std_deviation)
+    
+
+if __name__ == "__main__":
+    main()

dataprocesser/data_processing/data_process/convert_dicoms.py

@@ -0,0 +1,83 @@
+import dicom2nifti
+import SimpleITK as sitk
+from dicom2nifti.exceptions import ConversionValidationError
+from MsSeg.SegmentationNetworkBasis.NetworkBasis import image as Image
+import os
+import glob
+import pydicom
+
+#dataset_path = r"C:\Users\ms97\Documents\MRF-Daten\Messdaten"
+
+def fromrootgroupconvert(dataset_path, nii_name='test'):
+    i=0
+    for patient_folder in glob.glob(dataset_path + "/*/"):
+        print('-------------', patient_folder, '-------------')
+        i=i+1
+        t1_nii_path = os.path.join(dataset_path, patient_folder, f'{nii_name}_{i}.nii')
+        try:
+            try:
+                t1_dicom_path = os.path.join(dataset_path, patient_folder)
+                dicom2nifti.dicom_series_to_nifti(t1_dicom_path, t1_nii_path)
+            except OSError as err:
+                print("Finished for Sequence T1Map " + patient_folder)
+
+            t1_img = sitk.ReadImage(t1_nii_path)
+            data_info = Image.get_data_info(t1_img)
+            print('Data Info T1:  ', data_info)
+        except (KeyError, IndexError) as err:
+            print("Failed for Sequence T1Map " + patient_folder + "  ", err)
+
+def simplepatientconvert(patient_folder, nii_name='test'):
+    t1_nii_path = os.path.join(patient_folder, f'{nii_name}.nii')
+    try:
+        try:
+            dicom2nifti.dicom_series_to_nifti(patient_folder, t1_nii_path)
+        except OSError as err:
+            print("Finished for Sequence Dicom " + patient_folder)
+
+        t1_img = sitk.ReadImage(t1_nii_path)
+        data_info = Image.get_data_info(t1_img)
+        print('Data Info T1:  ', data_info)
+    except (KeyError, IndexError) as err:
+        print("Failed for Sequence Dicom " + patient_folder + "  ", err)
+
+def itkfromrootgroupconvert(dataset_path, nii_name='test'):
+    i=0
+    for patient_folder in glob.glob(dataset_path + "/*/"):
+        print('-------------', patient_folder, '-------------')
+        i=i+1
+        try:
+            try:
+                reader = sitk.ImageSeriesReader()
+                dicom_names = reader.GetGDCMSeriesFileNames(patient_folder)
+                reader.SetFileNames(dicom_names)
+                image = reader.Execute()
+                basefoldername=os.path.basename(os.path.normpath(patient_folder))
+                t1_nii_path = os.path.join(dataset_path, f'{basefoldername}.nii.gz')
+                # Added a call to PermuteAxes to change the axes of the data
+                image = sitk.PermuteAxes(image, [2, 1, 0])
+                sitk.WriteImage(image, t1_nii_path)
+            except OSError as err:
+                print("Finished for Sequence T1Map " + patient_folder)
+
+            t1_img = sitk.ReadImage(t1_nii_path)
+            data_info = Image.get_data_info(t1_img)
+            print('Data Info Dicom:  ', data_info)
+        except (KeyError, IndexError) as err:
+            print("Failed for Sequence Dicom " + patient_folder + "  ", err)
+
+def itkforpatientconvert(patient_folder, nii_name='test'):
+    reader = sitk.ImageSeriesReader()
+    dicom_names = reader.GetGDCMSeriesFileNames(patient_folder)
+    reader.SetFileNames(dicom_names)
+    image = reader.Execute()
+    t1_nii_path = os.path.join(patient_folder, f'{nii_name}.nii.gz')
+    # Added a call to PermuteAxes to change the axes of the data
+    image = sitk.PermuteAxes(image, [2, 1, 0])
+    sitk.WriteImage(image, t1_nii_path)
+
+if __name__=="__main__":
+    dataset_path = r"D:\Data\dataNeaotomAlpha\DICOM_Naeotom\DICOM\23072115"
+    itkfromrootgroupconvert(dataset_path)
+    #patient_folder = r"D:\Data\dataNeaotomAlpha\Q0Q1Q4"
+    #simpleitkconvert(patient_folder,'2511')

dataprocesser/data_processing/data_process/make_cond.py

@@ -0,0 +1,37 @@
+import glob
+import os
+import nibabel as nib
+import numpy as np
+from tqdm import tqdm
+def make_cond(dataset_path):
+    for patient_folder in tqdm(glob.glob(dataset_path + "/*/")):
+        if 'overview' not in patient_folder:
+            ct_file=os.path.join(patient_folder,'ct.nii.gz') 
+            ct_image_nifti = nib.load(ct_file)
+            ct_image_data = ct_image_nifti.get_fdata()
+            ct_slice_number=ct_image_data.shape[-1]
+            ct_slice_label=np.arange(0,ct_slice_number-1,1)
+            # write into csv
+            with open(os.path.join(patient_folder, 'ct_slice_cond.csv'), 'w') as f:
+                f.write('slice\n')
+                for i in range(len(ct_slice_label)):
+                    f.write(str(ct_slice_label[i])+'\n')
+
+            mr_file=os.path.join(patient_folder,'mr.nii.gz')
+            mr_image_nifti = nib.load(mr_file)
+            mr_image_data = mr_image_nifti.get_fdata()
+            mr_slice_number=mr_image_data.shape[-1]
+            mr_slice_label=np.arange(0,mr_slice_number-1,1)
+            # write into csv
+            with open(os.path.join(patient_folder, 'mr_slice_cond.csv'), 'w') as f:
+                f.write('slice\n')
+                for i in range(len(mr_slice_label)):
+                    f.write(str(mr_slice_label[i])+'\n')
+                    
+def main():
+    dataset_path=r'F:\yang_Projects\Datasets\Task1\pelvis'
+    dataset_path_razer=r'C:\Users\56991\Projects\Datasets\Task1\pelvis'
+    make_cond(dataset_path)
+
+if __name__=="__main__":
+    main()

dataprocesser/data_processing/data_process/matlab/BCELossIllustration.m

@@ -0,0 +1,53 @@
+% Create an example image (ground truth)
+images1 = rand(256, 256); % Random matrix as an example
+images1(images1 > 0.5) = 0.9999;
+images1(images1 <= 0.5) = 0.0001;
+
+% Add noise to the image
+noise = randn(size(images1)) * 0.01; % Gaussian noise
+images2 = images1+noise;
+
+% Ensure values are in the range [0, 1]
+% images2 = max(min(images2, 1), 0);
+
+% Plot the images
+subplot(1, 2, 1);
+imshow(images1);
+title('Original Image');
+
+subplot(1, 2, 2);
+imshow(images2);
+title('Noisy Image');
+
+% Calculate BCEWithLogitsLoss
+BCEWithLogitsLoss = calculateBCEWithLogitsLoss(images1, images2);
+disp(['BCEWithLogitsLoss: ', num2str(BCEWithLogitsLoss)]);
+
+BCELoss=calculateBCELoss(images1, images2);
+disp(['BCELoss: ', num2str(BCELoss)]);
+
+function loss = calculateBCELoss(images1, images2)
+    % Convert probabilities to logits
+    logits2 = images2;
+
+    % Calculate BCEWithLogitsLoss
+    loss = mean(mean(-images1 .* log(logits2) - (1 - images1) .* log(1 - logits2)));
+end
+
+function loss = calculateBCEWithLogitsLoss(images1, images2)
+    % Convert probabilities to logits
+    logits2 = images2;
+
+    % Calculate BCEWithLogitsLoss
+    loss = mean(mean(-images1 .* log(sigmoid(logits2)) - (1 - images1) .* log(1 - sigmoid(logits2))));
+end
+
+function logit = probToLogit(p)
+    % Convert probability to logit
+    logit = log(p ./ (1 - p));
+end
+
+function s = sigmoid(x)
+    % Sigmoid function
+    s = 1 ./ (1 + exp(-x));
+end