#coding:utf-8 ''' write by ygq create on 2025-08-26 update MnMs2 data clean nM2数据集的处理逻辑（个人理解，目前是按照这个思路来编写的处理脚本）： 1.LA或者SA需要分开存储处理； 2.ED/ES我理解是舒张|收缩状态的图像信息，只是对应CINE（LA或SA）的某一帧；考虑到没有找到对应的头文件信息，不知道具体对应哪一帧； 3.这个数据集应该不是最原始的MnM2数据集，像是经过某些处理后的；同时没有找到对应的头文件信息； 4.带gt的文件为label标注文件，包含0，1，2，3【0：背景 1：左心室腔（LV）2：右心室腔（RV）3：左心室心肌（Myo)】--需要帮忙确认下 a.需要单独保存LA-CINE以及SA-CINE的重处理后的文件； b.另外需要单独处理LA-ED,LA-ES以及SA-ED,SA-ES的重处理后的文件【spaceing以及size同CINE】；以及label标注文件； ##暂时将LA-ED/ES分开，可以考虑计算每个cine的时次图层的图像均值来判定ED/ES对应的所在帧【试验可行】；--20250825 分割标签：NIFTI 格式，标签值： 0：背景 1：左心室腔（LV） 2：右心室腔（RV） 3：左心室心肌（Myo 当前版本没有元文件信息 ''' import os import glob import pandas as pd import SimpleITK as sitk import argparse import json from tqdm import tqdm from util import meta_data import util import numpy as np # from bert_helper import * TASK_VALUE="segmentation" CLAMP_RANGE_CT = [-300,300] CLAMP_RANGE_MRI = None # MRI images threshold placeholder TBC... TARGET_VOXEL_SPACING=None LABEL_DICT={ "0":"backgroud", "1":"LV",#左心室 Blood Pools "3":"MYO",#左心室心肌 "2":"RV"#右心室 Blood Pools } # def find_metadata_files(path): # # for Cancer Image Archive (TCIA) dataset # search_pattern = os.path.join(path, '**', 'metadata.csv') # return glob.glob(search_pattern, recursive=True) def find_metadata_files(path): # for Cancer Image Archive (TCIA) dataset search_pattern = os.path.join(path, '*.csv') return glob.glob(search_pattern, recursive=True) ##added by yanguoqing on 20250527 def find_image_dirs(path): return os.listdir(path) ##modify by yanguoqing on 20250527 def load_dicom_images(folder_path): reader = sitk.ImageSeriesReader() dicom_names = reader.GetGDCMSeriesFileNames(folder_path) reader.SetFileNames(dicom_names) image = reader.Execute() return dicom_names,image ##added by yanguoqing on 20250527 def load_dicom_tag(imgs): reader = sitk.ImageFileReader() # dicom_names = reader.GetGDCMSeriesFileNames(folder_path) reader.SetFileName(imgs) reader.ReadImageInformation() # 仅读取元信息，不加载像素数据 # metadata_keys = reader.GetMetaDataKeys() tag=reader.Execute() return tag def load_nrrd(fp): return sitk.ReadImage(fp) def save_nifti(image, output_path, folder_path): # Set metadata in the NIfTI file's header output_dirpath = os.path.dirname(output_path) if not os.path.exists(output_dirpath): print(f"Creating directory {output_dirpath}") os.makedirs(output_dirpath) # Set metadata in the NIfTI file's header image.SetMetaData("FolderPath", folder_path) sitk.WriteImage(image, output_path) ##modify by yanguoqing on 20250527 def convert_windows_to_linux_path(windows_path): # Replace backslashes with forward slashes and remove the drive letter # Some meta files have windows paths, but the data is stored on a linux server linux_path = windows_path.replace('\\', '/') if ':' in linux_path: linux_path = linux_path.split(':', 1)[1] return linux_path def main(target_path, output_dir): metadata_files = find_metadata_files(target_path) pid_dirs=find_image_dirs(target_path) # pid_dirs=["Training","Testing","Validation"] failed_files = [] if not os.path.isdir(output_dir): os.makedirs(output_dir) json_output_path = os.path.join(output_dir, 'nifti_mappings.json') failed_files_path = os.path.join(output_dir, 'failed_files.json') meta = meta_data() # Initialize the JSON file if not os.path.exists(json_output_path): with open(json_output_path, 'w') as json_file: json.dump({}, json_file) meta_file=os.path.join(target_path,'211230_M&Ms_Dataset_information_diagnosis_opendataset.csv') if os.path.isfile(meta_file): mf_flag=True df_meta=pd.read_csv(meta_file,sep=',') else: mf_flag=False if pid_dirs: for pid_dir in tqdm(pid_dirs, desc="Processing pid dirs"): if not os.path.isdir(os.path.join(target_path,pid_dir)): continue meta_image_id=pid_dir modality="MRI" study='MnM2'##Dataset_name full_dir=os.path.join(target_path,pid_dir) dfs=find_image_dirs(full_dir)##list all nii.gz files if len(dfs)>0: for df in dfs: ##循环遍历查找SA.LA的CINE以及ES/ED以及对应的gt文件 if "CINE" in df: ##正常处理 label_flag=False if "_LA_" in df: la_flag=True else: la_flag=False elif "ES.nii.gz" in df: if "_LA_" in df: la_flag=True else: la_flag=False if os.path.isfile(os.path.join(full_dir,df.replace(".nii.gz","_gt.nii.gz"))): label_flag=True else: label_flag=False else: continue try: ##处理数据 full_path_image=os.path.join(full_dir,df) sitk_img_original = util.load_nifti(full_path_image) if sitk_img_original is None: print(f" Failed to load image: {full_path_image}") continue original_spacing = list(sitk_img_original.GetSpacing()) original_size = list(sitk_img_original.GetSize()) sitk_img_processed = sitk_img_original # is_4d_image = msd_dataset_info.get("tensorImageSize", "3D").upper() == "4D" or sitk_img_original.GetDimension() == 4 is_4d_image = sitk_img_original.GetDimension() == 4 frame_flag=False # --- Resampling Logic (Revised for 4D) --- if is_4d_image: # Always process 4D images channel-wise for resampling # logging.info(f" Processing 4D image channel-wise: {original_img_full_path}") # Keep log for errors only channels = [] num_channels = original_size[3] if len(original_size) == 4 and sitk_img_original.GetDimension() == 4 else 1 channel_target_spacing = TARGET_VOXEL_SPACING if TARGET_VOXEL_SPACING else original_spacing[:3] # Use 3D spacing for i in range(num_channels): extractor = sitk.ExtractImageFilter() current_3d_channel_size = original_size[:3] if sitk_img_original.GetDimension() == 4: extractor.SetSize([current_3d_channel_size[0], current_3d_channel_size[1], current_3d_channel_size[2], 0]) extractor.SetIndex([0,0,0,i]) channel_3d_img = extractor.Execute(sitk_img_original) else: channel_3d_img = sitk_img_original if i > 0: break channel_resampler = util.get_unisize_resampler( channel_3d_img, 'linear', spacing=channel_target_spacing, size=current_3d_channel_size ) if channel_resampler: channels.append(channel_resampler.Execute(channel_3d_img)) else: channels.append(channel_3d_img) if channels: if len(channels) > 1: # Only join if there are multiple channels sitk_img_processed = sitk.JoinSeriesImageFilter().Execute(channels) ##aded by yanguoqing on 2025-08-11 frame_flag=True # imgDict={} # for kf_idx in range(num_channels): # imgDict[str(kf_idx)]='none' # if str(meta_ed):imgDict[str(meta_ed)]='ed' # if str(meta_es):imgDict[str(meta_es)]='es' # meta.add_keyvalue('ImgDict',imgDict) elif len(channels) == 1: # If only one channel resulted (e.g. original was 3D misidentified as 4D by tensorImageSize) sitk_img_processed = channels[0] elif TARGET_VOXEL_SPACING: # 3D image with target spacing img_resampler_obj = util.get_unisize_resampler(sitk_img_original, 'linear', spacing=TARGET_VOXEL_SPACING, size=original_size) if img_resampler_obj: sitk_img_processed = img_resampler_obj.Execute(sitk_img_original) else: # 3D image, no TARGET_VOXEL_SPACING img_resampler_obj = util.get_unisize_resampler(sitk_img_original, 'linear', spacing=original_spacing, size=original_size) if img_resampler_obj: sitk_img_processed = img_resampler_obj.Execute(sitk_img_original) CIA_other_info = { 'metadata_file':'' # 'Series_Description':serise_desc } CIA_other_info['split'] = "train" CIA_other_info['Image_id']=meta_image_id if mf_flag: CIA_other_info['metadata_file']=meta_file is_processed_4d = sitk_img_processed.GetDimension() == 4 clamp_range_to_use=None if clamp_range_to_use and is_processed_4d: clamped_channels_final = [] num_channels_final = sitk_img_processed.GetSize()[3] if len(sitk_img_processed.GetSize()) == 4 else 1 for i in range(num_channels_final): extractor = sitk.ExtractImageFilter() proc_size_final = sitk_img_processed.GetSize() extractor.SetSize([proc_size_final[0], proc_size_final[1], proc_size_final[2], 0]) extractor.SetIndex([0,0,0,i]) channel_3d_img_to_clamp = extractor.Execute(sitk_img_processed) clamped_channels_final.append(util.clamp_image(channel_3d_img_to_clamp, clamp_range_to_use)) if clamped_channels_final: if len(clamped_channels_final) > 1: sitk_img_processed = sitk.JoinSeriesImageFilter().Execute(clamped_channels_final) elif len(clamped_channels_final) == 1: sitk_img_processed = clamped_channels_final[0] elif clamp_range_to_use: # 3D image sitk_img_processed = util.clamp_image(sitk_img_processed, clamp_range_to_use) output_path = os.path.join(output_dir,pid_dir, f"{df}") # output_path=convert_windows_to_linux_path(output_path) save_nifti(sitk_img_processed, output_path, full_path_image) print(f"Saved NIfTI file to {output_path}") label_path_dict = {} if label_flag: processed_lbl_full_path = os.path.join(output_dir, pid_dir, TASK_VALUE, f"{df}") full_path_label=os.path.join(full_dir,df.replace(".nii.gz","_gt.nii.gz")) sitk_lbl_original = util.load_nifti(full_path_label) if not sitk_lbl_original: print(f" Failed to load label: {full_path_label}") processed_lbl_full_path = None continue if sitk_lbl_original: label_resampler = sitk.ResampleImageFilter() reference_for_label = sitk_img_processed # Default to processed image if sitk_img_processed.GetDimension() == 4: num_comp_proc = sitk_img_processed.GetSize()[3] if len(sitk_img_processed.GetSize()) == 4 else 1 if num_comp_proc > 0: extractor = sitk.ExtractImageFilter() proc_img_size_for_lbl_ref = sitk_img_processed.GetSize() extractor.SetSize([proc_img_size_for_lbl_ref[0], proc_img_size_for_lbl_ref[1], proc_img_size_for_lbl_ref[2], 0]) extractor.SetIndex([0,0,0,0]) try: reference_for_label = extractor.Execute(sitk_img_processed) except Exception as ref_err: print(f" Failed to extract 3D reference from 4D image: {output_path} for label alignment.") # print(traceback.format_exc()) reference_for_label = None else: # Fallback if extraction fails print(f" Could not extract 3D reference for label from 4D image {output_path}. Label may not be correctly resampled.") reference_for_label = None # This will cause an issue below if not handled sitk_lbl_processed = None if reference_for_label and reference_for_label.GetDimension() > 0: label_resampler.SetInterpolator(sitk.sitkNearestNeighbor) label_resampler.SetOutputPixelType(sitk_lbl_original.GetPixelID()) if sitk_lbl_original.GetDimension() == 4: lbl_channels = [] lbl_size = list(sitk_lbl_original.GetSize()) for i in range(lbl_size[3]): extractor = sitk.ExtractImageFilter() extractor.SetSize([lbl_size[0], lbl_size[1], lbl_size[2], 0]) extractor.SetIndex([0, 0, 0, i]) single_channel = extractor.Execute(sitk_lbl_original) label_resampler.SetReferenceImage(reference_for_label) resampled_channel = label_resampler.Execute(single_channel) lbl_channels.append(resampled_channel) if len(lbl_channels) > 1: sitk_lbl_processed = sitk.JoinSeriesImageFilter().Execute(lbl_channels) elif len(lbl_channels) == 1: sitk_lbl_processed = lbl_channels[0] else: label_resampler.SetReferenceImage(reference_for_label) sitk_lbl_processed = label_resampler.Execute(sitk_lbl_original) if processed_lbl_full_path: if sitk_img_processed.GetSize()[:3] != sitk_lbl_processed.GetSize()[:3]: print(f" Mismatch between image and label size (ignoring channels):") print(f" Image size: {sitk_img_processed.GetSize()}") print(f" Label size: {sitk_lbl_processed.GetSize()}") util.save_nifti(sitk_lbl_processed, processed_lbl_full_path, full_path_label) else: print(f" Failed to set reference image for label resampling for {full_path_label}. Saving original label.") util.save_nifti(sitk_lbl_original, processed_lbl_full_path, full_path_label) # Save original # processed_lbl_full_path should still point to this saved original label sitk_lbl_processed=sitk_lbl_original else: processed_lbl_full_path = None else: processed_lbl_full_path = None if processed_lbl_full_path: label_path_dict['heart'] = processed_lbl_full_path print('compare image and label size',sitk_img_original.GetSize(),sitk_lbl_original.GetSize()) print('compare image and label size',sitk_img_processed.GetSize(),sitk_lbl_processed.GetSize()) try: assert sitk_img_processed.GetSize() == sitk_lbl_processed.GetSize() except Exception as e: failed_files.append(full_path_label) continue except RuntimeError: failed_files.append(full_path_image) print(f"Failed to load MnMs images from {full_path_image}") continue size_processed = list(sitk_img_processed.GetSize()) print('size_processed',size_processed,original_size) # meta.add_keyvalue('Image_id',meta_image_id) meta.add_keyvalue('Spacing_mm',min(original_spacing[:3]))##保留前三个x,y,z的最小spacing meta.add_keyvalue('OriImg_path',full_path_image) meta.add_keyvalue('Size',size_processed) # 这里用处理后的size -- YH Jachin meta.add_keyvalue('Modality',modality) meta.add_keyvalue('Dataset_name',study) meta.add_keyvalue('ROI','chest') if processed_lbl_full_path: print(label_path_dict.keys()) meta.add_keyvalue('Task',TASK_VALUE) # meta.add_keyvalue('Label_tissue',list(label_path_dict.keys())) meta.add_keyvalue('Label_path',{TASK_VALUE:label_path_dict}) meta.add_keyvalue('Label_Dict',LABEL_DICT) meta.add_extra_keyvalue('Metadata',CIA_other_info) # Write the mapping to the JSON file on the fly with open(json_output_path, 'r+') as json_file: existing_mappings = json.load(json_file) existing_mappings[output_path] = meta.get_meta_data() json_file.seek(0) print(existing_mappings) json.dump(existing_mappings, json_file, indent=4) json_file.truncate() else: continue with open(failed_files_path, "w") as json_file: json.dump(failed_files, json_file) print(f"The list has been written to {failed_files_path}") print(f"Saved NIfTI mappings to {json_output_path}") if __name__ == "__main__": parser = argparse.ArgumentParser(description="Process DICOM files and save as NIfTI.") parser.add_argument("--target_path", type=str, help="Path to the target directory containing metadata files.", default="/home/data/Github/data/data_gen_def/DATASETS/MnM2/MnM2/dataset/") parser.add_argument("--output_dir", type=str, help="Directory to save the NIfTI files.", default="/home/data/Github/data/data_gen_def/DATASETS_processed/MnM2/") args = parser.parse_args() print(args.target_path, args.output_dir) main(args.target_path, args.output_dir)